Skip to content

Fix OOM when updating very large metadata datasources #390

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
adimiz1 wants to merge 2 commits into
base: master
Choose a base branch
from
Open

Fix OOM when updating very large metadata datasources #390

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
e17f058
Add skip_response_body option and stream-parse API responses to avoid…
adimiz1 Oct 18, 2026
766cae4
Remove memoryTest gradle task, document heap-constrained run in test
adimiz1 Oct 18, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
43 changes: 41 additions & 2 deletions cloudinary-core/src/main/java/com/cloudinary/Api.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -759,8 +759,24 @@ public ApiResponse updateMetadataField(String fieldExternalId, MetadataField fie
* @throws Exception
*/
public ApiResponse updateMetadataFieldDatasource(String fieldExternalId, List<MetadataDataSource.Entry> entries) throws Exception {
return updateMetadataFieldDatasource(fieldExternalId, entries, null);
}

/**
* Update a data source's entries (see {@link #updateMetadataFieldDatasource(String, List)}), with extra options.
*
* @param fieldExternalId The id of the field to update
* @param entries A list of datasource entries. Existing entries (according to entry id) will be updated,
* new entries will be added.
* @param options Extra options. Set {@code skip_response_body} to {@code true} to skip parsing the response
* (which echoes the whole datasource) and return an empty response; recommended for very large
* datasources to avoid excessive memory usage.
* @return The updated field definition, or an empty response if {@code skip_response_body} is set.
* @throws Exception
*/
public ApiResponse updateMetadataFieldDatasource(String fieldExternalId, List<MetadataDataSource.Entry> entries, Map options) throws Exception {
List<String> uri = Arrays.asList("metadata_fields", fieldExternalId, "datasource");
return callApi(HttpMethod.PUT, uri, Collections.singletonMap("values", entries), Collections.singletonMap("content_type", "json"));
return callApi(HttpMethod.PUT, uri, Collections.singletonMap("values", entries), withJsonContentType(options));
}

/**
Expand All @@ -785,8 +801,31 @@ public ApiResponse deleteDatasourceEntries(String fieldExternalId, List<String>
* @throws Exception
*/
public ApiResponse restoreDatasourceEntries(String fieldExternalId, List<String> entriesExternalId) throws Exception {
return restoreDatasourceEntries(fieldExternalId, entriesExternalId, null);
}

/**
* Restore deleted data source entries for a given field, with extra options.
*
* @param fieldExternalId The id of the field to operate
* @param entriesExternalId The ids of all the entries to restore from the data source
* @param options Extra options. Set {@code skip_response_body} to {@code true} to skip parsing the response.
* @return The datasource entries state after restore, or an empty response if {@code skip_response_body} is set.
* @throws Exception
*/
public ApiResponse restoreDatasourceEntries(String fieldExternalId, List<String> entriesExternalId, Map options) throws Exception {
List<String> uri = Arrays.asList("metadata_fields", fieldExternalId, "datasource_restore");
return callApi(HttpMethod.POST, uri, Collections.singletonMap("external_ids", entriesExternalId), Collections.singletonMap("content_type", "json"));
return callApi(HttpMethod.POST, uri, Collections.singletonMap("external_ids", entriesExternalId), withJsonContentType(options));
}

@SuppressWarnings({"rawtypes", "unchecked"})
private static Map withJsonContentType(Map options) {
Map merged = new java.util.HashMap();
if (options != null) {
merged.putAll(options);
}
merged.put("content_type", "json");
return merged;
}

/**
Expand Down
82 changes: 50 additions & 32 deletions cloudinary-http5/src/main/java/com/cloudinary/http5/ApiStrategy.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,11 +24,15 @@
import org.apache.hc.core5.util.Timeout;
import org.cloudinary.json.JSONException;
import org.cloudinary.json.JSONObject;
import org.cloudinary.json.JSONTokener;

import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.lang.reflect.Constructor;
import java.net.URISyntaxException;
import java.nio.charset.StandardCharsets;
import java.util.Collections;
import java.util.List;
import java.util.Map;

Expand Down Expand Up @@ -83,55 +87,69 @@ public ApiResponse callApi(Api.HttpMethod method, String apiUrl, Map<String, ?>

request.setHeader("Authorization", autorizationHeader);

return getApiResponse(request);
return getApiResponse(request, options);
}

private ApiResponse getApiResponse(HttpUriRequestBase request) throws Exception {
String responseData = null;
int code = 0;
CloseableHttpResponse response;
try {
response = client.execute(request);
code = response.getCode();
@SuppressWarnings({"rawtypes", "unchecked"})
private ApiResponse getApiResponse(HttpUriRequestBase request, Map options) throws Exception {
boolean skipResponseBody = options != null && ObjectUtils.asBoolean(options.get("skip_response_body"), false);

try (CloseableHttpResponse response = client.execute(request)) {
int code = response.getCode();
HttpEntity entity = response.getEntity();
if (entity != null) {
responseData = EntityUtils.toString(entity, StandardCharsets.UTF_8);

if (code != 200) {
String responseData = entity != null ? EntityUtils.toString(entity, StandardCharsets.UTF_8) : null;
throw buildError(code, responseData);
}

if (skipResponseBody) {
EntityUtils.consumeQuietly(entity);
return new Response(response, Collections.<String, Object>emptyMap());
}

if (entity == null) {
throw new RuntimeException("Invalid JSON response from server: empty response body");
}
} catch (IOException e) {
throw new GeneralError("Error executing request: " + e.getMessage());
}

if (code != 200) {
// Parse straight from the stream (explicit UTF-8) to avoid buffering the whole body as a String.
Map<String, Object> result;
try {
JSONObject responseJSON = new JSONObject(responseData);
try (InputStream content = entity.getContent()) {
JSONObject responseJSON = new JSONObject(new JSONTokener(new InputStreamReader(content, StandardCharsets.UTF_8)));
result = ObjectUtils.toMap(responseJSON);
} catch (JSONException e) {
if (e.getCause() instanceof IOException) {
// network failure while reading the body: same error type as before streaming the parse
throw new GeneralError("Error executing request: " + e.getMessage());
}
throw new RuntimeException("Invalid JSON response from server " + e.getMessage());
}
EntityUtils.consumeQuietly(entity);

// Extract the error message from the result map
String message = (String) ((Map<String, Object>) result.get("error")).get("message");

// Get the appropriate exception class based on status code
Class<? extends Exception> exceptionClass = Api.CLOUDINARY_API_ERROR_CLASSES.get(code);
if (exceptionClass != null) {
Constructor<? extends Exception> exceptionConstructor = exceptionClass.getConstructor(String.class);
throw exceptionConstructor.newInstance(message);
} else {
throw new GeneralError("Server returned unexpected status code - " + code + " - " + responseData);
}
return new Response(response, result);
} catch (IOException e) {
throw new GeneralError("Error executing request: " + e.getMessage());
}
}

@SuppressWarnings("unchecked")
private Exception buildError(int code, String responseData) throws Exception {
Map<String, Object> result;
try {
JSONObject responseJSON = new JSONObject(responseData);
result = ObjectUtils.toMap(responseJSON);
} catch (JSONException e) {
result = ObjectUtils.toMap(new JSONObject(responseData));
} catch (JSONException | NullPointerException e) {
throw new RuntimeException("Invalid JSON response from server " + e.getMessage());
}

return new Response(response, result);
Object error = result.get("error");
String message = error instanceof Map ? (String) ((Map<String, Object>) error).get("message") : null;

Class<? extends Exception> exceptionClass = Api.CLOUDINARY_API_ERROR_CLASSES.get(code);
if (exceptionClass != null) {
Constructor<? extends Exception> exceptionConstructor = exceptionClass.getConstructor(String.class);
return exceptionConstructor.newInstance(message);
}
return new GeneralError("Server returned unexpected status code - " + code + " - " + responseData);
}

@Override
Expand All @@ -144,7 +162,7 @@ public ApiResponse callAccountApi(Api.HttpMethod method, String apiUrl, Map<Stri
request.setHeader("Authorization", authorizationHeader);

// Execute the request and return the response
return getApiResponse(request);
return getApiResponse(request, options);
}

private HttpUriRequestBase prepareRequest(Api.HttpMethod method, String apiUrl, Map<String, ? extends Object> params, Map<String, ?> options) throws URISyntaxException {
Expand Down
Loading
Loading