diff --git a/cloudinary-core/src/main/java/com/cloudinary/Api.java b/cloudinary-core/src/main/java/com/cloudinary/Api.java index c84be0d6..7cbfa510 100644 --- a/cloudinary-core/src/main/java/com/cloudinary/Api.java +++ b/cloudinary-core/src/main/java/com/cloudinary/Api.java @@ -759,8 +759,24 @@ public ApiResponse updateMetadataField(String fieldExternalId, MetadataField fie * @throws Exception */ public ApiResponse updateMetadataFieldDatasource(String fieldExternalId, List entries) throws Exception { + return updateMetadataFieldDatasource(fieldExternalId, entries, null); + } + + /** + * Update a data source's entries (see {@link #updateMetadataFieldDatasource(String, List)}), with extra options. + * + * @param fieldExternalId The id of the field to update + * @param entries A list of datasource entries. Existing entries (according to entry id) will be updated, + * new entries will be added. + * @param options Extra options. Set {@code skip_response_body} to {@code true} to skip parsing the response + * (which echoes the whole datasource) and return an empty response; recommended for very large + * datasources to avoid excessive memory usage. + * @return The updated field definition, or an empty response if {@code skip_response_body} is set. + * @throws Exception + */ + public ApiResponse updateMetadataFieldDatasource(String fieldExternalId, List entries, Map options) throws Exception { List uri = Arrays.asList("metadata_fields", fieldExternalId, "datasource"); - return callApi(HttpMethod.PUT, uri, Collections.singletonMap("values", entries), Collections.singletonMap("content_type", "json")); + return callApi(HttpMethod.PUT, uri, Collections.singletonMap("values", entries), withJsonContentType(options)); } /** @@ -785,8 +801,31 @@ public ApiResponse deleteDatasourceEntries(String fieldExternalId, List * @throws Exception */ public ApiResponse restoreDatasourceEntries(String fieldExternalId, List entriesExternalId) throws Exception { + return restoreDatasourceEntries(fieldExternalId, entriesExternalId, null); + } + + /** + * Restore deleted data source entries for a given field, with extra options. + * + * @param fieldExternalId The id of the field to operate + * @param entriesExternalId The ids of all the entries to restore from the data source + * @param options Extra options. Set {@code skip_response_body} to {@code true} to skip parsing the response. + * @return The datasource entries state after restore, or an empty response if {@code skip_response_body} is set. + * @throws Exception + */ + public ApiResponse restoreDatasourceEntries(String fieldExternalId, List entriesExternalId, Map options) throws Exception { List uri = Arrays.asList("metadata_fields", fieldExternalId, "datasource_restore"); - return callApi(HttpMethod.POST, uri, Collections.singletonMap("external_ids", entriesExternalId), Collections.singletonMap("content_type", "json")); + return callApi(HttpMethod.POST, uri, Collections.singletonMap("external_ids", entriesExternalId), withJsonContentType(options)); + } + + @SuppressWarnings({"rawtypes", "unchecked"}) + private static Map withJsonContentType(Map options) { + Map merged = new java.util.HashMap(); + if (options != null) { + merged.putAll(options); + } + merged.put("content_type", "json"); + return merged; } /** diff --git a/cloudinary-http5/src/main/java/com/cloudinary/http5/ApiStrategy.java b/cloudinary-http5/src/main/java/com/cloudinary/http5/ApiStrategy.java index 79ecc472..b6f41bd7 100644 --- a/cloudinary-http5/src/main/java/com/cloudinary/http5/ApiStrategy.java +++ b/cloudinary-http5/src/main/java/com/cloudinary/http5/ApiStrategy.java @@ -24,11 +24,15 @@ import org.apache.hc.core5.util.Timeout; import org.cloudinary.json.JSONException; import org.cloudinary.json.JSONObject; +import org.cloudinary.json.JSONTokener; import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; import java.lang.reflect.Constructor; import java.net.URISyntaxException; import java.nio.charset.StandardCharsets; +import java.util.Collections; import java.util.List; import java.util.Map; @@ -83,55 +87,69 @@ public ApiResponse callApi(Api.HttpMethod method, String apiUrl, Map request.setHeader("Authorization", autorizationHeader); - return getApiResponse(request); + return getApiResponse(request, options); } - private ApiResponse getApiResponse(HttpUriRequestBase request) throws Exception { - String responseData = null; - int code = 0; - CloseableHttpResponse response; - try { - response = client.execute(request); - code = response.getCode(); + @SuppressWarnings({"rawtypes", "unchecked"}) + private ApiResponse getApiResponse(HttpUriRequestBase request, Map options) throws Exception { + boolean skipResponseBody = options != null && ObjectUtils.asBoolean(options.get("skip_response_body"), false); + + try (CloseableHttpResponse response = client.execute(request)) { + int code = response.getCode(); HttpEntity entity = response.getEntity(); - if (entity != null) { - responseData = EntityUtils.toString(entity, StandardCharsets.UTF_8); + + if (code != 200) { + String responseData = entity != null ? EntityUtils.toString(entity, StandardCharsets.UTF_8) : null; + throw buildError(code, responseData); + } + + if (skipResponseBody) { + EntityUtils.consumeQuietly(entity); + return new Response(response, Collections.emptyMap()); + } + + if (entity == null) { + throw new RuntimeException("Invalid JSON response from server: empty response body"); } - } catch (IOException e) { - throw new GeneralError("Error executing request: " + e.getMessage()); - } - if (code != 200) { + // Parse straight from the stream (explicit UTF-8) to avoid buffering the whole body as a String. Map result; - try { - JSONObject responseJSON = new JSONObject(responseData); + try (InputStream content = entity.getContent()) { + JSONObject responseJSON = new JSONObject(new JSONTokener(new InputStreamReader(content, StandardCharsets.UTF_8))); result = ObjectUtils.toMap(responseJSON); } catch (JSONException e) { + if (e.getCause() instanceof IOException) { + // network failure while reading the body: same error type as before streaming the parse + throw new GeneralError("Error executing request: " + e.getMessage()); + } throw new RuntimeException("Invalid JSON response from server " + e.getMessage()); } + EntityUtils.consumeQuietly(entity); - // Extract the error message from the result map - String message = (String) ((Map) result.get("error")).get("message"); - - // Get the appropriate exception class based on status code - Class exceptionClass = Api.CLOUDINARY_API_ERROR_CLASSES.get(code); - if (exceptionClass != null) { - Constructor exceptionConstructor = exceptionClass.getConstructor(String.class); - throw exceptionConstructor.newInstance(message); - } else { - throw new GeneralError("Server returned unexpected status code - " + code + " - " + responseData); - } + return new Response(response, result); + } catch (IOException e) { + throw new GeneralError("Error executing request: " + e.getMessage()); } + } + @SuppressWarnings("unchecked") + private Exception buildError(int code, String responseData) throws Exception { Map result; try { - JSONObject responseJSON = new JSONObject(responseData); - result = ObjectUtils.toMap(responseJSON); - } catch (JSONException e) { + result = ObjectUtils.toMap(new JSONObject(responseData)); + } catch (JSONException | NullPointerException e) { throw new RuntimeException("Invalid JSON response from server " + e.getMessage()); } - return new Response(response, result); + Object error = result.get("error"); + String message = error instanceof Map ? (String) ((Map) error).get("message") : null; + + Class exceptionClass = Api.CLOUDINARY_API_ERROR_CLASSES.get(code); + if (exceptionClass != null) { + Constructor exceptionConstructor = exceptionClass.getConstructor(String.class); + return exceptionConstructor.newInstance(message); + } + return new GeneralError("Server returned unexpected status code - " + code + " - " + responseData); } @Override @@ -144,7 +162,7 @@ public ApiResponse callAccountApi(Api.HttpMethod method, String apiUrl, Map params, Map options) throws URISyntaxException { diff --git a/cloudinary-http5/src/test/java/com/cloudinary/test/DatasourceLargeResponseTest.java b/cloudinary-http5/src/test/java/com/cloudinary/test/DatasourceLargeResponseTest.java new file mode 100644 index 00000000..4e845028 --- /dev/null +++ b/cloudinary-http5/src/test/java/com/cloudinary/test/DatasourceLargeResponseTest.java @@ -0,0 +1,222 @@ +package com.cloudinary.test; + +import com.cloudinary.Cloudinary; +import com.cloudinary.api.ApiResponse; +import com.cloudinary.api.exceptions.BadRequest; +import com.cloudinary.api.exceptions.GeneralError; +import com.cloudinary.api.exceptions.NotFound; +import com.cloudinary.metadata.MetadataDataSource; +import com.cloudinary.utils.ObjectUtils; +import com.sun.net.httpserver.HttpServer; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.io.ByteArrayOutputStream; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.InetSocketAddress; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +/** + * Offline tests (local HTTP server) for handling of very large datasource responses. + * Run with a constrained heap (e.g. an IDE run configuration with {@code -Xmx256m}) to act as an OOM regression test: + * with 300k entries (override with {@code -Ddatasource.test.entries}) the pre-fix response handling runs out of memory. + */ +public class DatasourceLargeResponseTest { + private static final int LARGE_ENTRIES = Integer.getInteger("datasource.test.entries", 300_000); + + private HttpServer server; + private Cloudinary cloudinary; + private volatile int status = 200; + private volatile String errorBody; + private volatile int responseEntries = 3; + private volatile boolean truncate = false; + private volatile String rawBody; + private final AtomicInteger receivedEntries = new AtomicInteger(); + + @Before + public void setUp() throws Exception { + server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0); + server.createContext("/", exchange -> { + ByteArrayOutputStream body = new ByteArrayOutputStream(); + try (InputStream in = exchange.getRequestBody()) { + byte[] buf = new byte[8192]; + int n; + while ((n = in.read(buf)) > 0) body.write(buf, 0, n); + } + String req = new String(body.toByteArray(), StandardCharsets.UTF_8); + int count = 0; + for (int i = req.indexOf("\"external_id\""); i >= 0; i = req.indexOf("\"external_id\"", i + 1)) count++; + receivedEntries.set(count); + + exchange.getResponseHeaders().set("Content-Type", "application/json; charset=utf-8"); + exchange.getResponseHeaders().set("X-FeatureRateLimit-Limit", "500"); + exchange.getResponseHeaders().set("X-FeatureRateLimit-Remaining", "499"); + exchange.getResponseHeaders().set("X-FeatureRateLimit-Reset", "Wed, 01 Jan 2031 00:00:00 GMT"); + if (status != 200) { + byte[] err = errorBody.getBytes(StandardCharsets.UTF_8); + exchange.sendResponseHeaders(status, err.length); + try (OutputStream os = exchange.getResponseBody()) { os.write(err); } + return; + } + if (rawBody != null) { + byte[] raw = rawBody.getBytes(StandardCharsets.UTF_8); + exchange.sendResponseHeaders(200, raw.length == 0 ? -1 : raw.length); + if (raw.length > 0) { + try (OutputStream os = exchange.getResponseBody()) { os.write(raw); } + } + exchange.close(); + return; + } + if (truncate) { + // promise more bytes than we send, then drop the connection mid-body + exchange.sendResponseHeaders(200, 1_000_000); + OutputStream os = exchange.getResponseBody(); + os.write("{\"external_id\":\"upc\",\"datasource\":{\"values\":[{\"external_id\":\"a".getBytes(StandardCharsets.UTF_8)); + os.flush(); + exchange.close(); + return; + } + exchange.sendResponseHeaders(200, 0); // chunked, generated on the fly + try (OutputStream os = exchange.getResponseBody()) { + os.write("{\"external_id\":\"upc\",\"datasource\":{\"values\":[".getBytes(StandardCharsets.UTF_8)); + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < responseEntries; i++) { + sb.append(i == 0 ? "" : ",").append("{\"external_id\":\"id").append(i) + .append("\",\"value\":\"vé中").append(i).append("\"}"); + if (sb.length() > 8192) { os.write(sb.toString().getBytes(StandardCharsets.UTF_8)); sb.setLength(0); } + } + os.write(sb.append("]}}").toString().getBytes(StandardCharsets.UTF_8)); + } + }); + server.start(); + Map config = ObjectUtils.asMap( + "cloud_name", "test", "api_key", "key", "api_secret", "secret", + "upload_prefix", "http://127.0.0.1:" + server.getAddress().getPort()); + cloudinary = new Cloudinary(config); + } + + @After + public void tearDown() { + server.stop(0); + } + + private static List entries(int n) { + List list = new ArrayList<>(n); + for (int i = 0; i < n; i++) list.add(new MetadataDataSource.Entry("id" + i, "value" + i)); + return list; + } + + @Test + public void testSmallResponseParsedWithUtf8() throws Exception { + ApiResponse res = cloudinary.api().updateMetadataFieldDatasource("upc", entries(2)); + assertEquals("upc", res.get("external_id")); + List values = (List) ((Map) ((Map) res.get("datasource"))).get("values"); + assertEquals(3, values.size()); + assertEquals("vé中1", ((Map) values.get(1)).get("value")); + assertEquals(2, receivedEntries.get()); + assertEquals(500L, res.apiRateLimit().getLimit()); + } + + @Test + public void testSkipResponseBodyReturnsEmptyResponseWithRateLimits() throws Exception { + responseEntries = LARGE_ENTRIES; + ApiResponse res = cloudinary.api().updateMetadataFieldDatasource("upc", entries(10), + ObjectUtils.asMap("skip_response_body", true)); + assertTrue(res.isEmpty()); + assertEquals(499L, res.apiRateLimit().getRemaining()); + } + + @Test + public void testLargeRequestSentIntact() throws Exception { + cloudinary.api().updateMetadataFieldDatasource("upc", entries(LARGE_ENTRIES), + ObjectUtils.asMap("skip_response_body", true)); + assertEquals(LARGE_ENTRIES, receivedEntries.get()); + } + + @Test + public void testLargeResponseParsed() throws Exception { + responseEntries = LARGE_ENTRIES; + ApiResponse res = cloudinary.api().updateMetadataFieldDatasource("upc", entries(1)); + List values = (List) ((Map) res.get("datasource")).get("values"); + assertEquals(LARGE_ENTRIES, values.size()); + } + + @Test + public void testConnectionDroppedMidBodyThrowsGeneralError() throws Exception { + truncate = true; + try { + cloudinary.api().updateMetadataFieldDatasource("upc", entries(1)); + fail("expected GeneralError"); + } catch (GeneralError e) { + assertTrue(e.getMessage(), e.getMessage().startsWith("Error executing request")); + } + } + + @Test + public void testMalformedJsonThrowsInvalidJsonRuntimeException() throws Exception { + rawBody = "{\"external_id\": not json"; + try { + cloudinary.api().updateMetadataFieldDatasource("upc", entries(1)); + fail("expected RuntimeException"); + } catch (RuntimeException e) { + assertTrue(e.getMessage(), e.getMessage().startsWith("Invalid JSON response from server")); + } + } + + @Test + public void testEmptyBodyThrowsInvalidJsonRuntimeException() throws Exception { + rawBody = ""; + try { + cloudinary.api().updateMetadataFieldDatasource("upc", entries(1)); + fail("expected RuntimeException"); + } catch (RuntimeException e) { + assertTrue(e.getMessage(), e.getMessage().startsWith("Invalid JSON response from server")); + } + } + + @Test + public void testErrorStatusWithNonJsonBodyThrowsInvalidJsonRuntimeException() throws Exception { + status = 500; + errorBody = "Internal Server Error"; + try { + cloudinary.api().updateMetadataFieldDatasource("upc", entries(1)); + fail("expected RuntimeException"); + } catch (RuntimeException e) { + assertTrue(e.getMessage(), e.getMessage().startsWith("Invalid JSON response from server")); + } + } + + @Test + public void testErrorMappedToException() throws Exception { + status = 404; + errorBody = "{\"error\":{\"message\":\"nope\"}}"; + try { + cloudinary.api().updateMetadataFieldDatasource("upc", entries(1)); + fail("expected NotFound"); + } catch (NotFound e) { + assertEquals("nope", e.getMessage()); + } + } + + @Test + public void testErrorWithoutErrorKeyDoesNotCrash() throws Exception { + status = 400; + errorBody = "{\"message\":\"bad\"}"; + try { + cloudinary.api().updateMetadataFieldDatasource("upc", entries(1)); + fail("expected BadRequest"); + } catch (BadRequest e) { + // mapped to proper exception rather than NPE/ClassCastException + } + } +}