diff --git a/.genignore b/.genignore deleted file mode 100644 index cb92e874..00000000 --- a/.genignore +++ /dev/null @@ -1,41 +0,0 @@ -# https://www.speakeasy.com/docs/customize/code/monkey-patching - -# ignore human-written files and directories -_jupyter -_sample_docs -_test_unstructured_client - -# ignore Makefile -Makefile - -# No active endpoints in here, but we don't want to delete it -# Hold onto the custom regions until these endpoints come back -src/unstructured_client/users.py - -# Ignore general.py so we can patch in our partitioning url -# If we ever have a new endpoint under /general, we need to: -# - Comment out this ignore line -# - Generate locally, watch the new endpoint appear -# - Adjust the custom url snippets in the file -# - Bring back the ignore line and commit -src/unstructured_client/general.py - -# Custom min_attempts / absolute_max_elapsed_time_ms fields on BackoffStrategy. -# Push upstream to Speakeasy templates to remove this entry. -src/unstructured_client/utils/retries.py - -# Custom elements_file field on PartitionResponse, for the NDJSON elements-file mode. -# The field is client-side only - the server never returns it - so it cannot come from -# the OpenAPI spec, and regenerating would drop it. If /general/v0/general gains a new -# response field, follow the same procedure as general.py above. -# See test_regeneration_guards.py::test_partition_response_keeps_elements_file. -src/unstructured_client/models/operations/partition.py - -# Docs for that same custom elements_file field. Generated from the spec, so a -# regeneration would drop the row. -# -# Note: SDK generation is currently blocked at the Speakeasy account level -# ("generation access blocked"), so nothing can regenerate today and gen.lock has not -# moved since 2026-01. These entries are insurance for when that is restored, not a -# defence against an imminent run. -docs/models/operations/partitionresponse.md diff --git a/CHANGELOG.md b/CHANGELOG.md index b62d9527..a916ba82 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +## 0.46.2 + +### Fixes +* Accept the API URL the Transform Platform hands you as `server_url`. The app's API Keys page and the docs give you `https://platform-api.transform.unstructured.io/api/v1`, which works with curl but 404'd every Platform call in the SDK: the URL cleaner only stripped a path for `unstructuredapp.io` hosts, so the `/api/v1` survived and the operation's own `/api/v1/jobs/` was appended on top, producing `/api/v1/api/v1/jobs/`. Hosts under `unstructured.io` are now recognized too, and are matched on domain boundaries so a lookalike host like `unstructuredapp.io.example.com` keeps its path and scheme. A `server_url` passed to an individual operation is cleaned as well — previously only the client-level URL was, so `client.jobs.list_jobs(request={}, server_url=...)` still 404'd. + ## 0.46.1 ### Fixes diff --git a/RELEASES.md b/RELEASES.md index d30e16cc..858f1076 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -1261,3 +1261,13 @@ Based on: - [python v0.46.1] . ### Releases - [PyPI v0.46.1] https://pypi.org/project/unstructured-client/0.46.1 - . + +## 2026-08-22 00:00:00 +### Changes +Based on: +- OpenAPI Doc +- Speakeasy CLI 1.601.0 (2.680.0) https://github.com/speakeasy-api/speakeasy +### Generated +- [python v0.46.2] . +### Releases +- [PyPI v0.46.2] https://pypi.org/project/unstructured-client/0.46.2 - . diff --git a/_test_unstructured_client/unit/test_custom_hooks.py b/_test_unstructured_client/unit/test_custom_hooks.py index cc8e980b..38ee694e 100644 --- a/_test_unstructured_client/unit/test_custom_hooks.py +++ b/_test_unstructured_client/unit/test_custom_hooks.py @@ -3,14 +3,15 @@ import logging import re +import httpx import pytest import requests -import httpx -from httpx import Response, ConnectError +from httpx import ConnectError, Response from _test_unstructured_client.unit_utils import FixtureRequest, Mock, method_mock from unstructured_client import UnstructuredClient -from unstructured_client.models import shared, operations +from unstructured_client._hooks.custom.clean_server_url_hook import clean_server_url +from unstructured_client.models import operations, shared from unstructured_client.models.errors import SDKError from unstructured_client.utils.retries import BackoffStrategy, RetryConfig @@ -34,7 +35,10 @@ def test_unit_retry_with_backoff_does_retry(caplog): def mock_post(request): request_count[0] += 1 - if request.url == "https://api.unstructuredapp.io/general/v0/general" and request.method == "POST": + if ( + request.url == "https://api.unstructuredapp.io/general/v0/general" + and request.method == "POST" + ): return Response(502, request=request) transport = httpx.MockTransport(mock_post) @@ -69,7 +73,10 @@ def test_unit_backoff_strategy_logs_retries_5XX(status_code: int, caplog): ) def mock_post(request): - if request.url == "https://api.unstructuredapp.io/general/v0/general" and request.method == "POST": + if ( + request.url == "https://api.unstructuredapp.io/general/v0/general" + and request.method == "POST" + ): return Response(status_code, request=request) transport = httpx.MockTransport(mock_post) @@ -83,11 +90,13 @@ def mock_post(request): partition_parameters=shared.PartitionParameters(files=files) ) - with pytest.raises(Exception): + with pytest.raises(Exception): # noqa: B017 session.general.partition(request=req, retries=retries) - pattern = re.compile(f"Failed to process a request due to API server error with status code {status_code}. " - "Attempting retry number 1 after sleep.") + pattern = re.compile( + f"Failed to process a request due to API server error with status code {status_code}. " + "Attempting retry number 1 after sleep." + ) assert bool(pattern.search(caplog.text)) @@ -103,9 +112,11 @@ def mock_post(request): [502, True], [503, True], [504, True], - ] + ], ) -def test_unit_number_of_retries_in_failed_requests(status_code: int, expect_retry: bool): +def test_unit_number_of_retries_in_failed_requests( + status_code: int, expect_retry: bool +): filename = "README.md" backoff_strategy = BackoffStrategy( initial_interval=1, max_interval=10, exponent=1.5, max_elapsed_time=300 @@ -115,17 +126,19 @@ def test_unit_number_of_retries_in_failed_requests(status_code: int, expect_retr ) number_of_requests = [0] + def mock_post(request): - if request.url == "https://api.unstructuredapp.io/general/v0/general" and request.method == "POST": + if ( + request.url == "https://api.unstructuredapp.io/general/v0/general" + and request.method == "POST" + ): number_of_requests[0] += 1 return Response(status_code, request=request) - transport = httpx.MockTransport(mock_post) client = httpx.Client(transport=transport) session = UnstructuredClient(api_key_auth=FAKE_KEY, client=client) - with open(filename, "rb") as f: files = shared.Files(content=f.read(), file_name=filename) @@ -166,11 +179,13 @@ def mock_post(request): partition_parameters=shared.PartitionParameters(files=files) ) - with pytest.raises(Exception): + with pytest.raises(Exception): # noqa: B017 session.general.partition(request=req, retries=retries) - pattern = re.compile("Failed to process a request due to transport error .*? " - "Attempting retry number 1 after sleep.") + pattern = re.compile( + "Failed to process a request due to transport error .*? " + "Attempting retry number 1 after sleep." + ) assert bool(pattern.search(caplog.text)) @@ -187,31 +202,93 @@ def mock_post(request): ], ) def test_unit_clean_server_url_fixes_malformed_paid_api_url(server_url: str): - client = UnstructuredClient( - server_url=server_url, - api_key_auth=FAKE_KEY, - ) assert ( - client.general.sdk_configuration.server_url + clean_server_url(server_url) == "https://unstructured-000mock.api.unstructuredapp.io" ) +@pytest.mark.parametrize( + "server_url", + [ + # -- the value the Transform Platform's API Keys page hands you -- + "https://platform-api.transform.unstructured.io/api/v1", + "http://platform-api.transform.unstructured.io/api/v1", + "platform-api.transform.unstructured.io/api/v1", + # -- well-formed url -- + "https://platform-api.transform.unstructured.io", + "platform-api.transform.unstructured.io", + ], +) +def test_unit_clean_server_url_fixes_malformed_transform_platform_url(server_url: str): + assert ( + clean_server_url(server_url) == "https://platform-api.transform.unstructured.io" + ) + + +@pytest.mark.parametrize( + "server_url,expected_url", + [ + # -- the terminal root dot is a valid fully qualified name and still ours, so + # -- the path goes; the host is left exactly as the caller wrote it, because the + # -- dot is a deliberate DNS choice that changes the Host header and SNI -- + ( + "https://platform-api.transform.unstructured.io./api/v1", + "https://platform-api.transform.unstructured.io.", + ), + ( + "https://unstructured-000mock.api.unstructuredapp.io./general/v0/general", + "https://unstructured-000mock.api.unstructuredapp.io.", + ), + ], +) +def test_unit_clean_server_url_handles_a_fully_qualified_host( + server_url: str, expected_url: str +): + assert clean_server_url(server_url) == expected_url + + +@pytest.mark.parametrize( + "server_url,expected_url", + [ + # -- a host that merely CONTAINS an Unstructured domain is not ours, so its + # -- path and scheme are left alone -- + ( + "http://unstructuredapp.io.example.com/api/v1", + "http://unstructuredapp.io.example.com/api/v1", + ), + ( + "http://not-unstructured.io/api/v1", + "http://not-unstructured.io/api/v1", + ), + ], +) +def test_unit_clean_server_url_leaves_lookalike_domains_alone( + server_url: str, expected_url: str +): + assert clean_server_url(server_url) == expected_url + + @pytest.mark.parametrize( "server_url,expected_url", [ ("http://localhost:8000", "http://localhost:8000"), ("localhost:8000", "http://localhost:8000"), - ("localhost:8000/general/v0/general", "http://localhost:8000/general/v0/general"), - ("http://localhost:8000/general/v0/general", "http://localhost:8000/general/v0/general"), + ( + "localhost:8000/general/v0/general", + "http://localhost:8000/general/v0/general", + ), + ( + "http://localhost:8000/general/v0/general", + "http://localhost:8000/general/v0/general", + ), ], ) -def test_unit_clean_server_url_fixes_non_unst_domain_url(server_url: str, expected_url: str): - client = UnstructuredClient( - server_url=server_url, - api_key_auth=FAKE_KEY, - ) - assert client.general.sdk_configuration.server_url == expected_url +def test_unit_clean_server_url_fixes_non_unst_domain_url( + server_url: str, expected_url: str +): + assert clean_server_url(server_url) == expected_url + @pytest.mark.parametrize( "server_url", @@ -222,10 +299,11 @@ def test_unit_clean_server_url_fixes_non_unst_domain_url(server_url: str, expect "unstructured-000mock.api.unstructuredapp.io/general/v0/general", ], ) -def test_unit_clean_server_url_fixes_malformed_urls_with_positional_arguments(server_url: str): - client = UnstructuredClient(FAKE_KEY, server_url=server_url) +def test_unit_clean_server_url_fixes_malformed_urls_with_positional_arguments( + server_url: str, +): assert ( - client.general.sdk_configuration.server_url + clean_server_url(server_url) == "https://unstructured-000mock.api.unstructuredapp.io" ) @@ -247,11 +325,8 @@ def mock_post(request): ) with pytest.raises(SDKError, match="API error occurred: Status 401"): session.general.partition(request=req) - - assert any( - "Server responded with 401" - in message for message in caplog.messages - ) + + assert any("Server responded with 401" in message for message in caplog.messages) # -- fixtures -------------------------------------------------------------------------------- diff --git a/_test_unstructured_client/unit/test_regeneration_guards.py b/_test_unstructured_client/unit/test_regeneration_guards.py deleted file mode 100644 index 62dc76a1..00000000 --- a/_test_unstructured_client/unit/test_regeneration_guards.py +++ /dev/null @@ -1,149 +0,0 @@ -import re -from pathlib import Path -import tomllib - -from unstructured_client.models import shared -from unstructured_client.utils.forms import serialize_multipart_form - - -REPO_ROOT = Path(__file__).resolve().parents[2] - - -def _load_pyproject() -> dict: - return tomllib.loads((REPO_ROOT / "pyproject.toml").read_text()) - - -def test_pyproject_invariants(): - data = _load_pyproject() - project = data["project"] - - assert project["dynamic"] == ["version"] - assert "version" not in project - assert project["requires-python"] == ">=3.11" - assert "httpcore >=1.0.9" in project["dependencies"] - assert "pydantic >=2.12.5" in project["dependencies"] - assert not any("cryptography" in d for d in project["dependencies"]), \ - "cryptography is unused and must not be a runtime dependency" - - dynamic_version = data["tool"]["setuptools"]["dynamic"]["version"] - assert dynamic_version == {"attr": "unstructured_client._version.__version__"} - - build = data["build-system"] - assert build["build-backend"] == "setuptools.build_meta" - assert "setuptools>=80" in build["requires"] - - -def test_publish_script_is_hardened(): - publish_script = (REPO_ROOT / "scripts" / "publish.sh").read_text() - - assert "set -euo pipefail" in publish_script - assert "sys.version_info < (3, 11)" in publish_script - assert "uv build --out-dir dist --clear" in publish_script - - -def test_release_workflow_uses_trusted_publishing(): - workflow = (REPO_ROOT / ".github" / "workflows" / "speakeasy_sdk_publish.yaml").read_text() - - assert "release:" in workflow - assert "pypa/gh-action-pypi-publish" in workflow - assert "PYPI_TOKEN" not in workflow - assert "upload-artifact" in workflow - assert "download-artifact" in workflow - assert re.search(r"publish:\n\s+needs: build", workflow) - assert re.search(r"publish:\n(?:.*\n)*?\s+permissions:\n\s+contents: read\n\s+id-token: write", workflow) - - -def test_release_workflow_keeps_oidc_out_of_build_job(): - workflow = (REPO_ROOT / ".github" / "workflows" / "speakeasy_sdk_publish.yaml").read_text() - - build_job = workflow.split("\n publish:\n", maxsplit=1)[0] - - assert "id-token: write" not in build_job - - -def test_speakeasy_workflow_does_not_manage_pypi_publishing(): - workflow = (REPO_ROOT / ".speakeasy" / "workflow.yaml").read_text() - - assert "publish:" not in workflow - assert "PYPI_TOKEN" not in workflow - - -def test_makefile_installs_with_locked_uv_sync(): - makefile = (REPO_ROOT / "Makefile").read_text() - - assert "uv sync --locked" in makefile - - -def test_ci_installs_with_locked_uv_sync(): - workflow = (REPO_ROOT / ".github" / "workflows" / "ci.yaml").read_text() - - assert 'UV_LOCKED: "1"' in workflow - assert "run: make install" in workflow - - -def test_partition_response_keeps_elements_file(): - """`elements_file` is client-side only, so no spec change can restore it after a regen. - - Both the model and the enum value that selects it live in generated files; the - .genignore entries are the only thing keeping them. - - Note: SDK generation is currently blocked at the Speakeasy account level, so this - guards a path that cannot execute today. Whether it is worth keeping is a live - question -- see the discussion on PR #347. - """ - from unstructured_client.general import PartitionAcceptEnum - from unstructured_client.models import operations - - assert "elements_file" in operations.PartitionResponse.model_fields - assert "elements_file" in operations.PartitionResponseTypedDict.__annotations__ - assert PartitionAcceptEnum.APPLICATION_X_NDJSON.value == "application/x-ndjson" - - genignore = (REPO_ROOT / ".genignore").read_text() - for path in ( - "src/unstructured_client/general.py", - "src/unstructured_client/models/operations/partition.py", - "docs/models/operations/partitionresponse.md", - ): - assert path in genignore, f"{path} carries custom code and must stay in .genignore" - - # The docs row is generated from the spec too, so a regeneration would drop it - # without the .genignore entry above. - response_docs = (REPO_ROOT / "docs/models/operations/partitionresponse.md").read_text() - assert "elements_file" in response_docs - - -def test_body_create_job_input_files_are_serialized_as_multipart_files(): - request = shared.BodyCreateJob( - request_data="{}", - input_files=[ - shared.InputFiles( - content=b"hello", - file_name="hello.pdf", - content_type="application/pdf", - ) - ], - ) - - media_type, form, files = serialize_multipart_form("multipart/form-data", request) - - assert media_type == "multipart/form-data" - assert form == {"request_data": "{}"} - assert files == [("input_files[]", ("hello.pdf", b"hello", "application/pdf"))] - - -def test_body_run_workflow_input_files_are_serialized_as_multipart_files(): - request = shared.BodyRunWorkflow( - input_files=[ - shared.BodyRunWorkflowInputFiles( - content=b"hello", - file_name="hello.pdf", - content_type="application/pdf", - ) - ] - ) - - media_type, form, files = serialize_multipart_form("multipart/form-data", request) - - assert media_type == "multipart/form-data" - assert form == {} - assert files == [("input_files[]", ("hello.pdf", b"hello", "application/pdf"))] diff --git a/_test_unstructured_client/unit/test_server_urls.py b/_test_unstructured_client/unit/test_server_urls.py index 710891b6..091dc5ce 100644 --- a/_test_unstructured_client/unit/test_server_urls.py +++ b/_test_unstructured_client/unit/test_server_urls.py @@ -1,7 +1,9 @@ -import pytest from dataclasses import dataclass + +import pytest + from unstructured_client import UnstructuredClient, utils -from typing import Optional + # Raise one of these from our mock to return to the test code class BaseUrlCorrect(Exception): @@ -13,10 +15,7 @@ class BaseUrlIncorrect(Exception): def get_client_method_with_mock( - sdk_endpoint_name, - client_instance, - mocked_server_url, - monkeypatch + sdk_endpoint_name, client_instance, mocked_server_url, monkeypatch ): """ Given an endpoint name, e.g. "general.partition", return a reference @@ -26,6 +25,7 @@ def get_client_method_with_mock( Assert that the provided server_url is passed into _build_request. Raise a custom exception to get back to the test. """ + # Mock this to get past param validation def mock_unmarshal(*args, **kwargs): return {} @@ -51,6 +51,7 @@ def mock_build_request(*args, base_url, **kwargs): return endpoint_method + @dataclass class URLTestCase: description: str @@ -58,9 +59,10 @@ class URLTestCase: # expected url when actually making the HTTP request in build_request expected_url: str # url when you init the client (global for all endpoints) - client_url: Optional[str] = None + client_url: str | None = None # url when you init the SDK endpoint (vary per endpoint) - endpoint_url: Optional[str] = None + endpoint_url: str | None = None + @pytest.mark.asyncio @pytest.mark.parametrize( @@ -71,37 +73,37 @@ class URLTestCase: sdk_endpoint_name="general.partition_async", client_url="http://localhost:8000/", endpoint_url=None, - expected_url="http://localhost:8000" + expected_url="http://localhost:8000", ), URLTestCase( description="custom client-level URL, with path", sdk_endpoint_name="general.partition_async", client_url="http://localhost:8000/my/endpoint/", endpoint_url=None, - expected_url="http://localhost:8000/my/endpoint" + expected_url="http://localhost:8000/my/endpoint", ), URLTestCase( description="custom endpoint-level URL, no path", sdk_endpoint_name="general.partition_async", client_url=None, endpoint_url="http://localhost:8000/", - expected_url="http://localhost:8000" + expected_url="http://localhost:8000", ), URLTestCase( description="custom endpoint-level URL, with path", sdk_endpoint_name="general.partition_async", client_url=None, endpoint_url="http://localhost:8000/my/endpoint/", - expected_url="http://localhost:8000/my/endpoint" + expected_url="http://localhost:8000/my/endpoint", ), URLTestCase( description="default URL fallback", sdk_endpoint_name="general.partition_async", client_url=None, endpoint_url=None, - expected_url="https://api.unstructuredapp.io" + expected_url="https://api.unstructuredapp.io", ), - ] + ], ) async def test_async_endpoint_uses_correct_url(monkeypatch, case: URLTestCase): if case.client_url: @@ -110,10 +112,7 @@ async def test_async_endpoint_uses_correct_url(monkeypatch, case: URLTestCase): s = UnstructuredClient() client_method = get_client_method_with_mock( - case.sdk_endpoint_name, - s, - case.expected_url, - monkeypatch + case.sdk_endpoint_name, s, case.expected_url, monkeypatch ) try: @@ -124,9 +123,7 @@ async def test_async_endpoint_uses_correct_url(monkeypatch, case: URLTestCase): except BaseUrlCorrect: pass except BaseUrlIncorrect as e: - pytest.fail( - f"{case.description}: Expected {case.expected_url}, got {e}" - ) + pytest.fail(f"{case.description}: Expected {case.expected_url}, got {e}") @pytest.mark.parametrize( @@ -137,79 +134,86 @@ async def test_async_endpoint_uses_correct_url(monkeypatch, case: URLTestCase): sdk_endpoint_name="destinations.create_destination", client_url="http://localhost:8000/", endpoint_url=None, - expected_url="http://localhost:8000" + expected_url="http://localhost:8000", ), URLTestCase( description="custom client-level URL, with path", sdk_endpoint_name="sources.create_source", client_url="http://localhost:8000/my/endpoint/", endpoint_url=None, - expected_url="http://localhost:8000/my/endpoint" + expected_url="http://localhost:8000/my/endpoint", ), URLTestCase( description="custom endpoint-level URL, no path", sdk_endpoint_name="jobs.get_job", client_url=None, endpoint_url="http://localhost:8000", - expected_url="http://localhost:8000" + expected_url="http://localhost:8000", ), URLTestCase( description="custom endpoint-level URL, with path", sdk_endpoint_name="workflows.create_workflow", client_url=None, endpoint_url="http://localhost:8000/my/endpoint", - expected_url="http://localhost:8000/my/endpoint" + expected_url="http://localhost:8000/my/endpoint", + ), + URLTestCase( + description="transform platform client-level URL with the app's /api/v1 suffix", + sdk_endpoint_name="jobs.list_jobs", + client_url="https://platform-api.transform.unstructured.io/api/v1", + endpoint_url=None, + expected_url="https://platform-api.transform.unstructured.io", ), URLTestCase( description="partition client level with path", sdk_endpoint_name="general.partition", client_url="https://api.unstructuredapp.io/general/v0/general", endpoint_url=None, - expected_url="https://api.unstructuredapp.io" + expected_url="https://api.unstructuredapp.io", ), URLTestCase( description="partition endpoint level with path", sdk_endpoint_name="general.partition", client_url=None, endpoint_url="https://api.unstructuredapp.io/general/v0/general", - expected_url="https://api.unstructuredapp.io" + expected_url="https://api.unstructuredapp.io", ), URLTestCase( description="partition default url", sdk_endpoint_name="general.partition", client_url=None, endpoint_url=None, - expected_url="https://api.unstructuredapp.io" + expected_url="https://api.unstructuredapp.io", ), URLTestCase( description="default URL fallback", sdk_endpoint_name="destinations.create_destination", client_url=None, endpoint_url=None, - expected_url="https://platform.unstructuredapp.io" + expected_url="https://platform.unstructuredapp.io", ), URLTestCase( description="default URL fallback", sdk_endpoint_name="sources.create_source", client_url=None, endpoint_url=None, - expected_url="https://platform.unstructuredapp.io" + expected_url="https://platform.unstructuredapp.io", ), URLTestCase( description="default URL fallback", sdk_endpoint_name="jobs.get_job", client_url=None, endpoint_url=None, - expected_url="https://platform.unstructuredapp.io" + expected_url="https://platform.unstructuredapp.io", ), URLTestCase( description="default URL fallback", sdk_endpoint_name="workflows.create_workflow", client_url=None, endpoint_url=None, - expected_url="https://platform.unstructuredapp.io" + expected_url="https://platform.unstructuredapp.io", ), - ] + ], ) def test_endpoint_uses_correct_url(monkeypatch, case: URLTestCase): if case.client_url: @@ -218,10 +222,7 @@ def test_endpoint_uses_correct_url(monkeypatch, case: URLTestCase): s = UnstructuredClient() client_method = get_client_method_with_mock( - case.sdk_endpoint_name, - s, - case.expected_url, - monkeypatch + case.sdk_endpoint_name, s, case.expected_url, monkeypatch ) try: @@ -232,6 +233,94 @@ def test_endpoint_uses_correct_url(monkeypatch, case: URLTestCase): except BaseUrlCorrect: pass except BaseUrlIncorrect as e: - pytest.fail( - f"{case.description}: Expected {case.expected_url}, got {e}" - ) \ No newline at end of file + pytest.fail(f"{case.description}: Expected {case.expected_url}, got {e}") + + +@pytest.mark.parametrize( + "client_url,endpoint_url", + [ + # -- the value the Transform Platform's API Keys page hands you, passed at the + # -- client level and at the operation level -- + ("https://platform-api.transform.unstructured.io/api/v1", None), + (None, "https://platform-api.transform.unstructured.io/api/v1"), + # -- and the bare host, which must not regress -- + ("https://platform-api.transform.unstructured.io", None), + (None, "https://platform-api.transform.unstructured.io"), + ], +) +def test_platform_request_url_has_a_single_api_prefix(client_url, endpoint_url): + """The operation path already carries /api/v1, so the base URL must not repeat it. + + A doubled /api/v1/api/v1/jobs/ matches no route on the Platform API and 404s. + """ + import httpx + + sent = [] + + def capture(request: httpx.Request) -> httpx.Response: + sent.append(str(request.url)) + return httpx.Response(200, json=[]) + + client = UnstructuredClient( + api_key_auth="fake-key", + server_url=client_url, + client=httpx.Client(transport=httpx.MockTransport(capture)), + ) + + if endpoint_url: + client.jobs.list_jobs(request={}, server_url=endpoint_url) + else: + client.jobs.list_jobs(request={}) + + assert sent == ["https://platform-api.transform.unstructured.io/api/v1/jobs/"] + + +@pytest.mark.parametrize( + "client_url,endpoint_url,expected_url", + [ + # -- a self-hosted deployment at the root -- + ( + "http://localhost:8000", + None, + "http://localhost:8000/api/v1/jobs/", + ), + ( + None, + "http://localhost:8000", + "http://localhost:8000/api/v1/jobs/", + ), + # -- and one behind a subpath, whose path must survive -- + ( + "http://localhost:8000/my/endpoint", + None, + "http://localhost:8000/my/endpoint/api/v1/jobs/", + ), + ( + None, + "http://localhost:8000/my/endpoint", + "http://localhost:8000/my/endpoint/api/v1/jobs/", + ), + ], +) +def test_non_unstructured_host_keeps_its_path(client_url, endpoint_url, expected_url): + """A host that is not ours may legitimately serve the API beneath a subpath.""" + import httpx + + sent = [] + + def capture(request: httpx.Request) -> httpx.Response: + sent.append(str(request.url)) + return httpx.Response(200, json=[]) + + client = UnstructuredClient( + api_key_auth="fake-key", + server_url=client_url, + client=httpx.Client(transport=httpx.MockTransport(capture)), + ) + + if endpoint_url: + client.jobs.list_jobs(request={}, server_url=endpoint_url) + else: + client.jobs.list_jobs(request={}) + + assert sent == [expected_url] diff --git a/src/unstructured_client/_hooks/custom/__init__.py b/src/unstructured_client/_hooks/custom/__init__.py index 8917d508..d14ceeb7 100644 --- a/src/unstructured_client/_hooks/custom/__init__.py +++ b/src/unstructured_client/_hooks/custom/__init__.py @@ -1,4 +1,3 @@ -from .clean_server_url_hook import CleanServerUrlSDKInitHook from .logger_hook import LoggerHook from .split_pdf_hook import SplitPdfHook import logging diff --git a/src/unstructured_client/_hooks/custom/clean_server_url_hook.py b/src/unstructured_client/_hooks/custom/clean_server_url_hook.py index e582df14..dc76f8f1 100644 --- a/src/unstructured_client/_hooks/custom/clean_server_url_hook.py +++ b/src/unstructured_client/_hooks/custom/clean_server_url_hook.py @@ -1,10 +1,30 @@ from __future__ import annotations -from typing import Tuple from urllib.parse import ParseResult, urlparse, urlunparse -from unstructured_client._hooks.types import SDKInitHook -from unstructured_client.httpclient import HttpClient +# Domains Unstructured serves its APIs from. Every operation in this SDK already carries +# its own path prefix (`/api/v1/...`, `/general/v0/general`), so a base URL under one of +# these hosts must not carry a path of its own -- the app and the docs hand users a full +# API URL, and appending an operation path to that produces a doubled prefix that 404s. +UNSTRUCTURED_DOMAINS = ("unstructuredapp.io", "unstructured.io") + + +def is_unstructured_domain(hostname: str | None) -> bool: + """True if the hostname is one of Unstructured's own API domains, or a subdomain of one. + + Matched on domain boundaries, so a host that merely contains one of our domains + (`unstructuredapp.io.example.com`) is somebody else's and is left alone. A fully + qualified name carrying the terminal root dot (`api.unstructuredapp.io.`) is still + ours. + """ + if not hostname: + return False + + hostname = hostname.lower().rstrip(".") + return any( + hostname == domain or hostname.endswith(f".{domain}") + for domain in UNSTRUCTURED_DOMAINS + ) def clean_server_url(base_url: str | None) -> str: @@ -18,28 +38,17 @@ def clean_server_url(base_url: str | None) -> str: base_url = "http://" + base_url parsed_url: ParseResult = urlparse(base_url) - - if "unstructuredapp.io" in parsed_url.netloc: + + if is_unstructured_domain(parsed_url.hostname): if parsed_url.scheme != "https": parsed_url = parsed_url._replace(scheme="https") # We only want the base url for Unstructured domains - clean_url = urlunparse(parsed_url._replace(path="", params="", query="", fragment="")) - + clean_url = urlunparse( + parsed_url._replace(path="", params="", query="", fragment="") + ) + else: # For other domains, we want to keep the path clean_url = urlunparse(parsed_url._replace(params="", query="", fragment="")) return clean_url.rstrip("/") - - - -class CleanServerUrlSDKInitHook(SDKInitHook): - """Hook fixing common mistakes by users in defining `server_url` in the unstructured-client""" - - def sdk_init( - self, base_url: str, client: HttpClient - ) -> Tuple[str, HttpClient]: - """Concrete implementation for SDKInitHook.""" - cleaned_url = clean_server_url(base_url) - - return cleaned_url, client diff --git a/src/unstructured_client/_hooks/registration.py b/src/unstructured_client/_hooks/registration.py index 22cd276d..3bcf37bd 100644 --- a/src/unstructured_client/_hooks/registration.py +++ b/src/unstructured_client/_hooks/registration.py @@ -1,7 +1,6 @@ """Registration of custom, human-written hooks.""" from .custom import ( - CleanServerUrlSDKInitHook, LoggerHook, SplitPdfHook, ) @@ -21,7 +20,6 @@ def init_hooks(hooks: Hooks): """ # Initialize custom hooks - clean_server_url_hook = CleanServerUrlSDKInitHook() logger_hook = LoggerHook() split_pdf_hook = SplitPdfHook() @@ -29,7 +27,6 @@ def init_hooks(hooks: Hooks): # request and whether it will be retried which can be changed by e.g. split_pdf_hook # Register SDK Init hooks - hooks.register_sdk_init_hook(clean_server_url_hook) hooks.register_sdk_init_hook(logger_hook) hooks.register_sdk_init_hook(split_pdf_hook) @@ -42,4 +39,4 @@ def init_hooks(hooks: Hooks): # Register After Error hooks hooks.register_after_error_hook(split_pdf_hook) - hooks.register_after_error_hook(logger_hook) + hooks.register_after_error_hook(logger_hook) diff --git a/src/unstructured_client/_version.py b/src/unstructured_client/_version.py index 02e1257b..85d5419c 100644 --- a/src/unstructured_client/_version.py +++ b/src/unstructured_client/_version.py @@ -3,10 +3,10 @@ import importlib.metadata __title__: str = "unstructured-client" -__version__: str = "0.46.1" +__version__: str = "0.46.2" __openapi_doc_version__: str = "1.2.31" __gen_version__: str = "2.680.0" -__user_agent__: str = "speakeasy-sdk/python 0.46.1 2.680.0 1.2.31 unstructured-client" +__user_agent__: str = "speakeasy-sdk/python 0.46.2 2.680.0 1.2.31 unstructured-client" try: if __package__ is not None: diff --git a/src/unstructured_client/basesdk.py b/src/unstructured_client/basesdk.py index 4324324f..1a77c61d 100644 --- a/src/unstructured_client/basesdk.py +++ b/src/unstructured_client/basesdk.py @@ -1,23 +1,28 @@ """Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.""" import asyncio +from collections.abc import Callable, Mapping +from urllib.parse import parse_qs, urlparse -from .sdkconfiguration import SDKConfiguration import httpx -from typing import Callable, List, Mapping, Optional, Tuple + from unstructured_client import utils from unstructured_client._hooks import ( AfterErrorContext, AfterSuccessContext, BeforeRequestContext, ) +from unstructured_client._hooks.custom.clean_server_url_hook import ( + clean_server_url, +) from unstructured_client.models import errors from unstructured_client.utils import ( RetryConfig, SerializedRequestBody, get_body_content, ) -from urllib.parse import parse_qs, urlparse + +from .sdkconfiguration import SDKConfiguration class _RequestBoundCancelledError(asyncio.CancelledError): @@ -44,7 +49,13 @@ def _get_url(self, base_url, url_variables): if url_variables is None: url_variables = sdk_variables - return utils.template_url(base_url, url_variables) + # Note(pk): _get_url is the one point every operation's base URL passes through, + # so cleaning here normalizes both the client-level server_url and an + # operation-level `server_url=` override. There is no SDK-init hook doing this + # anymore, so `sdk_configuration.server_url` is stored raw -- only the request URL + # built here is cleaned. A caller reading that attribute directly sees the value + # exactly as it was passed. + return clean_server_url(utils.template_url(base_url, url_variables)) def _build_request_async( self, @@ -60,12 +71,10 @@ def _build_request_async( accept_header_value, _globals=None, security=None, - timeout_ms: Optional[int] = None, - get_serialized_body: Optional[ - Callable[[], Optional[SerializedRequestBody]] - ] = None, - url_override: Optional[str] = None, - http_headers: Optional[Mapping[str, str]] = None, + timeout_ms: int | None = None, + get_serialized_body: Callable[[], SerializedRequestBody | None] | None = None, + url_override: str | None = None, + http_headers: Mapping[str, str] | None = None, ) -> httpx.Request: client = self.sdk_configuration.async_client return self._build_request_with_client( @@ -102,12 +111,10 @@ def _build_request( accept_header_value, _globals=None, security=None, - timeout_ms: Optional[int] = None, - get_serialized_body: Optional[ - Callable[[], Optional[SerializedRequestBody]] - ] = None, - url_override: Optional[str] = None, - http_headers: Optional[Mapping[str, str]] = None, + timeout_ms: int | None = None, + get_serialized_body: Callable[[], SerializedRequestBody | None] | None = None, + url_override: str | None = None, + http_headers: Mapping[str, str] | None = None, ) -> httpx.Request: client = self.sdk_configuration.client return self._build_request_with_client( @@ -145,12 +152,10 @@ def _build_request_with_client( accept_header_value, _globals=None, security=None, - timeout_ms: Optional[int] = None, - get_serialized_body: Optional[ - Callable[[], Optional[SerializedRequestBody]] - ] = None, - url_override: Optional[str] = None, - http_headers: Optional[Mapping[str, str]] = None, + timeout_ms: int | None = None, + get_serialized_body: Callable[[], SerializedRequestBody | None] | None = None, + url_override: str | None = None, + http_headers: Mapping[str, str] | None = None, ) -> httpx.Request: query_params = {} @@ -178,7 +183,7 @@ def _build_request_with_client( headers["Accept"] = accept_header_value headers[user_agent_header] = self.sdk_configuration.user_agent - if security is not None: + if security is not None: # noqa: SIM102 if callable(security): security = security() @@ -229,7 +234,7 @@ def do_request( request, error_status_codes, stream=False, - retry_config: Optional[Tuple[RetryConfig, List[str]]] = None, + retry_config: tuple[RetryConfig, list[str]] | None = None, ) -> httpx.Response: client = self.sdk_configuration.client logger = self.sdk_configuration.debug_logger @@ -256,7 +261,7 @@ def do(): _, e = hooks.after_error(AfterErrorContext(hook_ctx), None, e) if e is not None: logger.debug("Request Exception", exc_info=True) - raise e + raise e # noqa: TRY201 if http_res is None: logger.debug("Raising no response SDK error") @@ -275,7 +280,7 @@ def do(): AfterErrorContext(hook_ctx), http_res, None ) if err is not None: - logger.debug("Request Exception", exc_info=True) + logger.debug("Request Exception", exc_info=True) # noqa: LOG014 raise err if result is not None: http_res = result @@ -301,7 +306,7 @@ async def do_request_async( request, error_status_codes, stream=False, - retry_config: Optional[Tuple[RetryConfig, List[str]]] = None, + retry_config: tuple[RetryConfig, list[str]] | None = None, ) -> httpx.Response: client = self.sdk_configuration.async_client logger = self.sdk_configuration.debug_logger @@ -309,8 +314,8 @@ async def do_request_async( hooks = self.sdk_configuration.__dict__["_hooks"] async def cleanup_cancelled_request( - req: Optional[httpx.Request], - response: Optional[httpx.Response], + req: httpx.Request | None, + response: httpx.Response | None, cancellation: asyncio.CancelledError, ) -> None: if req is None and response is None: @@ -334,7 +339,9 @@ async def cleanup_cancelled_request( if cleanup_task.done(): logger.debug("Cancellation cleanup cancelled", exc_info=True) return - logger.debug("Cancellation cleanup still running after cancellation") + logger.debug( + "Cancellation cleanup still running after cancellation" + ) except BaseException: logger.debug("Cancellation cleanup failed", exc_info=True) return @@ -343,7 +350,9 @@ async def do(): http_res = None req = None try: - req = await hooks.before_request_async(BeforeRequestContext(hook_ctx), request) + req = await hooks.before_request_async( + BeforeRequestContext(hook_ctx), request + ) logger.debug( "Request:\nMethod: %s\nURL: %s\nHeaders: %s\nBody: %s", req.method, @@ -360,10 +369,12 @@ async def do(): await cleanup_cancelled_request(req, None, cancellation) raise except Exception as e: - _, e = await hooks.after_error_async(AfterErrorContext(hook_ctx), None, e) + _, e = await hooks.after_error_async( + AfterErrorContext(hook_ctx), None, e + ) if e is not None: logger.debug("Request Exception", exc_info=True) - raise e + raise e # noqa: TRY201 if http_res is None: logger.debug("Raising no response SDK error") @@ -386,7 +397,7 @@ async def do(): await cleanup_cancelled_request(None, http_res, cancellation) raise if err is not None: - logger.debug("Request Exception", exc_info=True) + logger.debug("Request Exception", exc_info=True) # noqa: LOG014 raise err if result is not None: http_res = result @@ -405,7 +416,9 @@ async def do(): if not utils.match_status_codes(error_status_codes, http_res.status_code): try: - http_res = await hooks.after_success_async(AfterSuccessContext(hook_ctx), http_res) + http_res = await hooks.after_success_async( + AfterSuccessContext(hook_ctx), http_res + ) except asyncio.CancelledError as cancellation: await cleanup_cancelled_request(None, http_res, cancellation) raise diff --git a/src/unstructured_client/general.py b/src/unstructured_client/general.py index c2372c4c..d1f2b47c 100644 --- a/src/unstructured_client/general.py +++ b/src/unstructured_client/general.py @@ -29,8 +29,7 @@ class PartitionAcceptEnum(str, Enum): a large document never has to be held in memory.""" -# NDJSON elements-file support. `partition.py` and this module are both .genignore'd so -# these edits survive regeneration; see the notes in .genignore. +# NDJSON elements-file support. This is a client-side custom edit to `general.py`. def _new_elements_file(): return tempfile.NamedTemporaryFile( # pylint: disable=consider-using-with mode="wb", prefix="unst_elements_", suffix=".ndjson", delete=False