diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index bdb5129..7c17005 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -29,3 +29,9 @@ jobs: run: | pip install build python -m build + + - name: Import typed params from the wheel + run: | + python -m venv /tmp/wheel-env + /tmp/wheel-env/bin/pip install dist/*.whl + cd /tmp && /tmp/wheel-env/bin/python -c "import decodo; assert decodo._codegen_available; from decodo import ScrapeRequest" diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml index 4243175..34718ea 100644 --- a/.github/workflows/workflow.yml +++ b/.github/workflows/workflow.yml @@ -64,6 +64,13 @@ jobs: pip install build python -m build + - name: Import typed params from the wheel + if: steps.release.outputs.released == 'true' + run: | + python -m venv /tmp/wheel-env + /tmp/wheel-env/bin/pip install dist/*.whl + cd /tmp && /tmp/wheel-env/bin/python -c "import decodo; assert decodo._codegen_available; from decodo import ScrapeRequest" + - name: Publish to PyPI if: steps.release.outputs.released == 'true' uses: pypa/gh-action-pypi-publish@release/v1 diff --git a/README.md b/README.md index aee6152..8616206 100644 --- a/README.md +++ b/README.md @@ -545,13 +545,8 @@ Each target accepts one primary input parameter (`url`, `query`, `product_id`, o | Target | Parameter class | Description | Example input | | ------------------------------ | ----------------------------- | --------------------------------- | ---------------------------------------- | | `Target.GoogleSearch` | `GoogleSearchParams` | Google Search results for a query | `query="coffee shops"` | -| `Target.GoogleMaps` | `GoogleMapsParams` | Google Maps search results | `query="coffee shops brooklyn"` | -| `Target.GoogleShoppingSearch` | `GoogleShoppingSearchParams` | Google Shopping search results | `query="laptop"` | -| `Target.GoogleShoppingProduct` | `GoogleShoppingProductParams` | Google Shopping product page | `query="laptop"` | -| `Target.GoogleSuggest` | `GoogleSuggestParams` | Google Autocomplete suggestions | `query="coffee"` | | `Target.GoogleLens` | `GoogleLensParams` | Google Lens reverse image search | `query="https://example.com/cat.jpg"` | | `Target.GoogleTravelHotels` | `GoogleTravelHotelsParams` | Google Travel hotel listings | `query="hotels in paris"` | -| `Target.GoogleTrendsExplore` | `GoogleTrendsExploreParams` | Google Trends explore data | `query="coffee"` | | `Target.GoogleAds` | `GoogleAdsParams` | Google Ads results for a query | `query="laptop"` | | `Target.BingSearch` | `BingSearchParams` | Bing Search results | `query="electric vehicles"` | | `Target.Bing` | `BingParams` | Raw Bing URL scraping | `url="https://bing.com/search?q=laptop"` | @@ -571,8 +566,6 @@ Each target accepts one primary input parameter (`url`, `query`, `product_id`, o | `Target.TargetProduct` | `TargetProductParams` | Target.com product page by product ID | `product_id="92186007"` | | `Target.TargetSearch` | `TargetSearchParams` | Target.com search results | `query="laptop"` | | `Target.Target` | `TargetStoreParams` | Raw Target.com URL scraping | `url="https://target.com/p/-/A-92186007"` | -| `Target.LowesSearch` | `LowesSearchParams` | Lowe's search results | `query="drill"` | -| `Target.Ecommerce` | `EcommerceParams` | Generic eCommerce page with parser | `url="https://example.com/product/123"` | ### Social media @@ -581,18 +574,14 @@ Each target accepts one primary input parameter (`url`, `query`, `product_id`, o | `Target.RedditPost` | `RedditPostParams` | Reddit post by URL | `url="https://reddit.com/r/nba/comments/..."` | | `Target.RedditSubreddit` | `RedditSubredditParams` | Reddit subreddit by URL | `url="https://reddit.com/r/nba/"` | | `Target.RedditUser` | `RedditUserParams` | Reddit user profile by URL | `url="https://reddit.com/user/example/"` | -| `Target.YoutubeVideo` | `YoutubeVideoParams` | YouTube video by ID | `query="dFu9aKJoqGg"` | | `Target.YoutubeSearch` | `YoutubeSearchParams` | YouTube search results | `query="ambient music"` | -| `Target.YoutubeSearchMax` | `YoutubeSearchMaxParams` | YouTube search results (extended) | `query="ambient music"` | | `Target.YoutubeMetadata` | `YoutubeMetadataParams` | YouTube video metadata by ID | `query="dFu9aKJoqGg"` | -| `Target.YoutubeTranscript` | `YoutubeTranscriptParams` | YouTube video transcript by ID | `query="dFu9aKJoqGg"` | | `Target.YoutubeSubtitles` | `YoutubeSubtitlesParams` | YouTube video subtitles by ID | `query="dFu9aKJoqGg"` | | `Target.YoutubeChannel` | `YoutubeChannelParams` | YouTube channel by handle or ID | `query="@decodo_official"` | | `Target.TiktokPost` | `TiktokPostParams` | TikTok post by URL | `url="https://www.tiktok.com/@nba/video/..."` | | `Target.TiktokShopSearch` | `TiktokShopSearchParams` | TikTok Shop search results | `query="wireless earbuds"` | | `Target.TiktokShopProduct` | `TiktokShopProductParams` | TikTok Shop product page by product ID | `product_id="7100000000000000000"` | | `Target.Tiktok` | `TiktokParams` | Raw TikTok URL scraping | `url="https://www.tiktok.com/@nba"` | -| `Target.InstagramGraphqlProfile` | `InstagramGraphqlProfileParams` | Instagram profile via GraphQL | `query="nba"` | ### AI tools @@ -621,8 +610,6 @@ Each target accepts one primary input parameter (`url`, `query`, `product_id`, o | `Target.Google` | `GoogleParams` | Raw Google URL scraping | `url="https://google.com/search?q=laptop"` | | `Target.Amazon` | `AmazonParams` | Raw Amazon URL scraping | `url="https://amazon.com/dp/B09H74FXNW"` | -> `Target.UniversalEcommerce` isn't listed above because it doesn't accept a primary input parameter like `url`, `query`, `product_id`, or `prompt`. Its parameter class `UniversalEcommerceParams` only accepts optional configuration fields such as `callback_url`. - For the full target list and parameter details, see the API documentation: - [Target list](https://help.decodo.com/docs/web-scraping-api-targets) diff --git a/examples/web_scraping_api/google_maps.py b/examples/web_scraping_api/google_maps.py deleted file mode 100644 index 2859c5a..0000000 --- a/examples/web_scraping_api/google_maps.py +++ /dev/null @@ -1,29 +0,0 @@ -import json -import os - -from decodo import ( - DecodoClient, - DecodoConfig, - GoogleMapsParams, - Target, - WebScrapingApiConfig, -) - -api_key = os.environ["DECODO_API_KEY"] - -client = DecodoClient( - DecodoConfig( - web_scraping_api=WebScrapingApiConfig(api_key=api_key), - ) -) - -result = client.web_scraping_api.scrape( - GoogleMapsParams( - target=Target.GoogleMaps, - query='coffee shops brooklyn', - geo='United States', - locale='en', - ) -) - -print(json.dumps(result, indent=2)) diff --git a/examples/web_scraping_api/google_shopping_search.py b/examples/web_scraping_api/google_shopping_search.py deleted file mode 100644 index 4d77297..0000000 --- a/examples/web_scraping_api/google_shopping_search.py +++ /dev/null @@ -1,29 +0,0 @@ -import json -import os - -from decodo import ( - DecodoClient, - DecodoConfig, - GoogleShoppingSearchParams, - Target, - WebScrapingApiConfig, -) - -api_key = os.environ["DECODO_API_KEY"] - -client = DecodoClient( - DecodoConfig( - web_scraping_api=WebScrapingApiConfig(api_key=api_key), - ) -) - -result = client.web_scraping_api.scrape( - GoogleShoppingSearchParams( - target=Target.GoogleShoppingSearch, - query='laptop', - headless='html', - parse=True, - ) -) - -print(json.dumps(result, indent=2)) diff --git a/examples/web_scraping_api/google_trends_explore.py b/examples/web_scraping_api/google_trends_explore.py deleted file mode 100644 index 380affa..0000000 --- a/examples/web_scraping_api/google_trends_explore.py +++ /dev/null @@ -1,27 +0,0 @@ -import json -import os - -from decodo import ( - DecodoClient, - DecodoConfig, - GoogleTrendsExploreParams, - Target, - WebScrapingApiConfig, -) - -api_key = os.environ["DECODO_API_KEY"] - -client = DecodoClient( - DecodoConfig( - web_scraping_api=WebScrapingApiConfig(api_key=api_key), - ) -) - -result = client.web_scraping_api.scrape( - GoogleTrendsExploreParams( - target=Target.GoogleTrendsExplore, - query='seo optimization', - ) -) - -print(json.dumps(result, indent=2)) diff --git a/examples/web_scraping_api/instagram_profile.py b/examples/web_scraping_api/instagram_profile.py deleted file mode 100644 index 722720c..0000000 --- a/examples/web_scraping_api/instagram_profile.py +++ /dev/null @@ -1,27 +0,0 @@ -import json -import os - -from decodo import ( - DecodoClient, - DecodoConfig, - InstagramGraphqlProfileParams, - Target, - WebScrapingApiConfig, -) - -api_key = os.environ["DECODO_API_KEY"] - -client = DecodoClient( - DecodoConfig( - web_scraping_api=WebScrapingApiConfig(api_key=api_key), - ) -) - -result = client.web_scraping_api.scrape( - InstagramGraphqlProfileParams( - target=Target.InstagramGraphqlProfile, - query='nba', - ) -) - -print(json.dumps(result, indent=2)) diff --git a/examples/web_scraping_api/youtube_search_max.py b/examples/web_scraping_api/youtube_search_max.py deleted file mode 100644 index 5f4270a..0000000 --- a/examples/web_scraping_api/youtube_search_max.py +++ /dev/null @@ -1,28 +0,0 @@ -import json -import os - -from decodo import ( - DecodoClient, - DecodoConfig, - Target, - WebScrapingApiConfig, - YoutubeSearchMaxParams, -) - -api_key = os.environ["DECODO_API_KEY"] - -client = DecodoClient( - DecodoConfig( - web_scraping_api=WebScrapingApiConfig(api_key=api_key), - ) -) - -result = client.web_scraping_api.scrape( - YoutubeSearchMaxParams( - target=Target.YoutubeSearchMax, - query='How to care for chinchillas', - video_sort_by='relevance', - ) -) - -print(json.dumps(result, indent=2)) diff --git a/src/decodo/__init__.py b/src/decodo/__init__.py index c6740c4..0551cf8 100644 --- a/src/decodo/__init__.py +++ b/src/decodo/__init__.py @@ -36,239 +36,35 @@ TaskStatus, ) +_GENERATED_PARAMETER_NAMES = ("ParameterMeta", "parameter_meta") + _codegen_available = False +_CODEGEN_NAMES: frozenset[str] = frozenset() try: + from .generated import targets as _generated_targets from .generated.parameters import ParameterMeta, parameter_meta # noqa: F401 - from .generated.targets import ( # noqa: F401 - AirbnbBatchParams, - AirbnbParams, - AmazonBatchParams, - AmazonBestsellersBatchParams, - AmazonBestsellersParams, - AmazonParams, - AmazonPricingBatchParams, - AmazonPricingParams, - AmazonProductBatchParams, - AmazonProductParams, - AmazonSearchBatchParams, - AmazonSearchParams, - AmazonSellersBatchParams, - AmazonSellersParams, - AppleAppStoreBatchParams, - AppleAppStoreParams, - AutotraderBatchParams, - AutotraderParams, - BatchRequest, - BbbBatchParams, - BbbParams, - BingBatchParams, - BingParams, - BingSearchBatchParams, - BingSearchParams, - ChatgptBatchParams, - ChatgptParams, - EcommerceBatchParams, - EcommerceParams, - GeminiBatchParams, - GeminiParams, - GoogleAdsBatchParams, - GoogleAdsParams, - GoogleAiModeBatchParams, - GoogleAiModeParams, - GoogleBatchParams, - GoogleLensBatchParams, - GoogleLensParams, - GoogleMapsBatchParams, - GoogleMapsParams, - GoogleParams, - GoogleSearchBatchParams, - GoogleSearchParams, - GoogleShoppingProductBatchParams, - GoogleShoppingProductParams, - GoogleShoppingSearchBatchParams, - GoogleShoppingSearchParams, - GoogleSuggestBatchParams, - GoogleSuggestParams, - GoogleTravelHotelsBatchParams, - GoogleTravelHotelsParams, - GoogleTrendsExploreBatchParams, - GoogleTrendsExploreParams, - InstagramGraphqlProfileBatchParams, - InstagramGraphqlProfileParams, - LowesSearchBatchParams, - LowesSearchParams, - MobileBatchParams, - MobileParams, - PerplexityBatchParams, - PerplexityParams, - RedditPostBatchParams, - RedditPostParams, - RedditSubredditBatchParams, - RedditSubredditParams, - RedditUserBatchParams, - RedditUserParams, - ScrapeRequest, - TargetProductBatchParams, - TargetProductParams, - TargetSearchBatchParams, - TargetSearchParams, - TargetStoreBatchParams, - TargetStoreParams, - TiktokBatchParams, - TiktokParams, - TiktokPostBatchParams, - TiktokPostParams, - TiktokShopProductBatchParams, - TiktokShopProductParams, - TiktokShopSearchBatchParams, - TiktokShopSearchParams, - UniversalBatchParams, - UniversalEcommerceBatchParams, - UniversalEcommerceParams, - UniversalParams, - WalmartBatchParams, - WalmartParams, - WalmartProductBatchParams, - WalmartProductParams, - WalmartSearchBatchParams, - WalmartSearchParams, - YoutubeChannelBatchParams, - YoutubeChannelParams, - YoutubeMetadataBatchParams, - YoutubeMetadataParams, - YoutubeSearchBatchParams, - YoutubeSearchMaxBatchParams, - YoutubeSearchMaxParams, - YoutubeSearchParams, - YoutubeSubtitlesBatchParams, - YoutubeSubtitlesParams, - YoutubeTranscriptBatchParams, - YoutubeTranscriptParams, - YoutubeVideoBatchParams, - YoutubeVideoParams, - target_meta, - ) + + # Read before the star import: generated files from before __all__ was emitted + # fall through to the "run codegen" hint instead of re-exporting their imports. + _CODEGEN_NAMES = frozenset([*_GENERATED_PARAMETER_NAMES, *_generated_targets.__all__]) + from .generated.targets import * # noqa: F403 + _codegen_available = True -except ImportError: - pass +except (ImportError, AttributeError): + _CODEGEN_NAMES = frozenset() -_CODEGEN_NAMES: frozenset[str] = frozenset( - [ - "AirbnbBatchParams", - "AirbnbParams", - "AmazonBatchParams", - "AmazonBestsellersBatchParams", - "AmazonBestsellersParams", - "AmazonParams", - "AmazonPricingBatchParams", - "AmazonPricingParams", - "AmazonProductBatchParams", - "AmazonProductParams", - "AmazonSearchBatchParams", - "AmazonSearchParams", - "AmazonSellersBatchParams", - "AmazonSellersParams", - "AppleAppStoreBatchParams", - "AppleAppStoreParams", - "AutotraderBatchParams", - "AutotraderParams", - "BatchRequest", - "BbbBatchParams", - "BbbParams", - "BingBatchParams", - "BingParams", - "BingSearchBatchParams", - "BingSearchParams", - "ChatgptBatchParams", - "ChatgptParams", - "EcommerceBatchParams", - "EcommerceParams", - "GeminiBatchParams", - "GeminiParams", - "GoogleAdsBatchParams", - "GoogleAdsParams", - "GoogleAiModeBatchParams", - "GoogleAiModeParams", - "GoogleBatchParams", - "GoogleLensBatchParams", - "GoogleLensParams", - "GoogleMapsBatchParams", - "GoogleMapsParams", - "GoogleParams", - "GoogleSearchBatchParams", - "GoogleSearchParams", - "GoogleShoppingProductBatchParams", - "GoogleShoppingProductParams", - "GoogleShoppingSearchBatchParams", - "GoogleShoppingSearchParams", - "GoogleSuggestBatchParams", - "GoogleSuggestParams", - "GoogleTravelHotelsBatchParams", - "GoogleTravelHotelsParams", - "GoogleTrendsExploreBatchParams", - "GoogleTrendsExploreParams", - "InstagramGraphqlProfileBatchParams", - "InstagramGraphqlProfileParams", - "LowesSearchBatchParams", - "LowesSearchParams", - "MobileBatchParams", - "MobileParams", - "ParameterMeta", - "PerplexityBatchParams", - "PerplexityParams", - "RedditPostBatchParams", - "RedditPostParams", - "RedditSubredditBatchParams", - "RedditSubredditParams", - "RedditUserBatchParams", - "RedditUserParams", + +def _is_codegen_name(name: str) -> bool: + return name.endswith("Params") or name in { + *_GENERATED_PARAMETER_NAMES, "ScrapeRequest", - "TargetProductBatchParams", - "TargetProductParams", - "TargetSearchBatchParams", - "TargetSearchParams", - "TargetStoreBatchParams", - "TargetStoreParams", - "TiktokBatchParams", - "TiktokParams", - "TiktokPostBatchParams", - "TiktokPostParams", - "TiktokShopProductBatchParams", - "TiktokShopProductParams", - "TiktokShopSearchBatchParams", - "TiktokShopSearchParams", - "UniversalBatchParams", - "UniversalEcommerceBatchParams", - "UniversalEcommerceParams", - "UniversalParams", - "WalmartBatchParams", - "WalmartParams", - "WalmartProductBatchParams", - "WalmartProductParams", - "WalmartSearchBatchParams", - "WalmartSearchParams", - "YoutubeChannelBatchParams", - "YoutubeChannelParams", - "YoutubeMetadataBatchParams", - "YoutubeMetadataParams", - "YoutubeSearchBatchParams", - "YoutubeSearchMaxBatchParams", - "YoutubeSearchMaxParams", - "YoutubeSearchParams", - "YoutubeSubtitlesBatchParams", - "YoutubeSubtitlesParams", - "YoutubeTranscriptBatchParams", - "YoutubeTranscriptParams", - "YoutubeVideoBatchParams", - "YoutubeVideoParams", - "parameter_meta", + "BatchRequest", "target_meta", - ] -) + } def __getattr__(name: str) -> object: - if name in _CODEGEN_NAMES: + if not _codegen_available and _is_codegen_name(name): raise ImportError( f"'{name}' requires generated types. " "Editable install: run python -m decodo.codegen.codegen, then use 'from decodo import ...'. " @@ -317,5 +113,4 @@ def __getattr__(name: str) -> object: "CrawlTimeoutError", ] -if _codegen_available: - __all__ += list(_CODEGEN_NAMES) +__all__ += sorted(_CODEGEN_NAMES) diff --git a/src/decodo/codegen/web_scraping_api/generate_targets.py b/src/decodo/codegen/web_scraping_api/generate_targets.py index a64683f..78c19b1 100644 --- a/src/decodo/codegen/web_scraping_api/generate_targets.py +++ b/src/decodo/codegen/web_scraping_api/generate_targets.py @@ -191,6 +191,19 @@ def _batch_class_name(target_key: str) -> str: lines.append("]") lines.append("") + exported = [ + *(_class_name(k) for k in target_keys), + *(_batch_class_name(k) for k in target_keys), + "ScrapeRequest", + "BatchRequest", + "target_meta", + ] + lines.append("__all__ = [") + for name in exported: + lines.append(f" {json.dumps(name)},") + lines.append("]") + lines.append("") + return "\n".join(lines) diff --git a/tests/test_generated_exports.py b/tests/test_generated_exports.py new file mode 100644 index 0000000..d5a7b23 --- /dev/null +++ b/tests/test_generated_exports.py @@ -0,0 +1,41 @@ +from __future__ import annotations + +from typing import Any + +import pytest + +import decodo +from decodo.codegen.web_scraping_api.generate_targets import _get_targets_file_contents + + +class TestGeneratedTargetsAll: + def test_all_lists_every_generated_class_and_union(self, minimal_ir: dict[str, Any]) -> None: + namespace: dict[str, Any] = {} + exec(_get_targets_file_contents(minimal_ir["apis"]["webScrapingApi"]), namespace) + + assert namespace["__all__"] == [ + "GoogleSearchParams", + "GoogleSearchBatchParams", + "ScrapeRequest", + "BatchRequest", + "target_meta", + ] + for name in namespace["__all__"]: + assert name in namespace + + +@pytest.mark.skipif(not decodo._codegen_available, reason="generated types not present") +class TestPackageExports: + def test_every_generated_name_is_importable_from_decodo(self) -> None: + from decodo.generated import parameters, targets + + for name in targets.__all__: + assert getattr(decodo, name) is getattr(targets, name) + assert name in decodo.__all__ + for name in ["ParameterMeta", "parameter_meta"]: + assert getattr(decodo, name) is getattr(parameters, name) + assert name in decodo.__all__ + + def test_target_absent_from_ir_raises_attribute_error(self) -> None: + with pytest.raises(AttributeError): + decodo.NoSuchTargetParams # noqa: B018