Repository navigation
Daily check: is anything broken? #64
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # The only workflow that calls the real API. Needs the BRIGHTDATA_API_TOKEN | |
| # secret and spends about one credit per post. | |
| # | |
| # Four ways it runs: | |
| # on every push to main: one post from nasa and the README's links, one | |
| # credit and about three minutes, so a broken commit shows up at once. | |
| # A newer push cancels a run still going for an older commit. | |
| # by hand, from the Actions tab, to scrape whatever accounts you choose | |
| # every day at 06:00 UTC except Monday: one post from nasa, the async | |
| # path, and the field table, about two credits | |
| # every Monday at 06:00 UTC: the same, plus every code block in the README, | |
| # about 35 credits and 30 minutes | |
| # | |
| # A slow day at the API shows up as a red run. Rerun it before reading more | |
| # into it. | |
| # | |
| # The daily and Monday runs end by rewriting the "last verified" badge line at | |
| # the top of README.md with today's date, and committing it if it changed. Push | |
| # runs never do this, so your own push is never followed by a bot commit. Only | |
| # the main repository commits; a copy of the repository just runs the checks. | |
| name: Live check | |
| run-name: "${{ github.event.schedule == '0 6 * * 1' && 'Weekly check: does every README block still work?' || github.event_name == 'schedule' && 'Daily check: is anything broken?' || github.event_name == 'push' && format('Push check: {0}', github.event.head_commit.message) || format('Scrape {0}, {1} posts each', inputs.handles, inputs.limit) }}" | |
| on: | |
| push: | |
| branches: [main] | |
| schedule: | |
| - cron: "0 6 * * 0,2-6" # daily smoke, every day except Monday | |
| - cron: "0 6 * * 1" # Monday: the whole README | |
| workflow_dispatch: | |
| inputs: | |
| handles: | |
| description: Instagram handles, separated by spaces. No @ needed. | |
| default: nasa natgeo | |
| limit: | |
| description: How many recent posts to fetch for each handle | |
| default: "2" | |
| validate: | |
| description: Also run every README block, as the Monday run does | |
| type: boolean | |
| default: false | |
| only: | |
| description: Block numbers to run, comma separated, for a cheap check. Empty means all. | |
| default: "" | |
| permissions: | |
| contents: write # the daily check commits the regenerated field table | |
| concurrency: | |
| group: live-${{ github.event_name }}-${{ github.ref }} | |
| cancel-in-progress: ${{ github.event_name == 'push' }} # only the newest commit matters | |
| jobs: | |
| scrape: | |
| name: Fetch the posts and attach them to this run | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 20 # fetch, async path, table; the README blocks have their own jobs | |
| env: | |
| BRIGHTDATA_API_TOKEN: ${{ secrets.BRIGHTDATA_API_TOKEN }} | |
| # On the daily schedule there are no inputs, so fall back to one cheap post. | |
| HANDLES: ${{ inputs.handles || 'nasa' }} | |
| LIMIT: ${{ inputs.limit || '1' }} | |
| # the async path and the field table run on every scheduled run and on request | |
| CHECK: ${{ github.event_name == 'schedule' || inputs.validate }} | |
| steps: | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 | |
| - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7 | |
| with: | |
| python-version: "3.12" | |
| - name: Install the package and the Bright Data CLI | |
| run: | | |
| pip install -e . | |
| npm i -g @brightdata/cli@0.3.4 # pinned: this job holds the API secret | |
| - name: Do the README's links still resolve? | |
| run: | | |
| public=$(curl -s -o /dev/null -w '%{http_code}' https://github.com/brightdata/instagram-scraper-python) | |
| failed=0 | |
| for url in $(grep -ohE 'https?://[^ )>"`]+' README.md AGENTS.md | sed 's/[.,]$//' | sort -u); do | |
| case "$url" in | |
| *instagram.com/*) continue ;; # rate-limits runners; the weekly README run fetches these posts anyway | |
| *zendesk.com/*) continue ;; # answers 403 to anything that is not a browser | |
| *mcp.brightdata.com/*) continue ;; # carries a placeholder token; nothing to check without a real one | |
| *github.com/brightdata/instagram-scraper-python*) [ "$public" = "200" ] || continue ;; # unreachable on a private fork | |
| esac | |
| code=$(curl -s -o /dev/null -L --retry 3 --max-time 30 -A "Mozilla/5.0" -w '%{http_code}' "$url" || echo 000) | |
| if [ "$code" = "200" ]; then echo "200 $url"; else echo "::error::$code $url"; failed=1; fi | |
| done | |
| [ "$failed" = 0 ] || exit 1 | |
| - name: Fetch the posts from Instagram | |
| run: | | |
| test -n "$BRIGHTDATA_API_TOKEN" || { | |
| echo "No API token. Add BRIGHTDATA_API_TOKEN under" >&2 | |
| echo "Settings, Secrets and variables, Actions, then start this again." >&2 | |
| exit 1 | |
| } | |
| # One retry, only on an API timeout. A slow day is not a broken build. | |
| set +e | |
| python -m instagram_scraper $HANDLES --limit "$LIMIT" 2>&1 | tee run.log | |
| code=${PIPESTATUS[0]} | |
| if [ "$code" != "0" ] && grep -q "timeout" run.log; then | |
| echo "::warning::The API timed out. Waiting 30s and trying once more." | |
| sleep 30 | |
| python -m instagram_scraper $HANDLES --limit "$LIMIT" 2>&1 | tee run.log | |
| code=${PIPESTATUS[0]} | |
| fi | |
| exit "$code" | |
| - name: Show what was found | |
| if: always() | |
| run: | | |
| set -o pipefail | |
| test -f instagram.json || exit 0 | |
| python - <<'PY' | tee -a "$GITHUB_STEP_SUMMARY" | |
| import json | |
| doc = json.load(open("instagram.json")) | |
| print(f"Run at {doc['generated_at']}\n") | |
| print("| handle | posts | fields | newest post |") | |
| print("| --- | --- | --- | --- |") | |
| for entry in doc["handles"]: | |
| posts = entry["posts"] | |
| newest = max(posts, key=lambda p: p.get("date_posted") or "", default=None) | |
| print( | |
| f"| {entry['handle']} | {len(posts)} | {len(posts[0]) if posts else 0} " | |
| f"| {newest.get('url') if newest else '-'} |" | |
| ) | |
| for entry in doc["handles"]: | |
| if not entry["posts"]: | |
| continue | |
| print(f"\n### {entry['handle']}\n") | |
| print("| date_posted | url | likes | comments |") | |
| print("| --- | --- | --- | --- |") | |
| for post in entry["posts"]: | |
| print( | |
| f"| {post.get('date_posted')} | {post.get('url')} " | |
| f"| {post.get('likes')} | {post.get('num_comments')} |" | |
| ) | |
| first = next((p for e in doc["handles"] for p in e["posts"]), None) | |
| if first: | |
| print("\n<details><summary>First record in full</summary>\n") | |
| print("```json") | |
| print(json.dumps(first, indent=2, ensure_ascii=False)) | |
| print("```\n") | |
| print("</details>") | |
| PY | |
| # The two daily checks. Discovery is already covered by the step above. | |
| - name: Does the async trigger, status, fetch path still work? | |
| if: env.CHECK == 'true' | |
| run: | | |
| python - <<'PY' | tee -a "$GITHUB_STEP_SUMMARY" | |
| import time | |
| from brightdata import SyncBrightDataClient | |
| url = "https://www.instagram.com/p/Db_SePSltfz/" | |
| with SyncBrightDataClient(auto_create_zones=False) as c: | |
| job = c.scrape.instagram.posts_trigger(url) | |
| sid = job.snapshot_id | |
| for _ in range(72): # up to 6 minutes | |
| status = c.scrape.instagram.posts_status(sid) | |
| if status in ("ready", "failed", "canceled"): | |
| break | |
| time.sleep(5) | |
| assert status == "ready", f"snapshot {sid} ended as {status}" | |
| rows = c.scrape.instagram.posts_fetch(sid) | |
| rows = rows if isinstance(rows, list) else [rows] | |
| assert rows and rows[0].get("url"), "fetch returned no record" | |
| print(f"\nAsync path ok: trigger, status, fetch returned {len(rows)} record(s).") | |
| PY | |
| - name: Regenerate the README field table from the live schema | |
| if: always() && env.CHECK == 'true' && github.repository == 'brightdata/instagram-scraper-python' | |
| run: | | |
| python - <<'PY' | tee -a "$GITHUB_STEP_SUMMARY" | |
| import json | |
| import pathlib | |
| import re | |
| from brightdata import SyncBrightDataClient | |
| readme = pathlib.Path("README.md") | |
| text = readme.read_text(encoding="utf-8") | |
| start, end = "<!-- fields:start -->", "<!-- fields:end -->" | |
| assert start in text and end in text, "README field-table markers are missing" | |
| with SyncBrightDataClient(auto_create_zones=False) as c: | |
| fields = c.datasets.instagram_posts.get_metadata().fields | |
| sample = json.load(open("examples/sample_output.json"))["handles"][0]["posts"][0] | |
| rows = ["| field | type | description |", "| --- | --- | --- |"] | |
| for name, spec in fields.items(): | |
| desc = (spec.description or "").replace(" — ", ": ").replace("—", ":") | |
| desc = " ".join(desc.split()).replace("`", "'").replace("|", "/") # no fences, no table breaks | |
| rows.append(f"| `{name}` | {spec.type} | {desc} |") | |
| block = "\n".join([ | |
| start, | |
| "<details>", | |
| f"<summary>All {len(fields)} fields, with type and description</summary>", | |
| "", | |
| "Regenerated every day from the dataset schema, via", | |
| "`client.datasets.instagram_posts.get_metadata()`, so it cannot go stale. A", | |
| f"post carries the fields that apply to it; the sample file has {len(sample)}", | |
| f"of these {len(fields)}.", | |
| "", | |
| *rows, | |
| "", | |
| "</details>", | |
| end, | |
| ]) | |
| new = re.sub(re.escape(start) + r".*?" + re.escape(end), lambda _: block, text, flags=re.S) | |
| readme.write_text(new, encoding="utf-8") | |
| state = "updated" if new != text else "already current" | |
| print(f"\nSchema: {len(fields)} fields. README table {state}.") | |
| PY | |
| if git diff --quiet README.md; then | |
| echo "Nothing to commit." | |
| exit 0 | |
| fi | |
| # Never commit a README the tests would reject. | |
| pip install -q -e ".[dev]" && pytest -q | |
| git config user.name "anil-bd" | |
| git config user.email "210667061+anil-bd@users.noreply.github.com" | |
| git add README.md | |
| git commit -m "Regenerate the field table from the live schema (automated daily check)" | |
| git pull --rebase origin main | |
| git push | |
| echo "Committed the regenerated table." | |
| - name: Attach instagram.json so you can download it | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 | |
| if: always() | |
| with: | |
| name: instagram-json | |
| path: instagram.json | |
| if-no-files-found: warn | |
| # Every code block in the README, each in its own job, all at once. Mondays, | |
| # and on request. Wall clock is the slowest block, about five minutes, | |
| # instead of the sum of all fourteen. | |
| plan: | |
| name: List the README blocks | |
| if: github.event.schedule == '0 6 * * 1' || inputs.validate == true | |
| runs-on: ubuntu-latest | |
| outputs: | |
| blocks: ${{ steps.list.outputs.blocks }} | |
| steps: | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 | |
| - id: list | |
| env: | |
| ONLY: ${{ inputs.only }} | |
| run: | | |
| python - <<'PY' >> "$GITHUB_OUTPUT" | |
| import os | |
| import json | |
| import pathlib | |
| import re | |
| readme = pathlib.Path("README.md").read_text(encoding="utf-8") | |
| # never execute anything inside the generated field table: its text comes from | |
| # the API, and a fence in a description must not become a command | |
| readme = re.sub(r"<!-- fields:start -->.*?<!-- fields:end -->", "", readme, flags=re.S) | |
| SKIP = ("pip install", "export ", "npm i", "npm install", | |
| "npx -p @brightdata/cli bdata login", "bdata login") | |
| blocks, heading, seen = [], "README", {} | |
| for m in re.finditer(r"^#{2,3} ([^\n]+)$|```(python|bash)\n(.*?)```", readme, re.M | re.S): | |
| if m.group(1): | |
| heading = m.group(1).strip() # a block is named after its section, not its first line | |
| continue | |
| lang, code = m.group(2), m.group(3) | |
| lines = [l for l in code.splitlines() if l.strip() and not l.strip().startswith("#")] | |
| if lang == "bash": | |
| lines = [l for l in lines if not l.startswith(SKIP)] | |
| if not lines: | |
| continue | |
| # the CLI cannot open a browser here; the secret goes in by flag, and | |
| # only the shell ever sees its value | |
| code = "\n".join(l + ' -k "$BRIGHTDATA_API_TOKEN"' if "bdata pipelines" in l else l for l in lines) | |
| seen[heading] = seen.get(heading, 0) + 1 | |
| title = heading if seen[heading] == 1 else f"{heading} ({seen[heading]})" | |
| blocks.append({"lang": lang, "title": title[:50], "code": code}) | |
| wanted = {int(n) for n in os.environ.get("ONLY", "").split(",") if n.strip()} | |
| out = [{"n": i + 1, "lang": b["lang"], "title": b["title"]} for i, b in enumerate(blocks) if not wanted or i + 1 in wanted] | |
| print("blocks=" + json.dumps(out)) | |
| PY | |
| block: | |
| name: "${{ matrix.n }}. ${{ matrix.title }}" | |
| needs: plan | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 10 # the SDK gives up at 3 minutes, reels at 7; past this is stuck | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: ${{ fromJson(needs.plan.outputs.blocks) }} | |
| env: | |
| BRIGHTDATA_API_TOKEN: ${{ secrets.BRIGHTDATA_API_TOKEN }} | |
| steps: | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 | |
| - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7 | |
| with: | |
| python-version: "3.12" | |
| - name: Install the package and, for CLI blocks, the Bright Data CLI | |
| run: | | |
| pip install -e . | |
| if [ "${{ matrix.lang }}" = "bash" ]; then npm i -g @brightdata/cli@0.3.4; fi | |
| - name: Run block ${{ matrix.n }} exactly as the README shows it | |
| env: | |
| N: ${{ matrix.n }} | |
| run: | | |
| set -o pipefail # without this, tee's exit code hides a failing block | |
| python - <<'PY' | tee -a "$GITHUB_STEP_SUMMARY" | |
| import os | |
| import subprocess | |
| import sys | |
| import time | |
| import json | |
| import pathlib | |
| import re | |
| readme = pathlib.Path("README.md").read_text(encoding="utf-8") | |
| # never execute anything inside the generated field table: its text comes from | |
| # the API, and a fence in a description must not become a command | |
| readme = re.sub(r"<!-- fields:start -->.*?<!-- fields:end -->", "", readme, flags=re.S) | |
| SKIP = ("pip install", "export ", "npm i", "npm install", | |
| "npx -p @brightdata/cli bdata login", "bdata login") | |
| blocks, heading, seen = [], "README", {} | |
| for m in re.finditer(r"^#{2,3} ([^\n]+)$|```(python|bash)\n(.*?)```", readme, re.M | re.S): | |
| if m.group(1): | |
| heading = m.group(1).strip() # a block is named after its section, not its first line | |
| continue | |
| lang, code = m.group(2), m.group(3) | |
| lines = [l for l in code.splitlines() if l.strip() and not l.strip().startswith("#")] | |
| if lang == "bash": | |
| lines = [l for l in lines if not l.startswith(SKIP)] | |
| if not lines: | |
| continue | |
| # the CLI cannot open a browser here; the secret goes in by flag, and | |
| # only the shell ever sees its value | |
| code = "\n".join(l + ' -k "$BRIGHTDATA_API_TOKEN"' if "bdata pipelines" in l else l for l in lines) | |
| seen[heading] = seen.get(heading, 0) + 1 | |
| title = heading if seen[heading] == 1 else f"{heading} ({seen[heading]})" | |
| blocks.append({"lang": lang, "title": title[:50], "code": code}) | |
| b = blocks[int(os.environ["N"]) - 1] | |
| argv = ["bash", "-e", "-c", b["code"]] if b["lang"] == "bash" else [sys.executable, "-c", b["code"]] | |
| secret = os.environ.get("BRIGHTDATA_API_TOKEN", "") | |
| for attempt in (1, 2): | |
| started = time.time() | |
| try: | |
| run = subprocess.run(argv, capture_output=True, text=True, timeout=480) | |
| except subprocess.TimeoutExpired: | |
| run = subprocess.CompletedProcess(argv, 1, "", "block timeout: no result within 8 minutes") | |
| secs = round(time.time() - started) | |
| out, err = run.stdout.strip(), run.stderr.strip() | |
| if secret: # summaries are not secret-masked the way logs are | |
| out, err = out.replace(secret, "***"), err.replace(secret, "***") | |
| ok = run.returncode == 0 and bool(out if b["lang"] == "python" else out or err) | |
| if ok or ((out or err) and "timeout" not in (out + err).lower()): | |
| break | |
| time.sleep(30) # the API timed out or answered with nothing at all; one retry | |
| shown = (out.splitlines() or [""])[0] if ok else (err.splitlines() or out.splitlines() or ["no output"])[-1] | |
| print(f"\n`{b['title']}`: {'ok' if ok else 'FAILED'} in {secs}s\n\n {shown[:120]}") | |
| sys.exit(0 if ok else 1) | |
| PY | |
| badge: | |
| name: Put today's date on the README's "last verified" badge | |
| needs: [scrape, plan, block] | |
| # scheduled and manual runs only, and only in the main repository: a push is | |
| # never followed by a bot commit, and a copy never diverges from the original | |
| if: always() && !cancelled() && github.event_name != 'push' && github.repository == 'brightdata/instagram-scraper-python' | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 5 | |
| steps: | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 | |
| - name: Rewrite one badge line, commit only if it changed | |
| env: | |
| SCRAPE: ${{ needs.scrape.result }} | |
| BLOCKS: ${{ needs.block.result }} # skipped on the daily run, and that is fine | |
| run: | | |
| if [ "$SCRAPE" = "success" ] && { [ "$BLOCKS" = "success" ] || [ "$BLOCKS" = "skipped" ]; }; then | |
| label="last verified"; color="brightgreen" | |
| else | |
| label="last check failed"; color="red" | |
| fi | |
| today=$(date -u +'%-d %b %Y') | |
| line="[](https://github.com/brightdata/instagram-scraper-python/actions/workflows/live.yml) <!-- verified: rewritten by the daily run -->" | |
| python3 - "$line" <<'PY' | |
| import pathlib, re, sys | |
| p = pathlib.Path("README.md"); t = p.read_text(encoding="utf-8") | |
| new, n = re.subn(r"^\[!\[last [^\]]*\]\(https://img\.shields\.io/badge/[^)]*\)\]\([^)]*\) <!-- verified:[^\n]*$", sys.argv[1], t, count=1, flags=re.M) | |
| if n != 1: | |
| sys.exit("the badge line is missing from README.md") | |
| p.write_text(new, encoding="utf-8") | |
| PY | |
| if git diff --quiet; then echo "badge already shows $today"; exit 0; fi | |
| git config user.name "anil-bd" | |
| git config user.email "210667061+anil-bd@users.noreply.github.com" | |
| git commit -qam "Last verified $today (automated daily check)" | |
| git pull -q --rebase origin main | |
| git push -q origin main | |
| echo "badge now shows: $label, $today" |