diff --git a/.github/actions/cache-nextjs/action.yml b/.github/actions/cache-nextjs/action.yml index 6449ba374ad9..6d3a2a3e8171 100644 --- a/.github/actions/cache-nextjs/action.yml +++ b/.github/actions/cache-nextjs/action.yml @@ -13,7 +13,8 @@ runs: with: path: ${{ github.workspace }}/.next/cache # Packages and source files both invalidate the cache. - key: ${{ runner.os }}-nextjs-${{ hashFiles('**/package-lock.json') }}-${{ hashFiles('**/*.ts', '**/*.tsx') }} + # Use narrow globs: `**` walks node_modules and translations and takes about 35s per hash. + key: ${{ runner.os }}-nextjs-${{ hashFiles('package-lock.json') }}-${{ hashFiles('src/**/*.ts', 'src/**/*.tsx', '*.ts') }} # With matching restore-key prefixes, source-only changes restore the same-package cache. restore-keys: | - ${{ runner.os }}-nextjs-${{ hashFiles('**/package-lock.json') }}- + ${{ runner.os }}-nextjs-${{ hashFiles('package-lock.json') }}- diff --git a/.github/actions/clone-translations/action.yml b/.github/actions/clone-translations/action.yml index c139d16474af..398435a5245a 100644 --- a/.github/actions/clone-translations/action.yml +++ b/.github/actions/clone-translations/action.yml @@ -10,58 +10,41 @@ inputs: runs: using: 'composite' steps: - - name: Clone Spanish - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - with: - repository: github/docs-internal.es-es - token: ${{ inputs.token }} - path: translations/es-es - - - name: Clone Japanese - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - with: - repository: github/docs-internal.ja-jp - token: ${{ inputs.token }} - path: translations/ja-jp - - - name: Clone Portuguese - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - with: - repository: github/docs-internal.pt-br - token: ${{ inputs.token }} - path: translations/pt-br - - - name: Clone Simplified Chinese - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - with: - repository: github/docs-internal.zh-cn - token: ${{ inputs.token }} - path: translations/zh-cn - - - name: Clone Russian - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - with: - repository: github/docs-internal.ru-ru - token: ${{ inputs.token }} - path: translations/ru-ru - - - name: Clone French - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - with: - repository: github/docs-internal.fr-fr - token: ${{ inputs.token }} - path: translations/fr-fr - - - name: Clone Korean - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - with: - repository: github/docs-internal.ko-kr - token: ${{ inputs.token }} - path: translations/ko-kr - - - name: Clone German - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - with: - repository: github/docs-internal.de-de - token: ${{ inputs.token }} - path: translations/de-de + # Clone in parallel; sequential actions/checkout steps took about 30s. + - name: Clone all translations + shell: bash + working-directory: ${{ github.workspace }} + env: + TOKEN: ${{ inputs.token }} + run: | + auth=$(printf 'x-access-token:%s' "$TOKEN" | base64 | tr -d '\n') + echo "::add-mask::$auth" + # Pass auth through the environment so the token stays out of process + # arguments and out of the clones' git config. + export GIT_CONFIG_COUNT=1 + export GIT_CONFIG_KEY_0=http.https://github.com/.extraheader + export GIT_CONFIG_VALUE_0="AUTHORIZATION: basic $auth" + + clone() { + local lang=$1 + for attempt in 1 2 3; do + rm -rf "translations/$lang" + git clone --quiet --depth 1 "https://github.com/github/docs-internal.$lang.git" "translations/$lang" \ + && return 0 + echo "Clone of $lang failed on attempt $attempt" + sleep $((attempt * 5)) + done + return 1 + } + + pids=() + for lang in es-es ja-jp pt-br zh-cn ru-ru fr-fr ko-kr de-de; do + clone "$lang" & + pids+=($!) + done + + status=0 + for pid in "${pids[@]}"; do + wait "$pid" || status=1 + done + exit $status diff --git a/.github/actions/node-npm-setup/action.yml b/.github/actions/node-npm-setup/action.yml index ce5e67de33fb..8e4bb2999881 100644 --- a/.github/actions/node-npm-setup/action.yml +++ b/.github/actions/node-npm-setup/action.yml @@ -15,11 +15,13 @@ runs: path: node_modules key: ${{ runner.os }}-node_modules-${{ hashFiles('package*.json') }}-${{ hashFiles('.github/actions/node-npm-setup/action.yml') }} + # The npm download cache only helps npm ci, so skip its 153 MB restore when node_modules hits. - name: Setup Node.js uses: actions/setup-node@2028fbc5c25fe9cf00d9f06a71cc4710d4507903 # v6.0.0 with: node-version-file: 'package.json' - cache: npm + package-manager-cache: false + cache: ${{ steps.cache-node_modules.outputs.cache-hit != 'true' && 'npm' || '' }} - name: Install dependencies if: ${{ steps.cache-node_modules.outputs.cache-hit != 'true' }} diff --git a/.github/actions/setup-elasticsearch/action.yml b/.github/actions/setup-elasticsearch/action.yml index 408fe9a794a2..8fc62bfe5048 100644 --- a/.github/actions/setup-elasticsearch/action.yml +++ b/.github/actions/setup-elasticsearch/action.yml @@ -10,82 +10,49 @@ inputs: elasticsearch_version: description: Version of Elasticsearch to install required: true - # Version must match production and be published on Docker Hub. + # Version must match production and be published on docker.elastic.co. default: '8.12.0' + mode: + description: >- + 'both' starts Elasticsearch and waits until it is ready. + 'start' starts it in the background and returns right away. + 'wait' waits for a background start from an earlier 'start' step, then for readiness. + required: false + default: 'both' runs: using: 'composite' steps: - # Cache the Elasticsearch image to prevent Docker Hub rate limits. - - name: Cache Docker layers - id: cache-docker-layers - uses: actions/cache@v4 - with: - path: /tmp/docker-cache - key: ${{ runner.os }}-elasticsearch-${{ inputs.elasticsearch_version }} - restore-keys: | - ${{ runner.os }}-elasticsearch- - - - name: Load cached Docker image - shell: bash - if: steps.cache-docker-layers.outputs.cache-hit == 'true' - run: docker load -i /tmp/docker-cache/elasticsearch.tar || echo "No cache found for elasticsearch, pulling image" - - - name: Pull Docker image + - name: Start Elasticsearch + if: ${{ inputs.mode == 'both' }} shell: bash - if: steps.cache-docker-layers.outputs.cache-hit != 'true' env: ES_VERSION: ${{ inputs.elasticsearch_version }} - run: docker pull elasticsearch:${ES_VERSION} + run: bash "$GITHUB_ACTION_PATH/start.sh" - - name: Save Docker image to cache + # Later steps such as npm ci and the build run while the image pulls and Elasticsearch boots. + - name: Start Elasticsearch in the background + if: ${{ inputs.mode == 'start' }} shell: bash - if: steps.cache-docker-layers.outputs.cache-hit != 'true' env: ES_VERSION: ${{ inputs.elasticsearch_version }} + ES_STATUS_FILE: ${{ runner.temp }}/elasticsearch-start.status + ES_LOG_FILE: ${{ runner.temp }}/elasticsearch-start.log run: | - mkdir -p /tmp/docker-cache - docker save -o /tmp/docker-cache/elasticsearch.tar elasticsearch:${ES_VERSION} - - # Run a single-node container with settings copied from getong/elasticsearch-action. - - name: Run Docker container + rm -f "$ES_STATUS_FILE" + ( + status=0 + bash "$GITHUB_ACTION_PATH/start.sh" || status=$? + echo "$status" > "$ES_STATUS_FILE.tmp" + mv "$ES_STATUS_FILE.tmp" "$ES_STATUS_FILE" + ) > "$ES_LOG_FILE" 2>&1 < /dev/null & + echo "Starting Elasticsearch in the background. A mode: wait step prints the log." + + - name: Wait for Elasticsearch + if: ${{ inputs.mode != 'start' }} shell: bash env: - INPUT_ELASTICSEARCH_VERSION: ${{ inputs.elasticsearch_version }} - INPUT_HOST_PORT: 9200 - INPUT_CONTAINER_PORT: 9200 - INPUT_HOST_NODE_PORT: 9300 - INPUT_NODE_PORT: 9300 - INPUT_DISCOVERY_TYPE: 'single-node' - run: | - docker network create elastic - - docker run --network elastic \ - -e 'node.name=es1' \ - -e 'cluster.name=docker-elasticsearch' \ - -e 'cluster.initial_master_nodes=es1' \ - -e 'discovery.seed_hosts=es1' \ - -e 'cluster.routing.allocation.disk.threshold_enabled=false' \ - -e 'bootstrap.memory_lock=true' \ - -e 'ES_JAVA_OPTS=-Xms1g -Xmx1g' \ - -e 'xpack.security.enabled=false' \ - -e 'xpack.license.self_generated.type=basic' \ - --ulimit nofile=65536:65536 \ - --ulimit memlock=-1:-1 \ - --name='es1' \ - -d \ - -p $INPUT_HOST_PORT:$INPUT_CONTAINER_PORT \ - -p $INPUT_HOST_NODE_PORT:$INPUT_NODE_PORT \ - -e discovery_type=$INPUT_DISCOVERY_TYPE \ - elasticsearch:$INPUT_ELASTICSEARCH_VERSION - - for i in {1..120}; do - if curl --silent --fail http://localhost:9200; then - echo "Elasticsearch is up and running" - exit 0 - fi - echo "Waiting for Elasticsearch to be ready..." - sleep 1 - done - echo "Elasticsearch did not become ready in time" - exit 1 + ES_BACKGROUND: ${{ inputs.mode == 'wait' }} + ES_STATUS_FILE: ${{ runner.temp }}/elasticsearch-start.status + ES_LOG_FILE: ${{ runner.temp }}/elasticsearch-start.log + run: bash "$GITHUB_ACTION_PATH/wait.sh" diff --git a/.github/actions/setup-elasticsearch/start.sh b/.github/actions/setup-elasticsearch/start.sh new file mode 100755 index 000000000000..300ebcc94804 --- /dev/null +++ b/.github/actions/setup-elasticsearch/start.sh @@ -0,0 +1,45 @@ +#!/bin/bash + +# Pull the Elasticsearch image and start the es1 container. wait.sh checks readiness. + +set -euo pipefail + +image="docker.elastic.co/elasticsearch/elasticsearch:$ES_VERSION" + +# Pull from Elastic's registry to avoid Docker Hub rate limits. +# A pull is faster than restoring and loading a cached image tarball. +pulled=false +for attempt in 1 2 3; do + if docker pull "$image"; then + pulled=true + break + fi + echo "Pull attempt $attempt failed" + sleep $((attempt * 10)) +done +if [ "$pulled" != true ]; then + echo "Could not pull $image" + exit 1 +fi + +# Run a single-node container with settings copied from getong/elasticsearch-action. +docker network create elastic + +docker run --network elastic \ + -e 'node.name=es1' \ + -e 'cluster.name=docker-elasticsearch' \ + -e 'cluster.initial_master_nodes=es1' \ + -e 'discovery.seed_hosts=es1' \ + -e 'cluster.routing.allocation.disk.threshold_enabled=false' \ + -e 'bootstrap.memory_lock=true' \ + -e 'ES_JAVA_OPTS=-Xms1g -Xmx1g' \ + -e 'xpack.security.enabled=false' \ + -e 'xpack.license.self_generated.type=basic' \ + --ulimit nofile=65536:65536 \ + --ulimit memlock=-1:-1 \ + --name='es1' \ + -d \ + -p 9200:9200 \ + -p 9300:9300 \ + -e discovery_type=single-node \ + "$image" diff --git a/.github/actions/setup-elasticsearch/wait.sh b/.github/actions/setup-elasticsearch/wait.sh new file mode 100755 index 000000000000..2a0054a4fd00 --- /dev/null +++ b/.github/actions/setup-elasticsearch/wait.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +# Wait until Elasticsearch answers on localhost:9200. +# With ES_BACKGROUND=true, first wait for the background start.sh to finish. + +set -euo pipefail + +if [ "${ES_BACKGROUND:-false}" = true ]; then + for _ in {1..300}; do + [ -f "$ES_STATUS_FILE" ] && break + sleep 1 + done + echo "::group::Background start log" + cat "$ES_LOG_FILE" || true + echo "::endgroup::" + if [ ! -f "$ES_STATUS_FILE" ]; then + echo "::error::Elasticsearch start did not finish in time. Did a mode: start step run first?" + exit 1 + fi + status=$(cat "$ES_STATUS_FILE") + if [ "$status" != 0 ]; then + echo "::error::Elasticsearch start failed with exit code $status" + exit 1 + fi +fi + +for _ in {1..120}; do + if curl --silent --fail http://localhost:9200; then + echo + echo "Elasticsearch is up and running" + exit 0 + fi + echo "Waiting for Elasticsearch to be ready..." + sleep 1 +done +echo "::error::Elasticsearch did not become ready in time" +docker logs --tail 100 es1 || true +exit 1 diff --git a/.github/workflows/index-general-search-pr.yml b/.github/workflows/index-general-search-pr.yml index 26be1f8aabcd..74e5a6656ec4 100644 --- a/.github/workflows/index-general-search-pr.yml +++ b/.github/workflows/index-general-search-pr.yml @@ -12,7 +12,7 @@ on: # Debugging changes to this workflow need the same PR index test. - .github/workflows/index-general-search-pr.yml # Setup changes can break the local Elasticsearch path this workflow tests. - - .github/actions/setup-elasticsearch/action.yml + - '.github/actions/setup-elasticsearch/**' permissions: contents: read diff --git a/.github/workflows/package-lock-lint.yml b/.github/workflows/package-lock-lint.yml index 8cf827c7cfa2..43333e453355 100644 --- a/.github/workflows/package-lock-lint.yml +++ b/.github/workflows/package-lock-lint.yml @@ -1,10 +1,12 @@ name: Package lock lint # This workflow catches manual package.json edits that leave package-lock.json out of sync. +# It also blocks Microsoft package feed URLs, which break installs outside GitHub. on: pull_request: paths: + - .npmrc - package.json - package-lock.json - .github/workflows/package-lock-lint.yml @@ -25,6 +27,18 @@ jobs: - name: Check out repo uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - name: Check for package feed URLs + run: | + status=0 + grep -inE 'https?://[^/"]*(pkgs\.visualstudio\.com|pkgs\.dev\.azure\.com|packagefeedproxy\.microsoft\.io)' .npmrc package-lock.json || status=$? + if [ "$status" -eq 0 ]; then + echo "::error::Found Microsoft package feed URLs. Use https://registry.npmjs.org/ instead." + exit 1 + elif [ "$status" -ne 1 ]; then + echo "::error::Could not scan .npmrc and package-lock.json for package feed URLs." + exit "$status" + fi + - name: Setup Node.js uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 with: diff --git a/.github/workflows/purge-fastly.yml b/.github/workflows/purge-fastly.yml index a911270c2168..1e350232c7a5 100644 --- a/.github/workflows/purge-fastly.yml +++ b/.github/workflows/purge-fastly.yml @@ -11,13 +11,13 @@ on: description: "Languages: Comma separated languages, e.g. 'en,es,ja,pt,zh,ru,fr,ko,de'. Blank = all languages." required: false default: 'en' - hard: - description: 'Hard purge: Evict immediately instead of the default soft purge. Use when a soft purge fails to clear stale content.' + soft: + description: 'Soft purge: Mark stale instead of evicting. Readers keep getting the old copy while Fastly refetches. Off by default, so purges evict immediately.' type: boolean required: false default: false everything: - description: 'Everything: Hard-purge the entire Fastly cache... every key, all readers. Ignores the languages/hard inputs. To confirm, type exactly: "purge everything". Otherwise leave blank.' + description: 'Everything: Hard-purge the entire Fastly cache... every key, all readers. Ignores the languages/soft inputs. To confirm, type exactly: "purge everything". Otherwise leave blank.' required: false default: '' @@ -54,7 +54,7 @@ jobs: - name: Validate confirmation input # A full-cache purge requires the exact string "purge everything"; typos must fail - # instead of falling through to a green soft purge. + # instead of falling through to a green language purge. env: EVERYTHING_INPUT: ${{ inputs.everything }} run: | @@ -73,15 +73,15 @@ jobs: if: ${{ github.event_name == 'workflow_dispatch' }} env: LANGUAGES_INPUT: ${{ inputs.languages }} - HARD_INPUT: ${{ inputs.hard }} + SOFT_INPUT: ${{ inputs.soft }} EVERYTHING_INPUT: ${{ inputs.everything }} run: | args=() if [ -n "$LANGUAGES_INPUT" ]; then args+=(--languages "$LANGUAGES_INPUT") fi - if [ "$HARD_INPUT" = "true" ]; then - args+=(--hard) + if [ "$SOFT_INPUT" = "true" ]; then + args+=(--soft) fi if [ "$EVERYTHING_INPUT" = "purge everything" ]; then args+=(--everything) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3df8e48230d9..cfcbd5b88aeb 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -82,6 +82,11 @@ jobs: - name: languages isPrivateRepo: false + env: + # These suites make no HTTP requests, so they skip the Next.js build and test server. + # A suite that adds an HTTP test fails with a connection error until it's removed from this list. + START_VITEST_SERVER: ${{ contains(fromJSON('["color-schemes", "content-linter", "data-directory", "ghes-releases", "journeys", "links", "observability", "products", "workflows"]'), matrix.name) && 'false' || 'true' }} + steps: # Repeat each if on its step so skipped work still leaves the required check present. - name: Check out repo @@ -90,8 +95,11 @@ jobs: # Fetch 2 commits so tj-actions/changed-files can diff without extra API calls fetch-depth: 2 + # Start Elasticsearch early so the pull and boot overlap with setup and build. - uses: ./.github/actions/setup-elasticsearch if: ${{ matrix.name == 'search' || matrix.name == 'languages' }} + with: + mode: start - uses: ./.github/actions/node-npm-setup @@ -133,14 +141,21 @@ jobs: run: echo "${{ steps.changed_files.outputs.all_modified_files }}" > get_diff_files.txt - uses: ./.github/actions/cache-nextjs + if: ${{ env.START_VITEST_SERVER != 'false' }} - name: Run build script + if: ${{ env.START_VITEST_SERVER != 'false' }} run: npm run build - uses: ./.github/actions/warmup-remotejson-cache # Only routing tests cover archived enterprise server URLs. if: ${{ matrix.name == 'redirects' }} + - uses: ./.github/actions/setup-elasticsearch + if: ${{ matrix.name == 'search' || matrix.name == 'languages' }} + with: + mode: wait + - name: Index fixtures into the local Elasticsearch # Run indexing only for suites that query the local Elasticsearch service. if: ${{ matrix.name == 'search' || matrix.name == 'languages' }} diff --git a/.npmrc b/.npmrc index 083dd818fdcb..69d591817111 100644 --- a/.npmrc +++ b/.npmrc @@ -3,3 +3,7 @@ # Also we have Dependabot alerts configured in the GitHub repo. audit=false fund=false + +# Skip package versions published in the last 7 days, matching the Dependabot cooldown. +# Needs npm 11.10 or later. +min-release-age=7 diff --git a/Dockerfile b/Dockerfile index 8cb8e6af6a07..873b06c422cd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,7 +4,7 @@ # Update the base image digest from the gh-base-noble package page: # https://github.com/github/gh-base-image/pkgs/container/gh-base-image%2Fgh-base-noble -FROM ghcr.io/github/gh-base-image/gh-base-noble:20260914-014148-gb620b63bf@sha256:fe199dcd96e01f53c42d077dee87f428e8341379aab0987721e16b32462feb05 AS base +FROM ghcr.io/github/gh-base-image/gh-base-noble:20261007-171807-gaf2071665@sha256:57e57e0d0dd8cf12cbf047aa5ae7ead850f562bc66bace621598a460b803efc7 AS base # Install curl for NodeSource setup. # Install git for early-access and translation clones. # Ubuntu's nodejs package lags the Node LTS release line. diff --git a/content/copilot/concepts/agents/code-review.md b/content/copilot/concepts/agents/code-review.md index 3da2b29f1371..2e6fc8a9dfc1 100644 --- a/content/copilot/concepts/agents/code-review.md +++ b/content/copilot/concepts/agents/code-review.md @@ -98,18 +98,20 @@ Consumption generally increases with pull request size and repository custom ins Repository and organization administrators can set the default review effort level for automatic code reviews. For configuration steps, see [AUTOTITLE](/copilot/how-tos/copilot-on-github/set-up-copilot/configure-code-review#customizing-copilot-code-review-1). -## Code review usage +## Code review usage and billing Each time {% data variables.product.prodname_copilot_short %} reviews a pull request or reviews code in your IDE, the interaction consumes {% data variables.product.prodname_ai_credits_short %}. The amount depends on the model used and the number of tokens processed. Code reviews have two cost components: {% data variables.product.prodname_ai_credits_short %} for the model interaction (the review itself), and {% data variables.product.prodname_actions %} minutes for the agentic capabilities (context gathering and tool use). For more information on {% data variables.product.prodname_actions %} usage, see [Usage of {% data variables.product.prodname_actions %} runners for agentic capabilities in code review](#usage-of-github-actions-runners-for-agentic-capabilities-in-code-review). -If a repository is configured to automatically request a code review from {% data variables.product.prodname_copilot_short %} for all new pull requests, the {% data variables.product.prodname_ai_credits_short %} consumption is attributed to the pull request author. If a review is manually requested by another user, the consumption is attributed to that user instead. +By default, when a repository automatically requests a code review, {% data variables.product.prodname_ai_credits_short %} consumption is attributed to the pull request author. If another user manually requests a review, consumption is attributed to that user instead. For pull requests authored by {% data variables.copilot.copilot_cloud_agent %}, usage is attributed first to the human co-author associated with the change. If the co-author cannot be billed, usage is billed directly to the organization. For pull requests authored by other bots, or when a bot requests the review, usage is billed directly to the organization. These pull requests are eligible for agentic review. +For requests associated with organization members who have paid {% data variables.product.prodname_copilot_short %} licenses, the member's entitlement is billed by default. An enterprise or organization owner can instead choose to bill the organization. Billing the organization requires {% data variables.product.prodname_ai_credits_short %} paid usage, but a budget is optional. This choice applies to both manually requested and automatic code reviews, and changes billing only. On its own, it does not grant access to {% data variables.copilot.copilot_code-review_short %}. + ### Reviews requested with an external {% data variables.product.prodname_copilot_short %} license An external {% data variables.product.prodname_copilot_short %} license is one that is not provided by the organization or enterprise that owns the repository, such as a personal license or a license from another organization. By default, people can use an external license to request a review from {% data variables.product.prodname_copilot_short %}. @@ -122,17 +124,17 @@ To prevent this, organization owners and repository administrators can enable th For configuration steps, see [AUTOTITLE](/copilot/how-tos/copilot-on-github/set-up-copilot/configure-code-review#customizing-copilot-code-review). -### What happens when a budget is reached - -For {% data variables.copilot.copilot_business_short %} and {% data variables.copilot.copilot_enterprise_short %}, code review access is governed by budget controls. If a user reaches their user-level budget, or if the enterprise or cost center spending limit is exhausted, code reviews are blocked along with other {% data variables.product.prodname_ai_credits_short %}-consuming features. See [AUTOTITLE](/copilot/concepts/billing-and-usage/organizations-and-enterprises/budgets#what-happens-when-a-user-is-blocked). - ### Users without a {% data variables.product.prodname_copilot_short %} license or plan that includes {% data variables.copilot.copilot_code-review_short %} Users without access to {% data variables.copilot.copilot_code-review_short %} do not have a monthly allowance of {% data variables.product.prodname_ai_credits_short %} for it. This includes users who have no {% data variables.product.prodname_copilot_short %} license and users on the {% data variables.copilot.copilot_free_short %} plan, which does not include {% data variables.copilot.copilot_code-review_short %}. When {% data variables.copilot.copilot_code-review_short %} is enabled for these users, any {% data variables.product.prodname_ai_credits_short %} they consume are billed directly to the organization or enterprise as paid additional usage. This applies to both manually requested reviews and automatic code reviews. -{% data variables.product.prodname_ai_credits_short %} consumed by these users are not attributed to any individual user's budget. They appear as additional usage in billing reports. Users with a {% data variables.product.prodname_copilot_short %} license that includes code review consume {% data variables.product.prodname_ai_credits_short %} from the shared pool, subject to any user-level budgets configured by their administrator. +{% data variables.product.prodname_ai_credits_short %} consumed by these users are not attributed to any individual user's budget. They appear as additional usage in billing reports. + +### What happens when a budget is reached + +For {% data variables.copilot.copilot_business_short %} and {% data variables.copilot.copilot_enterprise_short %}, code review access is governed by the budgets that apply to the selected billing source. If usage is billed to a user, code reviews are blocked when the user reaches their budget. If usage is billed to an organization, code reviews are blocked when the applicable organization budget, cost center budget, or enterprise spending limit is exhausted. Code reviews are blocked along with other {% data variables.product.prodname_ai_credits_short %}-consuming features. See [AUTOTITLE](/copilot/concepts/billing-and-usage/organizations-and-enterprises/budgets#what-happens-when-a-user-is-blocked). ## Model usage diff --git a/content/copilot/how-tos/copilot-on-github/set-up-copilot/configure-code-review.md b/content/copilot/how-tos/copilot-on-github/set-up-copilot/configure-code-review.md index a632b9908be7..824537de3ae0 100644 --- a/content/copilot/how-tos/copilot-on-github/set-up-copilot/configure-code-review.md +++ b/content/copilot/how-tos/copilot-on-github/set-up-copilot/configure-code-review.md @@ -122,6 +122,11 @@ You can configure how {% data variables.copilot.copilot_code-review_short %} com * **Balanced**: Deeper analysis of complex logic, security-sensitive code, and cross-service changes. Balanced reviews use more {% data variables.product.prodname_ai_credits_short %}, and may consume marginally more {% data variables.product.prodname_actions %} minutes. See [AUTOTITLE](/copilot/concepts/agents/code-review#estimated-consumption). +1. To choose how {% data variables.product.prodname_ai_credits_short %} usage associated with members who have paid {% data variables.product.prodname_copilot_short %} licenses is billed, next to **Choose how members with a {% data variables.product.prodname_copilot_short %} license are billed**, select an option. + * **Member**: Use the member's {% data variables.product.prodname_copilot_short %} entitlement. If the member's quota is exhausted, the code review fails. This is the default. + * **Organization**: Bill the organization. + + This choice applies to both manually requested and automatic code reviews, and changes billing only. It does not grant access to {% data variables.copilot.copilot_code-review_short %}. To use **Organization**, {% data variables.product.prodname_ai_credits_short %} paid usage must be enabled for the organization. Setting a budget is optional. See [AUTOTITLE](/copilot/concepts/billing/organizations-and-enterprises/usage-based-billing#what-happens-if-i-exceed-my-included-ai-credits). 1. To choose whether {% data variables.product.prodname_copilot_short %} can approve pull requests in your repositories, select an option under "Approvals," next to "Count {% data variables.product.prodname_copilot_short %} approvals toward merge requirements." * **Enabled everywhere**: {% data variables.product.prodname_copilot_short %} approvals can count toward merge requirements in every repository in the organization. * **Let repositories decide**: Repository admins can decide this in repository settings. @@ -154,6 +159,12 @@ You can configure how {% data variables.copilot.copilot_code-review_short %} com 1. Scroll down to "Available Agents", then click **{% data variables.copilot.copilot_code-review_short %}**. 1. Next to "{% data variables.copilot.copilot_code-review_short %}", select a policy. 1. Then choose which {% data variables.copilot.copilot_code-review_short %} features to enable for your enterprise. +1. To choose who is billed for {% data variables.copilot.copilot_code-review_short %} usage by members with a paid {% data variables.product.prodname_copilot_short %} license, next to "Choose how members with a {% data variables.product.prodname_copilot_short %} license are billed," select a policy. + * **Member**: Use the member's {% data variables.product.prodname_copilot_short %} entitlement for organizations in the enterprise. + * **Organization**: Bill the organization that owns the repository. + * **Let organizations decide**: Organization owners can choose how to bill usage associated with members who have paid {% data variables.product.prodname_copilot_short %} licenses. + + Organizations billed must have {% data variables.product.prodname_ai_credits_short %} paid usage enabled. Setting a budget is optional. 1. To choose whether {% data variables.product.prodname_copilot_short %} can approve pull requests in your organizations, next to "Allow {% data variables.product.prodname_copilot_short %} to approve pull requests," select a policy. * **Let organizations decide**: Organization owners can choose whether to enable {% data variables.product.prodname_copilot_short %} approvals. * **Enable for selected organizations**: {% data variables.product.prodname_copilot_short %} approvals are enabled only for the organizations you select. diff --git a/content/copilot/how-tos/manage-your-account/get-started-with-a-copilot-plan.md b/content/copilot/how-tos/manage-your-account/get-started-with-a-copilot-plan.md index 5e73fdcadb98..97d129a96231 100644 --- a/content/copilot/how-tos/manage-your-account/get-started-with-a-copilot-plan.md +++ b/content/copilot/how-tos/manage-your-account/get-started-with-a-copilot-plan.md @@ -112,6 +112,15 @@ If the plan does not appear after following the checkout steps [when subscribing 1. If payment method verification failed or you did not complete the activation step, retry the checkout flow. 1. If you completed activation and the plan still does not appear, contact {% data variables.contact.contact_support_page %}. +### Plan canceled after a failed renewal + +If your personal {% data variables.copilot.copilot_pro_short %} or {% data variables.copilot.copilot_pro_plus_short %} subscription was canceled after failed renewal attempts, **Billing & licensing** may show {% data variables.copilot.copilot_free_short %}. You may not see **Pay now**, **Upgrade**, or **Reactivate**, even after updating your payment method. This is different from an incomplete checkout: updating your payment method alone does not restore a canceled plan. + +1. Confirm your current plan in your personal account settings under **Billing & licensing**. +1. When individual sign-up is available, start a new subscription by following the steps in [Subscribing to a paid plan](#subscribing-to-copilot-pro-copilot-pro-or-copilot-max). + +Restoring a canceled plan is treated as a new individual subscription. If individual sign-up is temporarily unavailable, there is no self-service way to restore paid access until sign-up becomes available again. + ### Blocked plan setup on a personal account Your personal account may have a {% data variables.product.prodname_copilot_short %} access restriction if you see: diff --git a/content/copilot/reference/copilot-billing/models-and-pricing.md b/content/copilot/reference/copilot-billing/models-and-pricing.md index 7ee292ac7c29..0a4752ce7d2a 100644 --- a/content/copilot/reference/copilot-billing/models-and-pricing.md +++ b/content/copilot/reference/copilot-billing/models-and-pricing.md @@ -107,7 +107,9 @@ For most {% data variables.product.prodname_copilot_short %} features, the model Each code review is billed in two ways: token consumption is billed in {% data variables.product.prodname_ai_credits_short %}, and the agentic infrastructure that powers the review consumes {% data variables.product.prodname_actions %} minutes. -{% data variables.product.prodname_actions %} minutes are attributed to the repository, and from there to the enterprise or cost center where applicable. {% data variables.product.prodname_ai_credits_short %} are charged to the person who requests the review, or to the author of a pull request where a policy automatically triggers a review. If that person does not have a {% data variables.product.prodname_copilot_short %} seat, usage is billed to the enterprise or cost center instead. For pull requests authored by {% data variables.copilot.copilot_cloud_agent %}, usage is attributed first to the human co-author associated with the change. If the co-author cannot be billed, usage is billed directly to the organization. For pull requests authored by other bots, or when a bot requests the review, usage is also billed directly to the organization. These pull requests are eligible for agentic review. +{% data variables.product.prodname_actions %} minutes are attributed to the repository, and from there to the enterprise or cost center where applicable. For requests associated with organization members who have paid {% data variables.product.prodname_copilot_short %} licenses, {% data variables.product.prodname_ai_credits_short %} are charged to the member or organization, depending on how the **Choose how members with a {% data variables.product.prodname_copilot_short %} license are billed** policy is configured. + +If the person associated with a review does not have a {% data variables.product.prodname_copilot_short %} seat, usage is billed to the enterprise or cost center instead. For pull requests authored by {% data variables.copilot.copilot_cloud_agent %}, usage is attributed first to the human co-author associated with the change. If the co-author cannot be billed, usage is billed directly to the organization. For pull requests authored by other bots, or when a bot requests the review, usage is also billed directly to the organization. These pull requests are eligible for agentic review. You can view your current {% data variables.product.prodname_actions %} usage for {% data variables.copilot.copilot_code-review_short %} in the following ways: diff --git a/package-lock.json b/package-lock.json index 19c42b66fa87..4981c2ace5bd 100644 --- a/package-lock.json +++ b/package-lock.json @@ -64,7 +64,7 @@ "mdast-util-to-hast": "^13.2.1", "mdast-util-to-markdown": "2.1.2", "mdast-util-to-string": "^4.0.0", - "next": "^16.3.6", + "next": "^16.3.8", "parse5": "8.0.1", "quick-lru": "7.0.1", "react": "^19.2.5", @@ -2182,15 +2182,15 @@ } }, "node_modules/@next/env": { - "version": "16.3.6", - "resolved": "https://registry.npmjs.org/@next/env/-/env-16.3.6.tgz", - "integrity": "sha512-x9Vblze1EbtltQYnNH38xCPWU3TVfBd1eXqA3+w9+BTpedkkdNpAaltXlGQ/nsc1+E0mVTNrtcbX3GoO09zeLQ==", + "version": "16.3.8", + "resolved": "https://registry.npmjs.org/@next/env/-/env-16.3.8.tgz", + "integrity": "sha512-Al9zqHVV7TJv0eFuOU4U7Lvv74PTih4Ch63sk2xCIpSTkE3udFnaOcnzP2lQVymiL7yS9Cj2iClUXlR3EQ5sEw==", "license": "MIT" }, "node_modules/@next/swc-darwin-arm64": { - "version": "16.3.6", - "resolved": "https://registry.npmjs.org/@next/swc-darwin-arm64/-/swc-darwin-arm64-16.3.6.tgz", - "integrity": "sha512-E/7GEqaUkt8mk/T8v9lAnrhzR06kdq1ZBkC12F8tAMkdIadwNp3H1KqHynDHrpcTlGCUdq/qu6vUL2aYVyYBdw==", + "version": "16.3.8", + "resolved": "https://registry.npmjs.org/@next/swc-darwin-arm64/-/swc-darwin-arm64-16.3.8.tgz", + "integrity": "sha512-2JPRMh2nmQG5CiL7cXGL9AGwnPWJQ//cTtAUCT+w511QHk79SYz3LGv/pc5X643B/WEO0rvu3Yww0hqwt3kgeA==", "cpu": [ "arm64" ], @@ -2204,9 +2204,9 @@ } }, "node_modules/@next/swc-darwin-x64": { - "version": "16.3.6", - "resolved": "https://registry.npmjs.org/@next/swc-darwin-x64/-/swc-darwin-x64-16.3.6.tgz", - "integrity": "sha512-yBE893/nDWTlaiBD1p+qgt7NUen4U5R6FXyH0s67Npq1S3E0cVSef1WIXC2xBRgQvwAvJq6DnS6Y6PrY0cy4Ew==", + "version": "16.3.8", + "resolved": "https://registry.npmjs.org/@next/swc-darwin-x64/-/swc-darwin-x64-16.3.8.tgz", + "integrity": "sha512-GZtCCOBKJ4leVIT/Th0llWKhD1ca92lzbQiS5R5ON9QkoiFnilFsebDae1JU2a3HWoKMEmEZWGs1AGLavVM72Q==", "cpu": [ "x64" ], @@ -2220,9 +2220,9 @@ } }, "node_modules/@next/swc-linux-arm64-gnu": { - "version": "16.3.6", - "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-gnu/-/swc-linux-arm64-gnu-16.3.6.tgz", - "integrity": "sha512-KJDpjBqBPYlvkivmyrp+Qys6k/7ksbqGQvRVc6ZEGfR+cjQxx+nUkJaWmNZJsmoOrqYNbaXByF8wa0lBwDhB3Q==", + "version": "16.3.8", + "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-gnu/-/swc-linux-arm64-gnu-16.3.8.tgz", + "integrity": "sha512-O659ygeQYqneJ1fBKMpFxIFqYkYswu8IAS1OCKK/4f3ZgJJm1dRz4fVJZRi/kLLWjnBKnebOePA4WNv+sV1pVA==", "cpu": [ "arm64" ], @@ -2239,9 +2239,9 @@ } }, "node_modules/@next/swc-linux-arm64-musl": { - "version": "16.3.6", - "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-musl/-/swc-linux-arm64-musl-16.3.6.tgz", - "integrity": "sha512-mqNg2K+hvWskSRb/QM+Ix412DvBsuSF0XV+frTSw5vmoucNnIlynFwKYew8D01bfATErMOM7Bujrf0BA5DRKFA==", + "version": "16.3.8", + "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-musl/-/swc-linux-arm64-musl-16.3.8.tgz", + "integrity": "sha512-dSjKSyWpzxoO1d3DIZZcP4XJcNaKeLmxQMFOiYl5vuBRMmweIqnAhty8tAmRsvTss779cK1FtYnDMj40e4TQlg==", "cpu": [ "arm64" ], @@ -2258,9 +2258,9 @@ } }, "node_modules/@next/swc-linux-x64-gnu": { - "version": "16.3.6", - "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-16.3.6.tgz", - "integrity": "sha512-nFncBNGAYouRHjRVaITs9beZRfhX4ssVwpnvPIAbkZVH6LtGoAVlH4bJ8Cnf9SOo9bsXgPFer/GdHtEE3JNOkw==", + "version": "16.3.8", + "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-16.3.8.tgz", + "integrity": "sha512-lbqOuz3RPRcv+o9msNsJw5x4+Y1ZwPTs6vmL6DCf7i0fZfvng/F59wyeDwqHIvV0mK//RBy/jJkZ+nCKsSMXjQ==", "cpu": [ "x64" ], @@ -2277,9 +2277,9 @@ } }, "node_modules/@next/swc-linux-x64-musl": { - "version": "16.3.6", - "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-musl/-/swc-linux-x64-musl-16.3.6.tgz", - "integrity": "sha512-5Mf3cHDGR/Iz0ng2Bj3zUR3p5QS9YK3Hn2QiAfavFmyF48zwThAjpFoiTKNIcOHLYS4zEk+gzyJ/9deQ2ZB8yQ==", + "version": "16.3.8", + "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-musl/-/swc-linux-x64-musl-16.3.8.tgz", + "integrity": "sha512-+316WswI8ScVgZeUd+1KGaXkHhaYQzCjvH/05TZSpJ8zBizb1a4G7DtO7F12jcBIqMOtsz9ji1t48fmKtzqsGA==", "cpu": [ "x64" ], @@ -2296,9 +2296,9 @@ } }, "node_modules/@next/swc-win32-arm64-msvc": { - "version": "16.3.6", - "resolved": "https://registry.npmjs.org/@next/swc-win32-arm64-msvc/-/swc-win32-arm64-msvc-16.3.6.tgz", - "integrity": "sha512-0jkJy0C2kbrJWTk4YLa3xk80pVBpx8FCHJym7CnUfDAXe/FWv5qT7SQJbR0KuemyxaEDlEx5WT4VQJoTW+/9Qw==", + "version": "16.3.8", + "resolved": "https://registry.npmjs.org/@next/swc-win32-arm64-msvc/-/swc-win32-arm64-msvc-16.3.8.tgz", + "integrity": "sha512-ji0gd4kMYUxO+1fJBIbiBVRCjzG/lloiyCccnlebvb1ZJ5qXCPZqYg4Jl1DrrixnWNMKylzgpmMWx0yNDYXlzw==", "cpu": [ "arm64" ], @@ -2312,9 +2312,9 @@ } }, "node_modules/@next/swc-win32-x64-msvc": { - "version": "16.3.6", - "resolved": "https://registry.npmjs.org/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-16.3.6.tgz", - "integrity": "sha512-/YXjI1e5OXcZ7YpxRwgP/1jAV/SBKTzeVKqN2mk7mLpcICsyn3Gl5+dIfDTJp70M0ccMhyMMRso4v6mPDCGepg==", + "version": "16.3.8", + "resolved": "https://registry.npmjs.org/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-16.3.8.tgz", + "integrity": "sha512-WcTlaKt/TWkh5kUjdJcUmB1XgZ+1c6fz4Y9fDHL73YNSdGaUWjceeWrrlwF0nv19iABYWC4iAq1oX1w4Bn0vfg==", "cpu": [ "x64" ], @@ -12183,12 +12183,12 @@ } }, "node_modules/next": { - "version": "16.3.6", - "resolved": "https://registry.npmjs.org/next/-/next-16.3.6.tgz", - "integrity": "sha512-L+otWM/aQbYTx98aZhgEoMb4bZAXx1YVW4UMA/vuCyCoWG5HJyZUili8QAkqzrcC+5///tsz3s0M+SlyB5bLMw==", + "version": "16.3.8", + "resolved": "https://registry.npmjs.org/next/-/next-16.3.8.tgz", + "integrity": "sha512-U7QEZaTini6wKrb8A8hqLLqYQyCetegKjCpJOyxk642vWoMoU1x5PyZCJFvgYgiptA8xc5j/9xYlZFO7w9Sjmw==", "license": "MIT", "dependencies": { - "@next/env": "16.3.6", + "@next/env": "16.3.8", "@swc/helpers": "0.5.23", "baseline-browser-mapping": "^2.9.19", "caniuse-lite": "^1.0.30001579", @@ -12202,14 +12202,14 @@ "node": ">=20.9.0" }, "optionalDependencies": { - "@next/swc-darwin-arm64": "16.3.6", - "@next/swc-darwin-x64": "16.3.6", - "@next/swc-linux-arm64-gnu": "16.3.6", - "@next/swc-linux-arm64-musl": "16.3.6", - "@next/swc-linux-x64-gnu": "16.3.6", - "@next/swc-linux-x64-musl": "16.3.6", - "@next/swc-win32-arm64-msvc": "16.3.6", - "@next/swc-win32-x64-msvc": "16.3.6", + "@next/swc-darwin-arm64": "16.3.8", + "@next/swc-darwin-x64": "16.3.8", + "@next/swc-linux-arm64-gnu": "16.3.8", + "@next/swc-linux-arm64-musl": "16.3.8", + "@next/swc-linux-x64-gnu": "16.3.8", + "@next/swc-linux-x64-musl": "16.3.8", + "@next/swc-win32-arm64-msvc": "16.3.8", + "@next/swc-win32-x64-msvc": "16.3.8", "sharp": "^0.35.4" }, "peerDependencies": { diff --git a/package.json b/package.json index 8fdd4b54d0f7..1400e5d04cf4 100644 --- a/package.json +++ b/package.json @@ -223,7 +223,7 @@ "mdast-util-to-hast": "^13.2.1", "mdast-util-to-markdown": "2.1.2", "mdast-util-to-string": "^4.0.0", - "next": "^16.3.6", + "next": "^16.3.8", "parse5": "8.0.1", "quick-lru": "7.0.1", "react": "^19.2.5", diff --git a/src/frame/lib/get-remote-json.ts b/src/frame/lib/get-remote-json.ts index de2b1b5c489c..4580a750d794 100644 --- a/src/frame/lib/get-remote-json.ts +++ b/src/frame/lib/get-remote-json.ts @@ -6,11 +6,17 @@ import zlib from 'zlib' import { fetchWithRetry } from './fetch-utils' import statsd from '@/observability/lib/statsd' -// Cache deflated raw redirect JSON strings instead of parsed objects. -// Parsed 5-10 MB files compress to about 1-2 MB with deflate. Each access pays -// about 1 ms to inflate and parse, which is small beside the memory savings. +// Cache deflated raw redirect JSON strings for every URL. +// Raw files of up to 20 MB compress to under 1 MB with deflate. export const cache = new Map() +// Also keep recently used parsed objects, bounded by raw JSON size. +// Inflating and parsing a 6-20 MB file costs about 15-40 ms per call. +// Parsed heap use is about 1.4 times the raw size. +export const PARSED_CACHE_MAX_BYTES = 24 * 1024 * 1024 +export const parsedCache = new Map() +let parsedCacheBytes = 0 + const inProd = process.env.NODE_ENV === 'production' interface GetRemoteJSONConfig { @@ -26,10 +32,43 @@ function compressStringToCache(cacheKey: string, jsonString: string): void { cache.set(cacheKey, zlib.deflateSync(Buffer.from(jsonString))) } +function forgetParsed(cacheKey: string): void { + const entry = parsedCache.get(cacheKey) + if (!entry) return + parsedCache.delete(cacheKey) + parsedCacheBytes -= entry.bytes +} + +function rememberParsed(cacheKey: string, value: unknown, bytes: number): void { + forgetParsed(cacheKey) + if (bytes > PARSED_CACHE_MAX_BYTES) return + // Callers share this object, so freeze it to prevent accidental mutation. + if (value && typeof value === 'object') Object.freeze(value) + parsedCache.set(cacheKey, { value, bytes }) + parsedCacheBytes += bytes + // Map iteration follows insertion order, so the first key is least recently used. + for (const [key] of parsedCache) { + if (parsedCacheBytes <= PARSED_CACHE_MAX_BYTES) break + forgetParsed(key) + } +} + +function getParsed(cacheKey: string): { value: unknown } | undefined { + const entry = parsedCache.get(cacheKey) + if (!entry) return undefined + // Reinsert to mark as most recently used. + parsedCache.delete(cacheKey) + parsedCache.set(cacheKey, entry) + return entry +} + function decompressFromCache(cacheKey: string): unknown { const compressed = cache.get(cacheKey) if (!compressed) return undefined - return JSON.parse(zlib.inflateSync(compressed).toString()) + const raw = zlib.inflateSync(compressed) + const value = JSON.parse(raw.toString()) + rememberParsed(cacheKey, value, raw.length) + return value } // Archived redirects.json files from docs-ghes- repos are large and static. @@ -62,8 +101,9 @@ export default async function getRemoteJSON( if (body) { try { // Compress the raw string after validation to avoid parse-stringify overhead. - JSON.parse(body) + const parsed = JSON.parse(body) compressStringToCache(cacheKey, body) + rememberParsed(cacheKey, parsed, Buffer.byteLength(body)) fromCache = 'disk' foundOnDisk = true } catch (error) { @@ -108,8 +148,9 @@ export default async function getRemoteJSON( } const body = await res.text() - JSON.parse(body) + const parsed = JSON.parse(body) compressStringToCache(cacheKey, body) + rememberParsed(cacheKey, parsed, Buffer.byteLength(body)) // Local review and tests persist downloads so later runs can reuse them. if (!inProd) { @@ -118,7 +159,9 @@ export default async function getRemoteJSON( } } } + const parsed = getParsed(cacheKey) const tags = [`from_cache:${fromCache}`] + if (fromCache === 'memory') tags.push(`parsed_cache:${parsed ? 'hit' : 'miss'}`) statsd.increment('middleware.get_remote_json', 1, tags) - return decompressFromCache(cacheKey) + return parsed ? parsed.value : decompressFromCache(cacheKey) } diff --git a/src/frame/middleware/find-page.ts b/src/frame/middleware/find-page.ts index 12ea98233c80..b72f95ad3dfe 100644 --- a/src/frame/middleware/find-page.ts +++ b/src/frame/middleware/find-page.ts @@ -95,8 +95,12 @@ async function rereadByPath( if (!match) return null const languageCode = match[1] const withoutLanguage = uri.replace(languagePrefixPathRegex, '/') - const withoutVersion = withoutLanguage.replace(`/${currentVersion}`, '') + // Page.read turns backslashes into slashes, so do the same before the root check. + const withoutVersion = withoutLanguage.replace(`/${currentVersion}`, '').replace(/\\/g, '/') const possible = path.join(contentRoot, withoutVersion) + // Express keeps `..` segments in request paths, so stay inside the content root. + const relativeToRoot = path.relative(contentRoot, possible) + if (relativeToRoot.startsWith('..') || path.isAbsolute(relativeToRoot)) return null const filePath = existsSync(possible) ? path.join(possible, 'index.md') : `${possible}.md` const relativePath = path.relative(contentRoot, filePath) const basePath = contentRoot diff --git a/src/frame/tests/find-page-middleware.ts b/src/frame/tests/find-page-middleware.ts index 9aa21f4047a5..d5e1352b9d13 100644 --- a/src/frame/tests/find-page-middleware.ts +++ b/src/frame/tests/find-page-middleware.ts @@ -3,7 +3,7 @@ import path from 'path' import http from 'http' import { Socket } from 'net' -import { describe, expect, test } from 'vitest' +import { describe, expect, test, vi } from 'vitest' import type { Response } from 'express' import Page from '@/frame/lib/page' @@ -149,6 +149,35 @@ describe('find page middleware', () => { expect(req.context?.page).toBeUndefined() }) + test.each(['/en/../README', '/en/sub\\..\\..\\README', '/en/child/..\\..\\README'])( + 'does not re-read paths outside the content root: %s', + async (url) => { + const [req, res] = makeRequestResponse(url) + const page = await Page.init({ + relativePath: 'page-with-redirects.md', + basePath: path.join(__dirname, '../../../src/fixtures/fixtures'), + languageCode: 'en', + }) + if (page && req.context) { + req.context.pages = { + [url]: page, + } + } + + const initSpy = vi.spyOn(Page, 'init') + try { + await findPage(req, res, () => {}, { + isDev: true, + contentRoot: path.join(__dirname, '../../../src/fixtures/fixtures'), + }) + expect(initSpy).not.toHaveBeenCalled() + expect(req.context?.page).toBe(page) + } finally { + initSpy.mockRestore() + } + }, + ) + test('re-reads from disk if in development mode and finds nothing', async () => { const [req, res] = makeRequestResponse('/en/never/heard/of') diff --git a/src/frame/tests/get-remote-json.ts b/src/frame/tests/get-remote-json.ts index 87e0e639ca0e..5796458a7427 100644 --- a/src/frame/tests/get-remote-json.ts +++ b/src/frame/tests/get-remote-json.ts @@ -5,7 +5,11 @@ import os from 'os' import { afterAll, afterEach, beforeAll, describe, expect, test } from 'vitest' import nock from 'nock' -import getRemoteJSON, { cache } from '@/frame/lib/get-remote-json' +import getRemoteJSON, { + cache, + parsedCache, + PARSED_CACHE_MAX_BYTES, +} from '@/frame/lib/get-remote-json' // Covers in-memory caching and disk-cache fallback after a memory miss. @@ -102,4 +106,38 @@ describe('getRemoteJSON', () => { }) await expect(getRemoteJSON(url, {})).rejects.toThrowError(/resulted in a non-JSON response/) }) + + test('reuses the parsed object while it stays in the parsed cache', async () => { + const url = 'http://example.com/parsed.json' + const { origin, pathname } = new URL(url) + nock(origin).get(pathname).reply(200, { a: '1' }) + const data = await getRemoteJSON(url, {}) + const data2 = await getRemoteJSON(url, {}) + expect(data2).toBe(data) + expect(Object.isFrozen(data)).toBe(true) + expect(parsedCache.has(url)).toBe(true) + }) + + test('evicts least recently used parsed objects past the byte limit', async () => { + // Each body is a bit over a third of the limit, so only two fit. + const big = 'x'.repeat(Math.floor(PARSED_CACHE_MAX_BYTES / 3)) + const urls = ['one', 'two', 'three'].map((name) => `http://example.com/lru-${name}.json`) + for (const url of urls) { + const { origin, pathname } = new URL(url) + nock(origin).get(pathname).reply(200, { big }) + } + await getRemoteJSON(urls[0], {}) + await getRemoteJSON(urls[1], {}) + // Touch the first URL so the second becomes least recently used. + await getRemoteJSON(urls[0], {}) + await getRemoteJSON(urls[2], {}) + expect(parsedCache.has(urls[0])).toBe(true) + expect(parsedCache.has(urls[1])).toBe(false) + expect(parsedCache.has(urls[2])).toBe(true) + + // Evicted entries still load from the deflated memory cache. + const data = (await getRemoteJSON(urls[1], {})) as Record + expect(data.big).toBe(big) + expect(parsedCache.has(urls[1])).toBe(true) + }) }) diff --git a/src/graphql/data/fpt/changelog.json b/src/graphql/data/fpt/changelog.json index 5f0cbabccd69..60c8057bd437 100644 --- a/src/graphql/data/fpt/changelog.json +++ b/src/graphql/data/fpt/changelog.json @@ -1,4 +1,17 @@ [ + { + "schemaChanges": [ + { + "title": "The GraphQL schema includes these changes:", + "changes": [ + "

Field fullDatabaseId was added to object type Repository

" + ] + } + ], + "previewChanges": [], + "upcomingChanges": [], + "date": "2026-10-08" + }, { "schemaChanges": [ { diff --git a/src/graphql/data/fpt/schema-pulls.json b/src/graphql/data/fpt/schema-pulls.json index 7a7990018d9c..932fb9a87c0e 100644 --- a/src/graphql/data/fpt/schema-pulls.json +++ b/src/graphql/data/fpt/schema-pulls.json @@ -178,7 +178,7 @@ "name": "archivePullRequest", "id": "archivepullrequest", "href": "/graphql/reference/pulls#mutation-archivepullrequest", - "description": "

Archive a pull request. Closes and marks the pull request as archived. Only repository admins can archive pull requests.

", + "description": "

Archive a pull request. Closes and marks the pull request as archived. Users\nwith the triage role or higher can archive pull requests.

", "isDeprecated": false, "inputFields": [ { @@ -959,7 +959,7 @@ "name": "unarchivePullRequest", "id": "unarchivepullrequest", "href": "/graphql/reference/pulls#mutation-unarchivepullrequest", - "description": "

Unarchive a pull request. Removes the archived flag from the pull request.\nDoes not automatically reopen or unlock the pull request. Only repository\nadmins can unarchive pull requests.

", + "description": "

Unarchive a pull request. Removes the archived flag from the pull request.\nDoes not automatically reopen or unlock the pull request. Users with the\ntriage role or higher can unarchive pull requests.

", "isDeprecated": false, "inputFields": [ { diff --git a/src/graphql/data/fpt/schema-repos.json b/src/graphql/data/fpt/schema-repos.json index b60e6a716522..32794b31426d 100644 --- a/src/graphql/data/fpt/schema-repos.json +++ b/src/graphql/data/fpt/schema-repos.json @@ -4056,6 +4056,13 @@ } ] }, + { + "name": "fullDatabaseId", + "description": "

Identifies the primary key from the database as a BigInt.

", + "type": "BigInt", + "id": "bigint", + "href": "/graphql/reference/other#scalar-bigint" + }, { "name": "fundingLinks", "description": "

The funding links for this repository.

", diff --git a/src/graphql/data/fpt/schema.docs.graphql b/src/graphql/data/fpt/schema.docs.graphql index b9b44ac81e75..c77c99f24a74 100644 --- a/src/graphql/data/fpt/schema.docs.graphql +++ b/src/graphql/data/fpt/schema.docs.graphql @@ -27670,7 +27670,8 @@ type Mutation @docsCategory(name: "meta") { ): ArchiveProjectV2ItemPayload @docsCategory(name: "projects") """ - Archive a pull request. Closes and marks the pull request as archived. Only repository admins can archive pull requests. + Archive a pull request. Closes and marks the pull request as archived. Users + with the triage role or higher can archive pull requests. """ archivePullRequest( """ @@ -29373,8 +29374,8 @@ type Mutation @docsCategory(name: "meta") { """ Unarchive a pull request. Removes the archived flag from the pull request. - Does not automatically reopen or unlock the pull request. Only repository - admins can unarchive pull requests. + Does not automatically reopen or unlock the pull request. Users with the + triage role or higher can unarchive pull requests. """ unarchivePullRequest( """ @@ -55145,6 +55146,11 @@ type Repository implements Node & visibility: RepositoryVisibility ): RepositoryConnection! + """ + Identifies the primary key from the database as a BigInt. + """ + fullDatabaseId: BigInt + """ The funding links for this repository """ diff --git a/src/graphql/data/ghec/schema-pulls.json b/src/graphql/data/ghec/schema-pulls.json index ee1fafe56ad7..9079f1eb9fd5 100644 --- a/src/graphql/data/ghec/schema-pulls.json +++ b/src/graphql/data/ghec/schema-pulls.json @@ -178,7 +178,7 @@ "name": "archivePullRequest", "id": "archivepullrequest", "href": "/graphql/reference/pulls#mutation-archivepullrequest", - "description": "

Archive a pull request. Closes and marks the pull request as archived. Only repository admins can archive pull requests.

", + "description": "

Archive a pull request. Closes and marks the pull request as archived. Users\nwith the triage role or higher can archive pull requests.

", "isDeprecated": false, "inputFields": [ { @@ -959,7 +959,7 @@ "name": "unarchivePullRequest", "id": "unarchivepullrequest", "href": "/graphql/reference/pulls#mutation-unarchivepullrequest", - "description": "

Unarchive a pull request. Removes the archived flag from the pull request.\nDoes not automatically reopen or unlock the pull request. Only repository\nadmins can unarchive pull requests.

", + "description": "

Unarchive a pull request. Removes the archived flag from the pull request.\nDoes not automatically reopen or unlock the pull request. Users with the\ntriage role or higher can unarchive pull requests.

", "isDeprecated": false, "inputFields": [ { diff --git a/src/graphql/data/ghec/schema-repos.json b/src/graphql/data/ghec/schema-repos.json index 3c331ae382a5..0ee8d0e55726 100644 --- a/src/graphql/data/ghec/schema-repos.json +++ b/src/graphql/data/ghec/schema-repos.json @@ -4056,6 +4056,13 @@ } ] }, + { + "name": "fullDatabaseId", + "description": "

Identifies the primary key from the database as a BigInt.

", + "type": "BigInt", + "id": "bigint", + "href": "/graphql/reference/other#scalar-bigint" + }, { "name": "fundingLinks", "description": "

The funding links for this repository.

", diff --git a/src/graphql/data/ghec/schema.docs.graphql b/src/graphql/data/ghec/schema.docs.graphql index b9b44ac81e75..c77c99f24a74 100644 --- a/src/graphql/data/ghec/schema.docs.graphql +++ b/src/graphql/data/ghec/schema.docs.graphql @@ -27670,7 +27670,8 @@ type Mutation @docsCategory(name: "meta") { ): ArchiveProjectV2ItemPayload @docsCategory(name: "projects") """ - Archive a pull request. Closes and marks the pull request as archived. Only repository admins can archive pull requests. + Archive a pull request. Closes and marks the pull request as archived. Users + with the triage role or higher can archive pull requests. """ archivePullRequest( """ @@ -29373,8 +29374,8 @@ type Mutation @docsCategory(name: "meta") { """ Unarchive a pull request. Removes the archived flag from the pull request. - Does not automatically reopen or unlock the pull request. Only repository - admins can unarchive pull requests. + Does not automatically reopen or unlock the pull request. Users with the + triage role or higher can unarchive pull requests. """ unarchivePullRequest( """ @@ -55145,6 +55146,11 @@ type Repository implements Node & visibility: RepositoryVisibility ): RepositoryConnection! + """ + Identifies the primary key from the database as a BigInt. + """ + fullDatabaseId: BigInt + """ The funding links for this repository """ diff --git a/src/languages/scripts/clone-translations.sh b/src/languages/scripts/clone-translations.sh index acb2625df99a..d111fa90c5a4 100755 --- a/src/languages/scripts/clone-translations.sh +++ b/src/languages/scripts/clone-translations.sh @@ -3,9 +3,18 @@ rm -rf translations mkdir translations +# Clone in parallel and fail if any clone fails. +pids=() for lang in "es-es" "ja-jp" "pt-br" "zh-cn" "ru-ru" "fr-fr" "ko-kr" "de-de" do - (cd translations && git clone --depth 1 "https://github.com/github/docs-internal.$lang.git" $lang) + git clone --quiet --depth 1 "https://github.com/github/docs-internal.$lang.git" "translations/$lang" & + pids+=($!) +done + +status=0 +for pid in "${pids[@]}"; do + wait "$pid" || status=1 done find . -name '.DS_Store' -type f -delete +exit $status diff --git a/src/redirects/tests/routing/developer-site-redirects.ts b/src/redirects/tests/routing/developer-site-redirects.ts index dd8d4cdb113b..43f11426455d 100644 --- a/src/redirects/tests/routing/developer-site-redirects.ts +++ b/src/redirects/tests/routing/developer-site-redirects.ts @@ -93,28 +93,31 @@ describe('developer redirects', () => { }) describe('fixtures', () => { - test.each(['developer', 'rest', 'graphql'])('%s redirects', async (label) => { - const FIXTURES = { - developer: './src/fixtures/fixtures/developer-redirects.json', - rest: './src/fixtures/fixtures/rest-redirects.json', - graphql: './src/fixtures/fixtures/graphql-redirects.json', - } - if (!(label in FIXTURES)) throw new Error('unrecognized label') - const fixtures = readJsonFile(FIXTURES[label as keyof typeof FIXTURES]) - // Avoid Promise.all here; event loop context switching makes it slower. - for (let [oldPath, newPath] of Object.entries(fixtures as Record)) { - // Versioned developer Enterprise paths support up to 2.21; versionless paths use latest. - newPath = (newPath as string).replace( + const FIXTURES = { + developer: './src/fixtures/fixtures/developer-redirects.json', + rest: './src/fixtures/fixtures/rest-redirects.json', + graphql: './src/fixtures/fixtures/graphql-redirects.json', + } + for (const [label, file] of Object.entries(FIXTURES)) { + const fixtures = readJsonFile(file) as Record + // Versioned developer Enterprise paths support up to 2.21; versionless paths use latest. + const cases = Object.entries(fixtures).map(([oldPath, newPath]) => [ + oldPath, + newPath.replace( '/enterprise-server/', `/enterprise-server@${enterpriseServerReleases.latest}/`, - ) - const res = await get(oldPath) - const sameFirstPrefix = oldPath.split('/')[1] === (newPath as string).split('/')[1] - expect(res.statusCode, `${oldPath} did not redirect to ${newPath}`).toBe( - sameFirstPrefix ? 301 : 302, - ) - expect(res.headers.location).toBe(newPath) - } - }) + ), + ]) + describe(`${label} redirects`, () => { + test.each(cases)('%s', async (oldPath, newPath) => { + const res = await get(oldPath) + const sameFirstPrefix = oldPath.split('/')[1] === newPath.split('/')[1] + expect(res.statusCode, `${oldPath} did not redirect to ${newPath}`).toBe( + sameFirstPrefix ? 301 : 302, + ) + expect(res.headers.location).toBe(newPath) + }) + }) + } }) }) diff --git a/src/secret-scanning/data/pattern-docs/fpt/public-docs.yml b/src/secret-scanning/data/pattern-docs/fpt/public-docs.yml index 67ddf33514dc..11a68d0a98cd 100644 --- a/src/secret-scanning/data/pattern-docs/fpt/public-docs.yml +++ b/src/secret-scanning/data/pattern-docs/fpt/public-docs.yml @@ -4257,7 +4257,7 @@ - provider: Salesforce supportedSecret: Salesforce Marketing Cloud API OAuth2 Token secretType: salesforce_marketing_cloud_api_oauth2_token - isPublic: false + isPublic: true isPrivateWithGhas: true hasPushProtection: true hasValidityCheck: false diff --git a/src/secret-scanning/data/pattern-docs/ghec/public-docs.yml b/src/secret-scanning/data/pattern-docs/ghec/public-docs.yml index 67ddf33514dc..11a68d0a98cd 100644 --- a/src/secret-scanning/data/pattern-docs/ghec/public-docs.yml +++ b/src/secret-scanning/data/pattern-docs/ghec/public-docs.yml @@ -4257,7 +4257,7 @@ - provider: Salesforce supportedSecret: Salesforce Marketing Cloud API OAuth2 Token secretType: salesforce_marketing_cloud_api_oauth2_token - isPublic: false + isPublic: true isPrivateWithGhas: true hasPushProtection: true hasValidityCheck: false diff --git a/src/tests/README.md b/src/tests/README.md index 8bb6eb851930..0b2e2a1d9307 100644 --- a/src/tests/README.md +++ b/src/tests/README.md @@ -142,6 +142,13 @@ Tests that make HTTP requests to `localhost:4000`: - Vitest automatically starts/stops server via hooks - Disable with `START_VITEST_SERVER=false` for manual server control +Unit tests that don't make HTTP requests can skip the server. This avoids the build, Elasticsearch, and translations: +```bash +START_VITEST_SERVER=false npm test -- src/languages/tests/correct-translation-content.ts +``` + +In CI, `.github/workflows/test.yml` sets `START_VITEST_SERVER=false` and skips the build for suites with no HTTP tests. If you add an HTTP test to one of those suites, remove the suite from that list. + Manual server for debugging: ```bash # Terminal 1 diff --git a/src/tests/vitest.setup.ts b/src/tests/vitest.setup.ts index 329e2a6a7762..0cb7c786141d 100644 --- a/src/tests/vitest.setup.ts +++ b/src/tests/vitest.setup.ts @@ -1,14 +1,16 @@ -import { main } from '@/frame/start-server' import { overrideEnvForTesting } from './mocks/start-mock-server' let teardownHappened = false -type PromiseType> = T extends Promise ? U : never -type Server = PromiseType> +type Server = Awaited> let server: Server | undefined export async function setup() { overrideEnvForTesting() + // Unit tests and manually started servers don't need the Vitest-managed server. + if (process.env.START_VITEST_SERVER === 'false') return + // Import lazily so skipping the server also skips loading the app. + const { main } = await import('@/frame/start-server') server = await main() } diff --git a/src/workflows/purge-fastly-changed-content.ts b/src/workflows/purge-fastly-changed-content.ts index c2c0ec6ab1b2..98d60b5f78e8 100644 --- a/src/workflows/purge-fastly-changed-content.ts +++ b/src/workflows/purge-fastly-changed-content.ts @@ -30,14 +30,13 @@ const PURGE_MAX_RATE_LIMIT_RETRIES = 5 // every POP at the same instant, so a request arriving in between can repopulate // an already-purged edge node from the not-yet-purged shield, leaving the edge // holding pre-deploy content again. The second pass evicts that copy. -// https://www.fastly.com/documentation/guides/concepts/cache/purging#race-conditions +// https://www.fastly.com/documentation/guides/full-site-delivery/purging/purging-a-url/ const PURGE_PASSES = 2 // The second pass waits long enough for repopulated edge copies to exist. // Otherwise the second purge runs too early, and re-population happens after it. -// purge-fastly.ts uses the same 20s because Fastly's suggested 2s has been too -// short in practice. This does not stagger keys within a pass because spacing -// protects the backend during whole-language purges, and this only purges changed pages. +// Fastly's suggested 2s has been too short in practice. +// Manual purges in purge-fastly.ts share this timing. const DELAY_BEFORE_SECOND_PURGE = 20 * 1000 // Jitter ceiling in ms keeps retries with the same reset timestamp from re-bursting. @@ -196,16 +195,15 @@ export function chunk(items: T[], size: number): T[][] { return batches } -// Hard-purge one batch of at most 256 surrogate keys. Fastly's batch endpoint is -// service-scoped; omitting the soft-purge header makes it a hard purge, so every -// object tagged with any listed key is evicted and the next request is a fresh -// miss. Retries on HTTP 429, honoring Fastly's rate-limit hint. +// Soft purges can fail to clear content when origin returns 304 Not Modified, +// because a 304 extends the stale object. // https://www.fastly.com/documentation/reference/api/purging/ -async function hardPurgeKeyBatch( +async function purgeKeyBatch( keys: string[], fastlyToken: string, serviceId: string, - rateLimitDelayFn: (response: Response, attempt: number) => number = rateLimitDelayMs, + soft: boolean, + rateLimitDelayFn: (response: Response, attempt: number) => number, ): Promise { for (let attempt = 0; ; attempt++) { const response = await fetchWithRetry( @@ -216,6 +214,7 @@ async function hardPurgeKeyBatch( 'fastly-key': fastlyToken, accept: 'application/json', 'content-type': 'application/json', + ...(soft ? { 'fastly-soft-purge': '1' } : {}), }, body: JSON.stringify({ surrogate_keys: keys }), }, @@ -249,20 +248,29 @@ async function hardPurgeKeyBatch( } } -// Hard-purge every key in batches of at most 256, one batch at a time, then do -// it all again after a delay to clear anything the origin shield repopulated. -// Collects failures so one bad batch doesn't drop the rest, then -// throws at the end if any failed so the workflow's failure alerting fires. -export async function hardPurgeSurrogateKeys( +type PurgeSurrogateKeysOptions = { + soft?: boolean + rateLimitDelayFn?: (response: Response, attempt: number) => number + sleepFn?: (ms: number) => Promise +} + +// Purge every batch twice, so the second pass clears edge copies refilled from the shield. +// Collect failures so one bad batch doesn't skip the rest, +// then throw so the workflow's failure alerting fires. +export async function purgeSurrogateKeys( keys: string[], fastlyToken: string, serviceId: string, - rateLimitDelayFn: (response: Response, attempt: number) => number = rateLimitDelayMs, - sleepFn: (ms: number) => Promise = sleep, + { + soft = false, + rateLimitDelayFn = rateLimitDelayMs, + sleepFn = sleep, + }: PurgeSurrogateKeysOptions = {}, ): Promise { const batches = chunk(keys, MAX_KEYS_PER_PURGE) const errors: Error[] = [] let attempts = 0 + const mode = soft ? 'Soft' : 'Hard' const purgeAllBatches = async (pass: number): Promise => { for (const [index, batch] of batches.entries()) { @@ -271,9 +279,9 @@ export async function hardPurgeSurrogateKeys( `(${batch.length} key(s))` attempts++ try { - console.log(`Hard-purging ${label}...`) - await hardPurgeKeyBatch(batch, fastlyToken, serviceId, rateLimitDelayFn) - console.log(`Hard-purged ${label}.`) + console.log(`${mode}-purging ${label}...`) + await purgeKeyBatch(batch, fastlyToken, serviceId, soft, rateLimitDelayFn) + console.log(`${mode}-purged ${label}.`) } catch (error) { console.error(error) errors.push(error instanceof Error ? error : new Error(String(error))) @@ -343,7 +351,7 @@ async function main() { } console.log(`Hard-purging ${keys.length} page key(s)...`) - await hardPurgeSurrogateKeys(keys, FASTLY_TOKEN, FASTLY_SERVICE_ID) + await purgeSurrogateKeys(keys, FASTLY_TOKEN, FASTLY_SERVICE_ID) console.log(`Hard-purged ${keys.length} page key(s).`) } diff --git a/src/workflows/purge-fastly.ts b/src/workflows/purge-fastly.ts index e70f1dc5b51d..f7dacf2b5f29 100644 --- a/src/workflows/purge-fastly.ts +++ b/src/workflows/purge-fastly.ts @@ -3,47 +3,37 @@ import { program } from 'commander' import { fetchWithRetry } from '@/frame/lib/fetch-utils' import { languageKeys } from '@/languages/lib/languages-server' import { makeLanguageSurrogateKey } from '@/frame/middleware/set-fastly-surrogate-key' +import { purgeSurrogateKeys } from '@/workflows/purge-fastly-changed-content' -// Purges Fastly by mode: entire cache, one surrogate key, or no-language plus -// every language key. --hard forces hard purges for targeted modes, and -// --everything always hard-purges. +// Purge hard by default, because a soft purge keeps serving the old copy during refetch. const { FASTLY_TOKEN, FASTLY_SERVICE_ID } = process.env -const DELAY_BETWEEN_KEYS = 10 * 1000 -const DELAY_BEFORE_SECOND_PURGE = 20 * 1000 - -// The pipelining in purgeKeys only lines up if the second-purge delay is a whole -// number of key slots; otherwise second purges would drift off the cadence. -// Enforce it so a future tweak to either constant can't silently break it. -if (DELAY_BEFORE_SECOND_PURGE % DELAY_BETWEEN_KEYS !== 0) { - throw new Error( - `DELAY_BEFORE_SECOND_PURGE (${DELAY_BEFORE_SECOND_PURGE}ms) must be a multiple of ` + - `DELAY_BETWEEN_KEYS (${DELAY_BETWEEN_KEYS}ms) to keep second purges ` + - `aligned with later first-purge slots`, - ) -} +// Fastly suggests two purge_all calls about 30 seconds apart, +// so the second clears edge copies refilled from a not-yet-purged shield. +// https://www.fastly.com/documentation/guides/full-site-delivery/purging/purging-all-content/#dealing-with-race-conditions +const DELAY_BEFORE_SECOND_PURGE_ALL = 30 * 1000 const sleep = (ms: number): Promise => new Promise((resolve) => setTimeout(resolve, ms)) program .description( - 'Purges Fastly after a deploy and on demand. Soft purge by default; can hard ' + - 'purge specific languages, or hard purge the entire cache.', + 'Purges Fastly on demand. Hard purge by default; can soft purge specific ' + + 'languages, or hard purge the entire cache.', ) .option( '--languages ', "Comma separated languages to purge, e.g. 'en,es,ja'. Blank/omitted = all languages.", ) .option('--surrogate-key ', 'Purge a single explicit surrogate key. e.g. api-search:en') - .option('--hard', 'Evict immediately instead of the default soft purge') - .option('--everything', 'Hard purge the ENTIRE cache: every key. Ignores --languages/--hard.') + .option('--soft', 'Mark stale instead of the default hard purge, which evicts immediately') + .option('--everything', 'Hard purge the ENTIRE cache: every key. Ignores --languages/--soft.') .parse(process.argv) type Options = { languages?: string surrogateKey?: string - hard?: boolean + soft?: boolean everything?: boolean } @@ -62,17 +52,34 @@ async function main(options: Options) { throw new Error('FASTLY_SERVICE_ID not detected; refusing to purge') } if (options.everything) { - console.log('Attempting hard purge of the entire cache...') - const result = await fastlyPurge('purge_all') - console.log('Fastly purge_all result:', result.status) + // Run the second call even if the first fails, because Fastly may have applied the first. + const errors: unknown[] = [] + for (let pass = 1; pass <= 2; pass++) { + if (pass > 1) { + console.log(`Waiting ${DELAY_BEFORE_SECOND_PURGE_ALL}ms before the second purge_all...`) + await sleep(DELAY_BEFORE_SECOND_PURGE_ALL) + } + try { + console.log(`Hard-purging the entire cache, pass ${pass}/2...`) + const result = await purgeAll() + console.log(`Fastly purge_all pass ${pass}/2 result:`, result.status) + } catch (error) { + console.error(error) + errors.push(error) + } + } + if (errors.length) { + throw new Error(`${errors.length} of 2 purge_all call(s) failed`) + } return } - const soft = !options.hard const surrogateKeys = options.surrogateKey ? [options.surrogateKey] : languageSurrogateKeys(options.languages) - await purgeKeys(surrogateKeys, soft) + await purgeSurrogateKeys(surrogateKeys, FASTLY_TOKEN, FASTLY_SERVICE_ID, { + soft: Boolean(options.soft), + }) } function languageSurrogateKeys(languagesInput?: string): string[] { @@ -102,71 +109,16 @@ function languagesFromString(str: string): string[] { return parsedLanguages } -type PurgePhase = 'first' | 'second' -type PurgeOutcome = { key: string; phase: PurgePhase; error?: unknown } - -// purgeKeys double-purges surrogate keys to clear Fastly edge nodes first and the -// origin shield after stale content can be re-fetched. DELAY_BETWEEN_KEYS spaces -// first purges to avoid a backend traffic spike. DELAY_BEFORE_SECOND_PURGE must -// remain a multiple of that delay so second purges share later first-purge slots. -// A single-key purge runs at 0s and 20s. Fastly's 30s figure applies to -// purge_all, not these targeted purges. -// https://www.fastly.com/documentation/guides/concepts/cache/purging#race-conditions -async function purgeKeys(surrogateKeys: string[], soft: boolean) { - // One wall-clock start time keeps network latency from drifting the purge cadence. - const startTime = Date.now() - const purges: Promise[] = [] - - // Each call resolves to an outcome so later scheduled purges can still finish. - async function runPurge( - key: string, - phase: PurgePhase, - targetTime: number, - ): Promise { - await sleep(Math.max(0, targetTime - Date.now())) - try { - console.log(`Triggering ${phase}-phase ${soft ? 'soft' : 'hard'} purge for '${key}'...`) - const result = await fastlyPurge(`purge/${encodeURIComponent(key)}`, { soft }) - console.log(`Fastly purge result for '${key}':`, result.status) - return { key, phase } - } catch (error) { - return { key, phase, error } - } - } - - for (const [index, key] of surrogateKeys.entries()) { - const slotStart = startTime + index * DELAY_BETWEEN_KEYS - purges.push(runPurge(key, 'first', slotStart)) - purges.push(runPurge(key, 'second', slotStart + DELAY_BEFORE_SECOND_PURGE)) - } - - const outcomes = await Promise.all(purges) - const failures = outcomes.filter((outcome) => outcome.error) - if (failures.length) { - for (const failure of failures) { - console.error(`Fastly ${failure.phase} purge failed for '${failure.key}':`, failure.error) - } - throw new Error(`${failures.length} Fastly purge(s) failed`) - } -} - -// fastlyPurge appends endpoint to the service path, such as purge/ or -// purge_all. Non-2xx responses throw with the body best-effort because Fastly -// puts permission and feature details there. Soft purge marks the object stale -// and serves stale-while-revalidate; hard purge evicts it outright. Soft can -// fail to clear content whose origin returns 304 Not Modified on revalidation, -// since a 304 extends the stale object. purge_all ignores the soft header. -async function fastlyPurge(endpoint: string, { soft = false }: { soft?: boolean } = {}) { +// purge_all ignores the soft-purge header, so this always hard-purges. +// Errors include the response body, because Fastly puts permission and feature details there. +async function purgeAll() { const headers: Record = { 'fastly-key': FASTLY_TOKEN as string, accept: 'application/json', 'Content-Type': 'application/json', } - if (soft) { - headers['fastly-soft-purge'] = '1' - } - const url = `https://api.fastly.com/service/${encodeURIComponent(FASTLY_SERVICE_ID as string)}/${endpoint}` + const url = `https://api.fastly.com/service/${encodeURIComponent(FASTLY_SERVICE_ID as string)}/purge_all` const response = await fetchWithRetry( url, { method: 'POST', headers }, diff --git a/src/workflows/tests/purge-fastly-changed-content.ts b/src/workflows/tests/purge-fastly-changed-content.ts index 7ea7c244a6cb..91b26b1ca876 100644 --- a/src/workflows/tests/purge-fastly-changed-content.ts +++ b/src/workflows/tests/purge-fastly-changed-content.ts @@ -11,7 +11,7 @@ const { getChangedContentFiles, contentFilesToPageKeys, chunk, - hardPurgeSurrogateKeys, + purgeSurrogateKeys, rateLimitDelayMs, } = await import('../purge-fastly-changed-content') @@ -153,7 +153,7 @@ describe('chunk', () => { }) }) -describe('hardPurgeSurrogateKeys', () => { +describe('purgeSurrogateKeys', () => { // Tests skip the 20-second between-pass delay. const noSleep = async () => {} @@ -174,12 +174,11 @@ describe('hardPurgeSurrogateKeys', () => { test('sends one hard batch purge per pass with a surrogate_keys body (no soft header)', async () => { fetchWithRetry.mockResolvedValue({ ok: true }) - await hardPurgeSurrogateKeys( + await purgeSurrogateKeys( ['language:en,path:a.md', 'language:en,path:b.md'], 'token-123', 'svc-1', - undefined, - noSleep, + { sleepFn: noSleep }, ) expect(fetchWithRetry).toHaveBeenCalledTimes(2) const [url, init] = fetchWithRetry.mock.calls[0] @@ -193,25 +192,30 @@ describe('hardPurgeSurrogateKeys', () => { expect(fetchWithRetry.mock.calls[1][1].body).toBe(init.body) }) + test('sends the soft-purge header when soft is set', async () => { + fetchWithRetry.mockResolvedValue({ ok: true }) + await purgeSurrogateKeys(['language:en'], 'tok', 'svc', { soft: true, sleepFn: noSleep }) + expect(fetchWithRetry).toHaveBeenCalledTimes(2) + for (const [, init] of fetchWithRetry.mock.calls) { + expect(init.headers['fastly-soft-purge']).toBe('1') + } + }) + test('waits between the two passes to let the shield re-populate first', async () => { fetchWithRetry.mockResolvedValue({ ok: true }) const waits: number[] = [] - await hardPurgeSurrogateKeys( - ['language:en,path:a.md'], - 'tok', - 'svc', - undefined, - async (ms: number) => { + await purgeSurrogateKeys(['language:en,path:a.md'], 'tok', 'svc', { + sleepFn: async (ms: number) => { waits.push(ms) }, - ) + }) expect(waits).toEqual([20_000]) }) test('splits more than 256 keys into multiple batches, per pass', async () => { fetchWithRetry.mockResolvedValue({ ok: true }) const keys = Array.from({ length: 257 }, (_unused, i) => `language:en,path:p${i}.md`) - await hardPurgeSurrogateKeys(keys, 'tok', 'svc', undefined, noSleep) + await purgeSurrogateKeys(keys, 'tok', 'svc', { sleepFn: noSleep }) // 2 batches x 2 passes. expect(fetchWithRetry).toHaveBeenCalledTimes(4) expect(JSON.parse(fetchWithRetry.mock.calls[0][1].body).surrogate_keys).toHaveLength(256) @@ -229,7 +233,7 @@ describe('hardPurgeSurrogateKeys', () => { }) fetchWithRetry.mockResolvedValue({ ok: true }) const keys = Array.from({ length: 300 }, (_unused, i) => `language:en,path:p${i}.md`) - await expect(hardPurgeSurrogateKeys(keys, 'tok', 'svc', undefined, noSleep)).rejects.toThrow( + await expect(purgeSurrogateKeys(keys, 'tok', 'svc', { sleepFn: noSleep })).rejects.toThrow( /1 of 4 batch purge\(s\) failed/, ) expect(fetchWithRetry).toHaveBeenCalledTimes(4) @@ -245,7 +249,7 @@ describe('hardPurgeSurrogateKeys', () => { }) .mockResolvedValue({ ok: true }) await expect( - hardPurgeSurrogateKeys(['language:en,path:a.md'], 'tok', 'svc', undefined, noSleep), + purgeSurrogateKeys(['language:en,path:a.md'], 'tok', 'svc', { sleepFn: noSleep }), ).rejects.toThrow(/1 of 2 batch purge\(s\) failed/) expect(fetchWithRetry).toHaveBeenCalledTimes(2) }) @@ -254,7 +258,10 @@ describe('hardPurgeSurrogateKeys', () => { fetchWithRetry .mockResolvedValueOnce(fakeResponse(429, { headers: { 'retry-after': '0' } })) .mockResolvedValue(fakeResponse(200, { ok: true })) - await hardPurgeSurrogateKeys(['language:en,path:a.md'], 'tok', 'svc', () => 0, noSleep) + await purgeSurrogateKeys(['language:en,path:a.md'], 'tok', 'svc', { + rateLimitDelayFn: () => 0, + sleepFn: noSleep, + }) // The first pass gets a 429 and retries once; the second pass makes one call. expect(fetchWithRetry).toHaveBeenCalledTimes(3) }) @@ -262,7 +269,10 @@ describe('hardPurgeSurrogateKeys', () => { test('gives up after the retry budget and reports the batch as failed', async () => { fetchWithRetry.mockResolvedValue(fakeResponse(429, { headers: { 'retry-after': '0' } })) await expect( - hardPurgeSurrogateKeys(['language:en,path:a.md'], 'tok', 'svc', () => 0, noSleep), + purgeSurrogateKeys(['language:en,path:a.md'], 'tok', 'svc', { + rateLimitDelayFn: () => 0, + sleepFn: noSleep, + }), ).rejects.toThrow(/2 of 2 batch purge\(s\) failed/) // Initial attempt plus 5 retries, times 2 passes. expect(fetchWithRetry).toHaveBeenCalledTimes(12)