From 6dbbd429f4f8fa8dd7248fdc16aaf27c84ef441c Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 13 Oct 2026 00:25:36 +0400 Subject: [PATCH 1/5] Move the Cloud Run entrypoint deploy logic into tested scripts deploy_entrypoint now captures the stable service state and resolves the candidate metadata with cloud-run-simulation-entry-stable-state.sh and cloud-run-simulation-entry-candidate.sh. Their bodies are the former inline shell unchanged, except that the service name comes from the step's env: instead of an interpolated expression; the $GITHUB_OUTPUT names and values, step ids and job outputs are unchanged. Tests run them against fake gcloud output with the real jq. Building and pushing the image become two single-line steps. Co-Authored-By: Claude Opus 5.5 --- .../cloud-run-simulation-entry-candidate.sh | 43 +++ ...cloud-run-simulation-entry-stable-state.sh | 31 ++ .../workflows/simulation-deploy.reusable.yml | 71 +---- docs/generated/rules.md | 2 +- .../tests/test_deploy_workflow_scripts.py | 277 ++++++++++++++++++ rules/baselines/ci-multiline-run.txt | 3 - rules/registry.toml | 2 +- 7 files changed, 363 insertions(+), 66 deletions(-) create mode 100755 .github/scripts/cloud-run-simulation-entry-candidate.sh create mode 100755 .github/scripts/cloud-run-simulation-entry-stable-state.sh create mode 100644 projects/policyengine-simulation-entry/tests/test_deploy_workflow_scripts.py diff --git a/.github/scripts/cloud-run-simulation-entry-candidate.sh b/.github/scripts/cloud-run-simulation-entry-candidate.sh new file mode 100755 index 000000000..52cf435c4 --- /dev/null +++ b/.github/scripts/cloud-run-simulation-entry-candidate.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash + +# Resolve the URL, revision and image digest of the no-traffic candidate +# revision that carries the given Cloud Run tag. +# +# Required env vars: ENTRYPOINT_SERVICE, TAG, PROJECT_ID, REGION, GITHUB_OUTPUT. +# Outputs: url, revision and image in GITHUB_OUTPUT. Fails when the tag has no +# URL or revision, or the revision's image is not pinned by digest. + +set -euo pipefail + +service_json="$(gcloud run services describe "${ENTRYPOINT_SERVICE}" \ + --project "${PROJECT_ID}" \ + --region "${REGION}" \ + --format=json)" +url="$(jq -er --arg tag "${TAG}" ' + .status.traffic[] + | select(.tag == $tag) + | .url + | select(type == "string" and length > 0) +' <<<"${service_json}")" +revision="$(jq -er --arg tag "${TAG}" ' + .status.traffic[] + | select(.tag == $tag) + | .revisionName + | select(type == "string" and length > 0) +' <<<"${service_json}")" +revision_json="$(gcloud run revisions describe "${revision}" \ + --project "${PROJECT_ID}" \ + --region "${REGION}" \ + --format=json)" +image="$(jq -er ' + (.status.imageDigest // .spec.containers[0].image) + | select( + type == "string" + and contains("@sha256:") + ) +' <<<"${revision_json}")" +{ + echo "url=${url}" + echo "revision=${revision}" + echo "image=${image}" +} >> "$GITHUB_OUTPUT" diff --git a/.github/scripts/cloud-run-simulation-entry-stable-state.sh b/.github/scripts/cloud-run-simulation-entry-stable-state.sh new file mode 100755 index 000000000..a225bdb69 --- /dev/null +++ b/.github/scripts/cloud-run-simulation-entry-stable-state.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash + +# Record the stable Cloud Run service URL and the one revision serving 100 +# percent of its traffic, before a candidate is deployed. +# +# Required env vars: ENTRYPOINT_SERVICE, PROJECT_ID, REGION, GITHUB_OUTPUT. +# Outputs: stable_url and revision in GITHUB_OUTPUT. Fails when the service has +# no URL or not exactly one revision at 100 percent. + +set -euo pipefail + +service_json="$(gcloud run services describe "${ENTRYPOINT_SERVICE}" \ + --project "${PROJECT_ID}" \ + --region "${REGION}" \ + --format=json)" +stable_url="$(jq -er '.status.url | select(type == "string" and length > 0)' <<<"${service_json}")" +revision="$(jq -er ' + [ + .status.traffic[]? + | select((.percent // 0) == 100) + | .revisionName + | select(type == "string" and length > 0) + ] + | if length == 1 then .[0] + else error("service must have exactly one revision at 100 percent") + end +' <<<"${service_json}")" +{ + echo "stable_url=${stable_url}" + echo "revision=${revision}" +} >> "$GITHUB_OUTPUT" diff --git a/.github/workflows/simulation-deploy.reusable.yml b/.github/workflows/simulation-deploy.reusable.yml index ac6d6e40a..e95ae2744 100644 --- a/.github/workflows/simulation-deploy.reusable.yml +++ b/.github/workflows/simulation-deploy.reusable.yml @@ -199,39 +199,19 @@ jobs: - name: Capture stable service state before deployment id: previous - run: | - service_json="$(gcloud run services describe "${{ inputs.entrypoint_service }}" \ - --project "${PROJECT_ID}" \ - --region "${REGION}" \ - --format=json)" - stable_url="$(jq -er '.status.url | select(type == "string" and length > 0)' <<<"${service_json}")" - revision="$(jq -er ' - [ - .status.traffic[]? - | select((.percent // 0) == 100) - | .revisionName - | select(type == "string" and length > 0) - ] - | if length == 1 then .[0] - else error("service must have exactly one revision at 100 percent") - end - ' <<<"${service_json}")" - { - echo "stable_url=${stable_url}" - echo "revision=${revision}" - } >> "$GITHUB_OUTPUT" + env: + ENTRYPOINT_SERVICE: ${{ inputs.entrypoint_service }} + run: .github/scripts/cloud-run-simulation-entry-stable-state.sh - name: Select immutable image id: image run: echo "uri=${REGION}-docker.pkg.dev/${PROJECT_ID}/${REPOSITORY}/${IMAGE_NAME}:${GITHUB_SHA}" >> "$GITHUB_OUTPUT" - - name: Build and push entrypoint image - run: | - docker build \ - --file "${PROJECT_DIR}/Dockerfile" \ - --tag "${{ steps.image.outputs.uri }}" \ - . - docker push "${{ steps.image.outputs.uri }}" + - name: Build entrypoint image + run: docker build --file "${PROJECT_DIR}/Dockerfile" --tag "${{ steps.image.outputs.uri }}" . + + - name: Push entrypoint image + run: docker push "${{ steps.image.outputs.uri }}" - name: Select no-traffic revision tag id: metadata @@ -269,40 +249,9 @@ jobs: - name: Resolve exact candidate metadata id: candidate env: + ENTRYPOINT_SERVICE: ${{ inputs.entrypoint_service }} TAG: ${{ steps.metadata.outputs.tag }} - run: | - service_json="$(gcloud run services describe "${{ inputs.entrypoint_service }}" \ - --project "${PROJECT_ID}" \ - --region "${REGION}" \ - --format=json)" - url="$(jq -er --arg tag "${TAG}" ' - .status.traffic[] - | select(.tag == $tag) - | .url - | select(type == "string" and length > 0) - ' <<<"${service_json}")" - revision="$(jq -er --arg tag "${TAG}" ' - .status.traffic[] - | select(.tag == $tag) - | .revisionName - | select(type == "string" and length > 0) - ' <<<"${service_json}")" - revision_json="$(gcloud run revisions describe "${revision}" \ - --project "${PROJECT_ID}" \ - --region "${REGION}" \ - --format=json)" - image="$(jq -er ' - (.status.imageDigest // .spec.containers[0].image) - | select( - type == "string" - and contains("@sha256:") - ) - ' <<<"${revision_json}")" - { - echo "url=${url}" - echo "revision=${revision}" - echo "image=${image}" - } >> "$GITHUB_OUTPUT" + run: .github/scripts/cloud-run-simulation-entry-candidate.sh - name: Install entrypoint unit-test dependencies working-directory: ${{ env.PROJECT_DIR }} diff --git a/docs/generated/rules.md b/docs/generated/rules.md index d6913f2df..38498f94c 100644 --- a/docs/generated/rules.md +++ b/docs/generated/rules.md @@ -60,7 +60,7 @@ Every rule that governs `policyengine-sim-api` is listed here with how it is enf ### Ratcheted (existing violations still tolerated) -- `sim.ci.multiline-run-is-script`: 13 remaining in `rules/baselines/ci-multiline-run.txt` +- `sim.ci.multiline-run-is-script`: 10 remaining in `rules/baselines/ci-multiline-run.txt` ### Exceptions and review dates diff --git a/projects/policyengine-simulation-entry/tests/test_deploy_workflow_scripts.py b/projects/policyengine-simulation-entry/tests/test_deploy_workflow_scripts.py new file mode 100644 index 000000000..2a027da03 --- /dev/null +++ b/projects/policyengine-simulation-entry/tests/test_deploy_workflow_scripts.py @@ -0,0 +1,277 @@ +"""Tests for the scripts the simulation deployment workflows run. + +Each test runs a script with fake gcloud or uv commands first on PATH (jq is +real) and asserts on the exact calls, $GITHUB_OUTPUT and exit status, so the +production steps keep behaving as the inline shell they replaced. +""" + +from __future__ import annotations + +import json +import re +from pathlib import Path + +import pytest + +from workflow_script_support import ( + WORKFLOWS_DIR, + FakeCommands, + run_script, +) + +REUSABLE_DEPLOY_WORKFLOW = WORKFLOWS_DIR / "simulation-deploy.reusable.yml" + + +@pytest.fixture +def fakes(tmp_path: Path) -> FakeCommands: + return FakeCommands.create(tmp_path) + + +def _step(workflow: str, name: str) -> str: + """The text of one workflow step, from its ``- name:`` to the next step.""" + match = re.search( + rf"^(?P +)- name: {re.escape(name)}\n(?P.*?)(?=^(?P=indent)- |^ \S|\Z)", + workflow, + flags=re.MULTILINE | re.DOTALL, + ) + assert match is not None, name + return match.group("body") + + +def _output(path: Path) -> str: + return path.read_text(encoding="utf-8") if path.exists() else "" + + +# --- deploy_entrypoint: Cloud Run state ----------------------------------- + +STABLE_STATE = "cloud-run-simulation-entry-stable-state.sh" +CANDIDATE = "cloud-run-simulation-entry-candidate.sh" +SERVICE = "policyengine-simulation-entry-staging" +PROJECT = "policyengine-api" +REGION = "us-central1" +STABLE_URL = "https://policyengine-simulation-entry-staging-abc-uc.a.run.app" +CANDIDATE_URL = "https://s-42---policyengine-simulation-entry-staging-abc-uc.a.run.app" +DIGEST_IMAGE = ( + "us-central1-docker.pkg.dev/policyengine-api/policyengine-simulation-entry/" + "policyengine-simulation-entry@sha256:" + "a" * 64 +) + + +FAKE_GCLOUD = """\ +case "$1 $2 $3" in + "run services describe") printf '%s' "$FAKE_SERVICE_JSON" ;; + "run revisions describe") printf '%s' "$FAKE_REVISION_JSON" ;; + *) exit 2 ;; +esac +exit "${FAKE_GCLOUD_EXIT:-0}" +""" + + +def _service(traffic: list[dict], url: str | None = STABLE_URL) -> str: + status: dict = {"traffic": traffic} + if url is not None: + status["url"] = url + return json.dumps({"status": status}) + + +def _run_cloud_run_script( + fakes: FakeCommands, + tmp_path: Path, + script: str, + service_json: str, + revision_json: str = "{}", + **env: str, +): + output = tmp_path / "github-output" + fakes.add("gcloud", FAKE_GCLOUD) + result = run_script( + script, + env=fakes.env( + **{ + "ENTRYPOINT_SERVICE": SERVICE, + "PROJECT_ID": PROJECT, + "REGION": REGION, + "TAG": "s-42", + "GITHUB_OUTPUT": str(output), + "FAKE_SERVICE_JSON": service_json, + "FAKE_REVISION_JSON": revision_json, + **env, + } + ), + ) + return result, _output(output) + + +def _describe_service() -> list[str]: + return [ + "gcloud", + "run", + "services", + "describe", + SERVICE, + "--project", + PROJECT, + "--region", + REGION, + "--format=json", + ] + + +def test_deploy_workflow_runs_the_cloud_run_state_scripts(): + workflow = REUSABLE_DEPLOY_WORKFLOW.read_text(encoding="utf-8") + + previous = _step(workflow, "Capture stable service state before deployment") + assert "id: previous" in previous + assert "ENTRYPOINT_SERVICE: ${{ inputs.entrypoint_service }}" in previous + assert f"run: .github/scripts/{STABLE_STATE}\n" in previous + + candidate = _step(workflow, "Resolve exact candidate metadata") + assert "id: candidate" in candidate + assert "ENTRYPOINT_SERVICE: ${{ inputs.entrypoint_service }}" in candidate + assert "TAG: ${{ steps.metadata.outputs.tag }}" in candidate + assert f"run: .github/scripts/{CANDIDATE}\n" in candidate + + # The job still reads every output name the scripts write. + for output in ( + "image: ${{ steps.candidate.outputs.image }}", + "revision: ${{ steps.candidate.outputs.revision }}", + "candidate_url: ${{ steps.candidate.outputs.url }}", + "stable_url: ${{ steps.previous.outputs.stable_url }}", + "previous_revision: ${{ steps.previous.outputs.revision }}", + ): + assert output in workflow + + +def test_stable_state_records_the_url_and_the_fully_serving_revision(fakes, tmp_path): + service_json = _service( + [ + {"revisionName": "entry-00001-old", "percent": 100}, + {"revisionName": "entry-00002-new", "tag": "s-41"}, + ] + ) + + result, output = _run_cloud_run_script(fakes, tmp_path, STABLE_STATE, service_json) + + assert result.returncode == 0, result.stderr + assert output == f"stable_url={STABLE_URL}\nrevision=entry-00001-old\n" + assert fakes.calls() == [_describe_service()] + + +@pytest.mark.parametrize( + "traffic", + [ + [ + {"revisionName": "entry-00001-one", "percent": 50}, + {"revisionName": "entry-00002-two", "percent": 50}, + ], + [ + {"revisionName": "entry-00001-one", "percent": 100}, + {"revisionName": "entry-00002-two", "percent": 100}, + ], + [], + ], + ids=["split", "two-at-100", "no-traffic"], +) +def test_stable_state_requires_exactly_one_revision_at_100_percent( + fakes, tmp_path, traffic +): + result, output = _run_cloud_run_script( + fakes, tmp_path, STABLE_STATE, _service(traffic) + ) + + assert result.returncode != 0 + assert "service must have exactly one revision at 100 percent" in result.stderr + assert output == "" + + +def test_stable_state_requires_a_service_url(fakes, tmp_path): + service_json = _service( + [{"revisionName": "entry-00001-old", "percent": 100}], url=None + ) + + result, output = _run_cloud_run_script(fakes, tmp_path, STABLE_STATE, service_json) + + assert result.returncode != 0 + assert output == "" + + +def test_stable_state_fails_when_gcloud_fails(fakes, tmp_path): + result, output = _run_cloud_run_script( + fakes, tmp_path, STABLE_STATE, "{}", FAKE_GCLOUD_EXIT="1" + ) + + assert result.returncode != 0 + assert output == "" + + +CANDIDATE_TRAFFIC = [ + {"revisionName": "entry-00001-old", "percent": 100}, + {"revisionName": "entry-00002-new", "tag": "s-42", "url": CANDIDATE_URL}, +] + + +@pytest.mark.parametrize( + "revision_json", + [ + {"status": {"imageDigest": DIGEST_IMAGE}}, + {"spec": {"containers": [{"image": DIGEST_IMAGE}]}}, + ], + ids=["status-digest", "spec-image"], +) +def test_candidate_resolves_the_tagged_revision_and_its_digest( + fakes, tmp_path, revision_json +): + result, output = _run_cloud_run_script( + fakes, + tmp_path, + CANDIDATE, + _service(CANDIDATE_TRAFFIC), + json.dumps(revision_json), + ) + + assert result.returncode == 0, result.stderr + assert output == ( + f"url={CANDIDATE_URL}\nrevision=entry-00002-new\nimage={DIGEST_IMAGE}\n" + ) + assert fakes.calls() == [ + _describe_service(), + [ + "gcloud", + "run", + "revisions", + "describe", + "entry-00002-new", + "--project", + PROJECT, + "--region", + REGION, + "--format=json", + ], + ] + + +def test_candidate_rejects_an_image_not_pinned_by_digest(fakes, tmp_path): + revision_json = { + "spec": {"containers": [{"image": "example/policyengine-simulation-entry:a"}]} + } + + result, output = _run_cloud_run_script( + fakes, + tmp_path, + CANDIDATE, + _service(CANDIDATE_TRAFFIC), + json.dumps(revision_json), + ) + + assert result.returncode != 0 + assert output == "" + + +def test_candidate_fails_when_the_tag_is_not_deployed(fakes, tmp_path): + result, output = _run_cloud_run_script( + fakes, tmp_path, CANDIDATE, _service(CANDIDATE_TRAFFIC), TAG="s-43" + ) + + assert result.returncode != 0 + assert output == "" + assert fakes.calls() == [_describe_service()] diff --git a/rules/baselines/ci-multiline-run.txt b/rules/baselines/ci-multiline-run.txt index 0b690cf88..2a87da05d 100644 --- a/rules/baselines/ci-multiline-run.txt +++ b/rules/baselines/ci-multiline-run.txt @@ -1,7 +1,4 @@ # Tolerated existing violations. This list may only shrink. -.github/workflows/simulation-deploy.reusable.yml :: deploy_entrypoint :: Build and push entrypoint image -.github/workflows/simulation-deploy.reusable.yml :: deploy_entrypoint :: Capture stable service state before deployment -.github/workflows/simulation-deploy.reusable.yml :: deploy_entrypoint :: Resolve exact candidate metadata .github/workflows/simulation-deploy.reusable.yml :: deploy_executor :: Select versioned executor app .github/workflows/simulation-deploy.reusable.yml :: deploy_stage12_v2 :: Select the bundle-derived v2 application name .github/workflows/simulation-deploy.reusable.yml :: deploy_stage12_v2 :: Validate installed bundles, secrets, datasets, and calculations diff --git a/rules/registry.toml b/rules/registry.toml index 733f164ef..340fb6af2 100644 --- a/rules/registry.toml +++ b/rules/registry.toml @@ -670,7 +670,7 @@ job = "quality-guards" invocation = "run_quality_guards.py" [rule.baseline] file = "rules/baselines/ci-multiline-run.txt" -count = 13 +count = 10 [rule.proof] bad_fixture = "scripts/rules/tests/fixtures/ci_scripts/multiline_bad" good_fixture = "scripts/rules/tests/fixtures/ci_scripts/multiline_good" From 28b15a440e7ba61243b8310f7bf9de3b4cd99136 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 13 Oct 2026 00:26:07 +0400 Subject: [PATCH 2/5] Move the Modal routing publisher into a tested script update_routing now runs modal-publish-routing.sh with the Modal environment as its argument, as the other Modal scripts do; its body is the former inline shell, so the publisher receives the same arguments and --force-latest only when FORCE_LATEST is true. Tests assert on the exact publisher calls. The UK private-data credential sync was one backslash-continued command and is now one line. The executor app name step substitutes the version inside its single echo instead of through a temporary variable; the output is the same for any version string. The workflow-text tests now look for the force-latest flag in the script and keep the routing publisher out of the Stage 12 jobs by script name too. Co-Authored-By: Claude Opus 5.5 --- .github/scripts/modal-publish-routing.sh | 23 +++++ .../workflows/simulation-deploy.reusable.yml | 23 +---- docs/generated/rules.md | 2 +- .../tests/test_deploy_workflow_scripts.py | 84 +++++++++++++++++++ .../tests/test_deployment_assets.py | 2 + .../tests/test_modal_scripts.py | 7 +- .../tests/test_update_v2_version_manifest.py | 1 + rules/baselines/ci-multiline-run.txt | 3 - rules/registry.toml | 2 +- 9 files changed, 121 insertions(+), 26 deletions(-) create mode 100755 .github/scripts/modal-publish-routing.sh diff --git a/.github/scripts/modal-publish-routing.sh b/.github/scripts/modal-publish-routing.sh new file mode 100755 index 000000000..703e02e28 --- /dev/null +++ b/.github/scripts/modal-publish-routing.sh @@ -0,0 +1,23 @@ +#!/bin/bash +# Publish the active routing state for the deployed versioned executor app. +# Usage: modal-publish-routing.sh +# Run from projects/policyengine-simulation-executor. +# Required env vars: APP_NAME, POLICYENGINE_VERSION, POLICYENGINE_US_VERSION, +# POLICYENGINE_UK_VERSION and FORCE_LATEST; FORCE_LATEST=true forces the latest +# routing pointers for an intentional rollback. + +set -euo pipefail + +MODAL_ENV="${1:?Modal environment required}" + +args=( + --app-name "${APP_NAME}" + --policyengine-version "${POLICYENGINE_VERSION}" + --us-version "${POLICYENGINE_US_VERSION}" + --uk-version "${POLICYENGINE_UK_VERSION}" + --environment "${MODAL_ENV}" +) +if [[ "${FORCE_LATEST}" == "true" ]]; then + args+=(--force-latest) +fi +uv run python -m src.modal.utils.update_version_registry "${args[@]}" diff --git a/.github/workflows/simulation-deploy.reusable.yml b/.github/workflows/simulation-deploy.reusable.yml index e95ae2744..24dc8b09b 100644 --- a/.github/workflows/simulation-deploy.reusable.yml +++ b/.github/workflows/simulation-deploy.reusable.yml @@ -136,11 +136,7 @@ jobs: MODAL_TOKEN_ID: ${{ secrets.MODAL_TOKEN_ID }} MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }} PE_UK_PRIVATE_HF_READ_TOKEN: ${{ secrets.PE_UK_PRIVATE_HF_READ_TOKEN }} - run: | - uv run modal secret create pe-uk-private-hf-read-token \ - "HUGGING_FACE_TOKEN=${PE_UK_PRIVATE_HF_READ_TOKEN}" \ - --env="${{ inputs.modal_environment }}" \ - --force + run: uv run modal secret create pe-uk-private-hf-read-token "HUGGING_FACE_TOKEN=${PE_UK_PRIVATE_HF_READ_TOKEN}" --env="${{ inputs.modal_environment }}" --force - name: Synchronize Modal runtime secrets if: ${{ inputs.deploy_existing_stack }} @@ -350,9 +346,7 @@ jobs: - name: Select versioned executor app id: app - run: | - version_safe="${POLICYENGINE_VERSION//./-}" - echo "name=policyengine-simulation-py${version_safe}" >> "$GITHUB_OUTPUT" + run: echo "name=policyengine-simulation-py${POLICYENGINE_VERSION//./-}" >> "$GITHUB_OUTPUT" - name: Deploy versioned Modal executor working-directory: projects/policyengine-simulation-executor @@ -406,18 +400,7 @@ jobs: POLICYENGINE_US_VERSION: ${{ needs.prepare.outputs.us_version }} POLICYENGINE_UK_VERSION: ${{ needs.prepare.outputs.uk_version }} FORCE_LATEST: ${{ inputs.force_latest }} - run: | - args=( - --app-name "${APP_NAME}" - --policyengine-version "${POLICYENGINE_VERSION}" - --us-version "${POLICYENGINE_US_VERSION}" - --uk-version "${POLICYENGINE_UK_VERSION}" - --environment "${{ inputs.modal_environment }}" - ) - if [[ "${FORCE_LATEST}" == "true" ]]; then - args+=(--force-latest) - fi - uv run python -m src.modal.utils.update_version_registry "${args[@]}" + run: ../../.github/scripts/modal-publish-routing.sh "${{ inputs.modal_environment }}" configure_stage12_v2: name: Validate pre-provisioned Stage 12 runtime access diff --git a/docs/generated/rules.md b/docs/generated/rules.md index 38498f94c..d741f040c 100644 --- a/docs/generated/rules.md +++ b/docs/generated/rules.md @@ -60,7 +60,7 @@ Every rule that governs `policyengine-sim-api` is listed here with how it is enf ### Ratcheted (existing violations still tolerated) -- `sim.ci.multiline-run-is-script`: 10 remaining in `rules/baselines/ci-multiline-run.txt` +- `sim.ci.multiline-run-is-script`: 7 remaining in `rules/baselines/ci-multiline-run.txt` ### Exceptions and review dates diff --git a/projects/policyengine-simulation-entry/tests/test_deploy_workflow_scripts.py b/projects/policyengine-simulation-entry/tests/test_deploy_workflow_scripts.py index 2a027da03..e2ca8a765 100644 --- a/projects/policyengine-simulation-entry/tests/test_deploy_workflow_scripts.py +++ b/projects/policyengine-simulation-entry/tests/test_deploy_workflow_scripts.py @@ -275,3 +275,87 @@ def test_candidate_fails_when_the_tag_is_not_deployed(fakes, tmp_path): assert result.returncode != 0 assert output == "" assert fakes.calls() == [_describe_service()] + + +# --- update_routing -------------------------------------------------------- + +PUBLISH_ROUTING = "modal-publish-routing.sh" +ROUTING_ENV = { + "APP_NAME": "policyengine-simulation-py4-1-0", + "POLICYENGINE_VERSION": "4.1.0", + "POLICYENGINE_US_VERSION": "1.500.0", + "POLICYENGINE_UK_VERSION": "2.60.0", +} + + +def test_deploy_workflow_publishes_routing_with_the_tested_script(): + workflow = REUSABLE_DEPLOY_WORKFLOW.read_text(encoding="utf-8") + step = _step(workflow, "Publish active routing state") + + assert "working-directory: projects/policyengine-simulation-executor" in step + assert "APP_NAME: ${{ needs.deploy_executor.outputs.app_name }}" in step + assert "FORCE_LATEST: ${{ inputs.force_latest }}" in step + assert ( + f'run: ../../.github/scripts/{PUBLISH_ROUTING} "${{{{ inputs.modal_environment }}}}"' + in step + ) + + +@pytest.mark.parametrize( + ("force_latest", "extra"), + [("false", []), ("true", ["--force-latest"]), ("", [])], +) +def test_publish_routing_passes_the_deployed_versions(fakes, force_latest, extra): + fakes.add("uv") + + result = run_script( + PUBLISH_ROUTING, + "staging", + env=fakes.env(**ROUTING_ENV, FORCE_LATEST=force_latest), + ) + + assert result.returncode == 0, result.stderr + assert fakes.calls() == [ + [ + "uv", + "run", + "python", + "-m", + "src.modal.utils.update_version_registry", + "--app-name", + "policyengine-simulation-py4-1-0", + "--policyengine-version", + "4.1.0", + "--us-version", + "1.500.0", + "--uk-version", + "2.60.0", + "--environment", + "staging", + *extra, + ] + ] + + +def test_publish_routing_fails_when_the_publisher_fails(fakes): + fakes.add("uv", "exit 3\n") + + result = run_script( + PUBLISH_ROUTING, + "main", + env=fakes.env(**ROUTING_ENV, FORCE_LATEST="false"), + ) + + assert result.returncode == 3 + + +def test_publish_routing_requires_the_modal_environment(fakes): + fakes.add("uv") + + result = run_script( + PUBLISH_ROUTING, env=fakes.env(**ROUTING_ENV, FORCE_LATEST="false") + ) + + assert result.returncode != 0 + assert "Modal environment required" in result.stderr + assert fakes.calls() == [] diff --git a/projects/policyengine-simulation-entry/tests/test_deployment_assets.py b/projects/policyengine-simulation-entry/tests/test_deployment_assets.py index cbaf16c1a..49314062f 100644 --- a/projects/policyengine-simulation-entry/tests/test_deployment_assets.py +++ b/projects/policyengine-simulation-entry/tests/test_deployment_assets.py @@ -727,6 +727,8 @@ def test_stage12_only_deployment_cannot_redeploy_existing_modal_resources(): stage12_start = reusable_workflow.index("\n deploy_stage12_v2:") stage12_section = reusable_workflow[stage12_start:] assert "src.modal.utils.update_version_registry" not in stage12_section + # The v1 routing publisher runs from this script, only in update_routing. + assert "modal-publish-routing.sh" not in stage12_section def test_traffic_changes_are_exact_revision_validated_and_reversible(tmp_path): diff --git a/projects/policyengine-simulation-executor/tests/test_modal_scripts.py b/projects/policyengine-simulation-executor/tests/test_modal_scripts.py index d00fbb7cc..c7444c3f6 100644 --- a/projects/policyengine-simulation-executor/tests/test_modal_scripts.py +++ b/projects/policyengine-simulation-executor/tests/test_modal_scripts.py @@ -85,7 +85,12 @@ def test_deploy_workflow_threads_force_latest_to_routing_publisher(self): assert "force_latest:" in deploy_workflow assert "force_latest: ${{ inputs.force_latest || false }}" in deploy_workflow assert "force_latest:" in reusable_workflow - assert "args+=(--force-latest)" in reusable_workflow + assert "FORCE_LATEST: ${{ inputs.force_latest }}" in reusable_workflow + assert "modal-publish-routing.sh" in reusable_workflow + routing_script = (SCRIPTS_DIR / "modal-publish-routing.sh").read_text( + encoding="utf-8" + ) + assert "args+=(--force-latest)" in routing_script assert reusable_workflow.index("deploy_executor:") < reusable_workflow.index( "update_routing:" ) diff --git a/projects/policyengine-simulation-executor/tests/test_update_v2_version_manifest.py b/projects/policyengine-simulation-executor/tests/test_update_v2_version_manifest.py index 6e37359b5..96a39c842 100644 --- a/projects/policyengine-simulation-executor/tests/test_update_v2_version_manifest.py +++ b/projects/policyengine-simulation-executor/tests/test_update_v2_version_manifest.py @@ -63,6 +63,7 @@ def test_deployment_workflow_keeps_v2_optional_and_separate() -> None: assert update_job < v2_job v2_section = workflow[v2_job:] assert "src.modal.utils.update_version_registry" not in v2_section + assert "modal-publish-routing.sh" not in v2_section deploy_section = v2_section[: v2_section.index("\n publish_stage12_v2_manifest:")] for declaration in ( "OBSERVABILITY_SERVICE_NAMESPACE: ${{ vars.OBSERVABILITY_SERVICE_NAMESPACE }}", diff --git a/rules/baselines/ci-multiline-run.txt b/rules/baselines/ci-multiline-run.txt index 2a87da05d..7c7bb4188 100644 --- a/rules/baselines/ci-multiline-run.txt +++ b/rules/baselines/ci-multiline-run.txt @@ -1,11 +1,8 @@ # Tolerated existing violations. This list may only shrink. -.github/workflows/simulation-deploy.reusable.yml :: deploy_executor :: Select versioned executor app .github/workflows/simulation-deploy.reusable.yml :: deploy_stage12_v2 :: Select the bundle-derived v2 application name .github/workflows/simulation-deploy.reusable.yml :: deploy_stage12_v2 :: Validate installed bundles, secrets, datasets, and calculations .github/workflows/simulation-deploy.reusable.yml :: deploy_stage12_v2 :: Validate the packaged PolicyEngine.py bundle .github/workflows/simulation-deploy.reusable.yml :: deployment_ready :: Check deployment result -.github/workflows/simulation-deploy.reusable.yml :: prepare :: Synchronize the Modal UK private-data credential .github/workflows/simulation-deploy.reusable.yml :: publish_stage12_v2_manifest :: Publish only the separate v2 manifest .github/workflows/simulation-deploy.reusable.yml :: stage12_v2_ready :: Check Stage 12 deployment result -.github/workflows/simulation-deploy.reusable.yml :: update_routing :: Publish active routing state .github/workflows/simulation-deploy.yml :: summary :: Report promotion status diff --git a/rules/registry.toml b/rules/registry.toml index 340fb6af2..74c79b5ae 100644 --- a/rules/registry.toml +++ b/rules/registry.toml @@ -670,7 +670,7 @@ job = "quality-guards" invocation = "run_quality_guards.py" [rule.baseline] file = "rules/baselines/ci-multiline-run.txt" -count = 10 +count = 7 [rule.proof] bad_fixture = "scripts/rules/tests/fixtures/ci_scripts/multiline_bad" good_fixture = "scripts/rules/tests/fixtures/ci_scripts/multiline_good" From bb18a12839476e76063387283d090e816275edf0 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 13 Oct 2026 00:26:36 +0400 Subject: [PATCH 3/5] Move the Stage 12 v2 deploy logic into tested scripts deploy_stage12_v2 now selects the v2 application name and validates the deployed workers with stage12-v2-application-name.sh and stage12-validate-v2-workers.sh, and stage12_v2_ready checks the job results with stage12-deployment-ready.sh. Their bodies are the former inline shell, so they write the same name= and file= outputs, and a failing bundle lookup or validation still fails the step before anything is written. Tests cover the outputs, the exact uv calls, the failure paths and every combination of requested and skipped Stage 12 jobs. The packaged-bundle validation and the v2 manifest publication were one backslash-continued command each and are now one line. Co-Authored-By: Claude Opus 5.5 --- .github/scripts/stage12-deployment-ready.sh | 17 ++ .../scripts/stage12-v2-application-name.sh | 10 ++ .../scripts/stage12-validate-v2-workers.sh | 17 ++ .../workflows/simulation-deploy.reusable.yml | 35 +--- docs/generated/rules.md | 2 +- .../tests/test_deploy_workflow_scripts.py | 161 ++++++++++++++++++ rules/baselines/ci-multiline-run.txt | 5 - rules/registry.toml | 2 +- 8 files changed, 212 insertions(+), 37 deletions(-) create mode 100755 .github/scripts/stage12-deployment-ready.sh create mode 100755 .github/scripts/stage12-v2-application-name.sh create mode 100755 .github/scripts/stage12-validate-v2-workers.sh diff --git a/.github/scripts/stage12-deployment-ready.sh b/.github/scripts/stage12-deployment-ready.sh new file mode 100755 index 000000000..68a7932eb --- /dev/null +++ b/.github/scripts/stage12-deployment-ready.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +# Require the Stage 12 v2 jobs to have run exactly when they were requested: +# all skipped when not requested, all successful when requested. +# Required env vars: REQUESTED, CONFIGURATION_RESULT, DEPLOYMENT_RESULT, +# PUBLICATION_RESULT (GitHub job results). + +set -euo pipefail + +if [[ "${REQUESTED}" != "true" ]]; then + test "${CONFIGURATION_RESULT}" = "skipped" + test "${DEPLOYMENT_RESULT}" = "skipped" + test "${PUBLICATION_RESULT}" = "skipped" + exit 0 +fi +test "${CONFIGURATION_RESULT}" = "success" +test "${DEPLOYMENT_RESULT}" = "success" +test "${PUBLICATION_RESULT}" = "success" diff --git a/.github/scripts/stage12-v2-application-name.sh b/.github/scripts/stage12-v2-application-name.sh new file mode 100755 index 000000000..ad637df6e --- /dev/null +++ b/.github/scripts/stage12-v2-application-name.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# Select the versioned v2 Modal application name from the packaged +# PolicyEngine.py bundle. +# Run from projects/policyengine-simulation-executor. +# Outputs: name in GITHUB_OUTPUT. + +set -euo pipefail + +name="$(uv run python -c 'from policyengine_simulation_contract.stage12_manifest import v2_application_name; from policyengine_simulation_executor.stage12_bundle import load_stage12_bundle; print(v2_application_name(load_stage12_bundle().bundle.policyengine_version))')" +echo "name=${name}" >> "$GITHUB_OUTPUT" diff --git a/.github/scripts/stage12-validate-v2-workers.sh b/.github/scripts/stage12-validate-v2-workers.sh new file mode 100755 index 000000000..ad0ed6e7b --- /dev/null +++ b/.github/scripts/stage12-validate-v2-workers.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +# Validate the installed bundles, secrets, datasets and calculations of the +# deployed v2 workers, and keep the result for manifest publication. +# Usage: stage12-validate-v2-workers.sh +# Run from projects/policyengine-simulation-executor. +# Required env vars: RUNNER_TEMP, GITHUB_OUTPUT. +# Outputs: file (the validation result path) in GITHUB_OUTPUT. + +set -euo pipefail + +MODAL_ENV="${1:?Modal environment required}" + +output="${RUNNER_TEMP}/stage12-worker-validation.json" +uv run python -m src.modal.utils.validate_v2_worker_release \ + --environment "${MODAL_ENV}" \ + --output "${output}" +echo "file=${output}" >> "$GITHUB_OUTPUT" diff --git a/.github/workflows/simulation-deploy.reusable.yml b/.github/workflows/simulation-deploy.reusable.yml index 24dc8b09b..323f767b6 100644 --- a/.github/workflows/simulation-deploy.reusable.yml +++ b/.github/workflows/simulation-deploy.reusable.yml @@ -479,19 +479,12 @@ jobs: - name: Validate the packaged PolicyEngine.py bundle working-directory: projects/policyengine-simulation-executor - run: | - uv run python -m src.modal.utils.extract_stage12_bundle \ - --expect "policyengine_version=${{ needs.prepare.outputs.policyengine_version }}" \ - --expect "countries.us.country_package_version=${{ needs.prepare.outputs.us_version }}" \ - --expect "countries.uk.country_package_version=${{ needs.prepare.outputs.uk_version }}" \ - --output stage12-bundle.json + run: uv run python -m src.modal.utils.extract_stage12_bundle --expect "policyengine_version=${{ needs.prepare.outputs.policyengine_version }}" --expect "countries.us.country_package_version=${{ needs.prepare.outputs.us_version }}" --expect "countries.uk.country_package_version=${{ needs.prepare.outputs.uk_version }}" --output stage12-bundle.json - name: Select the bundle-derived v2 application name id: application working-directory: projects/policyengine-simulation-executor - run: | - name="$(uv run python -c 'from policyengine_simulation_contract.stage12_manifest import v2_application_name; from policyengine_simulation_executor.stage12_bundle import load_stage12_bundle; print(v2_application_name(load_stage12_bundle().bundle.policyengine_version))')" - echo "name=${name}" >> "$GITHUB_OUTPUT" + run: ../../.github/scripts/stage12-v2-application-name.sh - name: Synchronize the separate Stage 12 runtime secret working-directory: projects/policyengine-simulation-executor @@ -530,12 +523,7 @@ jobs: env: MODAL_TOKEN_ID: ${{ secrets.MODAL_TOKEN_ID }} MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }} - run: | - output="${RUNNER_TEMP}/stage12-worker-validation.json" - uv run python -m src.modal.utils.validate_v2_worker_release \ - --environment "${{ inputs.modal_environment }}" \ - --output "${output}" - echo "file=${output}" >> "$GITHUB_OUTPUT" + run: ../../.github/scripts/stage12-validate-v2-workers.sh "${{ inputs.modal_environment }}" - name: Retain worker validation for manifest publication uses: actions/upload-artifact@v7 @@ -574,11 +562,7 @@ jobs: env: MODAL_TOKEN_ID: ${{ secrets.MODAL_TOKEN_ID }} MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }} - run: | - uv run python -m src.modal.utils.update_v2_version_manifest \ - --environment "${{ inputs.modal_environment }}" \ - --manifest-name simulation-api-v2-version-manifest \ - --validation-file "${RUNNER_TEMP}/stage12-worker-validation/stage12-worker-validation.json" + run: uv run python -m src.modal.utils.update_v2_version_manifest --environment "${{ inputs.modal_environment }}" --manifest-name simulation-api-v2-version-manifest --validation-file "${RUNNER_TEMP}/stage12-worker-validation/stage12-worker-validation.json" stage12_v2_ready: name: Require requested Stage 12 v2 deployment to succeed @@ -593,16 +577,7 @@ jobs: CONFIGURATION_RESULT: ${{ needs.configure_stage12_v2.result }} DEPLOYMENT_RESULT: ${{ needs.deploy_stage12_v2.result }} PUBLICATION_RESULT: ${{ needs.publish_stage12_v2_manifest.result }} - run: | - if [[ "${REQUESTED}" != "true" ]]; then - test "${CONFIGURATION_RESULT}" = "skipped" - test "${DEPLOYMENT_RESULT}" = "skipped" - test "${PUBLICATION_RESULT}" = "skipped" - exit 0 - fi - test "${CONFIGURATION_RESULT}" = "success" - test "${DEPLOYMENT_RESULT}" = "success" - test "${PUBLICATION_RESULT}" = "success" + run: .github/scripts/stage12-deployment-ready.sh integration: name: Verify deployed three-service stack diff --git a/docs/generated/rules.md b/docs/generated/rules.md index d741f040c..3aba69567 100644 --- a/docs/generated/rules.md +++ b/docs/generated/rules.md @@ -60,7 +60,7 @@ Every rule that governs `policyengine-sim-api` is listed here with how it is enf ### Ratcheted (existing violations still tolerated) -- `sim.ci.multiline-run-is-script`: 7 remaining in `rules/baselines/ci-multiline-run.txt` +- `sim.ci.multiline-run-is-script`: 2 remaining in `rules/baselines/ci-multiline-run.txt` ### Exceptions and review dates diff --git a/projects/policyengine-simulation-entry/tests/test_deploy_workflow_scripts.py b/projects/policyengine-simulation-entry/tests/test_deploy_workflow_scripts.py index e2ca8a765..14247d7f8 100644 --- a/projects/policyengine-simulation-entry/tests/test_deploy_workflow_scripts.py +++ b/projects/policyengine-simulation-entry/tests/test_deploy_workflow_scripts.py @@ -359,3 +359,164 @@ def test_publish_routing_requires_the_modal_environment(fakes): assert result.returncode != 0 assert "Modal environment required" in result.stderr assert fakes.calls() == [] + + +# --- deploy_stage12_v2 and stage12_v2_ready ------------------------------- + +V2_APPLICATION_NAME = "stage12-v2-application-name.sh" +VALIDATE_V2_WORKERS = "stage12-validate-v2-workers.sh" +STAGE12_READY = "stage12-deployment-ready.sh" +V2_APPLICATION_CODE = ( + "from policyengine_simulation_contract.stage12_manifest import " + "v2_application_name; from policyengine_simulation_executor.stage12_bundle " + "import load_stage12_bundle; print(v2_application_name(" + "load_stage12_bundle().bundle.policyengine_version))" +) + + +def test_deploy_workflow_runs_the_stage12_worker_scripts(): + workflow = REUSABLE_DEPLOY_WORKFLOW.read_text(encoding="utf-8") + + application = _step(workflow, "Select the bundle-derived v2 application name") + assert "id: application" in application + assert "working-directory: projects/policyengine-simulation-executor" in ( + application + ) + assert f"run: ../../.github/scripts/{V2_APPLICATION_NAME}\n" in application + + validation = _step( + workflow, "Validate installed bundles, secrets, datasets, and calculations" + ) + assert "id: validation" in validation + assert "working-directory: projects/policyengine-simulation-executor" in ( + validation + ) + assert ( + f'run: ../../.github/scripts/{VALIDATE_V2_WORKERS} "${{{{ inputs.modal_environment }}}}"' + in validation + ) + + assert "application_name: ${{ steps.application.outputs.name }}" in workflow + assert "validation_file: ${{ steps.validation.outputs.file }}" in workflow + assert "path: ${{ steps.validation.outputs.file }}" in workflow + + +def test_v2_application_name_comes_from_the_packaged_bundle(fakes, tmp_path): + output = tmp_path / "github-output" + fakes.add("uv", "echo policyengine-simulation-v2-py4-1-0\n") + + result = run_script(V2_APPLICATION_NAME, env=fakes.env(GITHUB_OUTPUT=str(output))) + + assert result.returncode == 0, result.stderr + assert _output(output) == "name=policyengine-simulation-v2-py4-1-0\n" + assert fakes.calls() == [["uv", "run", "python", "-c", V2_APPLICATION_CODE]] + + +def test_v2_application_name_fails_without_writing_when_the_bundle_fails( + fakes, tmp_path +): + output = tmp_path / "github-output" + fakes.add("uv", "exit 1\n") + + result = run_script(V2_APPLICATION_NAME, env=fakes.env(GITHUB_OUTPUT=str(output))) + + assert result.returncode == 1 + assert _output(output) == "" + + +def _validate_v2_workers(fakes: FakeCommands, tmp_path: Path, *args: str): + output = tmp_path / "github-output" + runner_temp = tmp_path / "runner-temp" + runner_temp.mkdir() + result = run_script( + VALIDATE_V2_WORKERS, + *args, + env=fakes.env(GITHUB_OUTPUT=str(output), RUNNER_TEMP=str(runner_temp)), + ) + return result, _output(output), runner_temp / "stage12-worker-validation.json" + + +def test_validate_v2_workers_records_the_validation_file(fakes, tmp_path): + fakes.add("uv") + + result, output, validation_file = _validate_v2_workers(fakes, tmp_path, "main") + + assert result.returncode == 0, result.stderr + assert output == f"file={validation_file}\n" + assert fakes.calls() == [ + [ + "uv", + "run", + "python", + "-m", + "src.modal.utils.validate_v2_worker_release", + "--environment", + "main", + "--output", + str(validation_file), + ] + ] + + +def test_validate_v2_workers_fails_without_writing_when_validation_fails( + fakes, tmp_path +): + fakes.add("uv", "exit 1\n") + + result, output, _ = _validate_v2_workers(fakes, tmp_path, "main") + + assert result.returncode == 1 + assert output == "" + + +def test_validate_v2_workers_requires_the_modal_environment(fakes, tmp_path): + fakes.add("uv") + + result, output, _ = _validate_v2_workers(fakes, tmp_path) + + assert result.returncode != 0 + assert "Modal environment required" in result.stderr + assert output == "" + assert fakes.calls() == [] + + +def test_deploy_workflow_checks_stage12_results_with_the_tested_script(): + workflow = REUSABLE_DEPLOY_WORKFLOW.read_text(encoding="utf-8") + step = _step(workflow, "Check Stage 12 deployment result") + + for declaration in ( + "REQUESTED: ${{ inputs.deploy_stage12_v2 }}", + "CONFIGURATION_RESULT: ${{ needs.configure_stage12_v2.result }}", + "DEPLOYMENT_RESULT: ${{ needs.deploy_stage12_v2.result }}", + "PUBLICATION_RESULT: ${{ needs.publish_stage12_v2_manifest.result }}", + f"run: .github/scripts/{STAGE12_READY}\n", + ): + assert declaration in step + + +@pytest.mark.parametrize( + ("requested", "configuration", "deployment", "publication", "passes"), + [ + ("true", "success", "success", "success", True), + ("true", "success", "success", "failure", False), + ("true", "success", "skipped", "skipped", False), + ("true", "skipped", "skipped", "skipped", False), + ("false", "skipped", "skipped", "skipped", True), + ("false", "success", "skipped", "skipped", False), + ("false", "skipped", "skipped", "failure", False), + ], +) +def test_stage12_ready_requires_all_or_none_of_the_v2_jobs( + fakes, requested, configuration, deployment, publication, passes +): + result = run_script( + STAGE12_READY, + env=fakes.env( + REQUESTED=requested, + CONFIGURATION_RESULT=configuration, + DEPLOYMENT_RESULT=deployment, + PUBLICATION_RESULT=publication, + ), + ) + + assert result.returncode == (0 if passes else 1) diff --git a/rules/baselines/ci-multiline-run.txt b/rules/baselines/ci-multiline-run.txt index 7c7bb4188..7e5aac418 100644 --- a/rules/baselines/ci-multiline-run.txt +++ b/rules/baselines/ci-multiline-run.txt @@ -1,8 +1,3 @@ # Tolerated existing violations. This list may only shrink. -.github/workflows/simulation-deploy.reusable.yml :: deploy_stage12_v2 :: Select the bundle-derived v2 application name -.github/workflows/simulation-deploy.reusable.yml :: deploy_stage12_v2 :: Validate installed bundles, secrets, datasets, and calculations -.github/workflows/simulation-deploy.reusable.yml :: deploy_stage12_v2 :: Validate the packaged PolicyEngine.py bundle .github/workflows/simulation-deploy.reusable.yml :: deployment_ready :: Check deployment result -.github/workflows/simulation-deploy.reusable.yml :: publish_stage12_v2_manifest :: Publish only the separate v2 manifest -.github/workflows/simulation-deploy.reusable.yml :: stage12_v2_ready :: Check Stage 12 deployment result .github/workflows/simulation-deploy.yml :: summary :: Report promotion status diff --git a/rules/registry.toml b/rules/registry.toml index 74c79b5ae..c938df93b 100644 --- a/rules/registry.toml +++ b/rules/registry.toml @@ -670,7 +670,7 @@ job = "quality-guards" invocation = "run_quality_guards.py" [rule.baseline] file = "rules/baselines/ci-multiline-run.txt" -count = 7 +count = 2 [rule.proof] bad_fixture = "scripts/rules/tests/fixtures/ci_scripts/multiline_bad" good_fixture = "scripts/rules/tests/fixtures/ci_scripts/multiline_good" From 8a900f54ace075c50d59797196d5700412b5c430 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 13 Oct 2026 00:27:12 +0400 Subject: [PATCH 4/5] Move the deployment readiness check into a tested script deployment_ready now runs simulation-deployment-ready.sh, the former inline checks unchanged: every phase must succeed, and the promotion must succeed when requested and be skipped otherwise. Tests cover each combination; the existing guard test now finds the result checks in the script and the results in the step's env:. The deployment summary call was one backslash-continued command and is now one line. That removes the last multi-line run: block, so sim.ci.multiline-run-is-script is enforced with an empty baseline. Co-Authored-By: Claude Opus 5.5 --- .../scripts/simulation-deployment-ready.sh | 17 +++++++ .../workflows/simulation-deploy.reusable.yml | 10 +--- .github/workflows/simulation-deploy.yml | 7 +-- docs/generated/rules.md | 8 +-- .../tests/test_deploy_workflow_scripts.py | 51 +++++++++++++++++++ .../tests/test_deployment_assets.py | 14 ++++- rules/baselines/ci-multiline-run.txt | 2 - rules/registry.toml | 5 +- 8 files changed, 88 insertions(+), 26 deletions(-) create mode 100755 .github/scripts/simulation-deployment-ready.sh diff --git a/.github/scripts/simulation-deployment-ready.sh b/.github/scripts/simulation-deployment-ready.sh new file mode 100755 index 000000000..275ad9878 --- /dev/null +++ b/.github/scripts/simulation-deployment-ready.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# Require every deployment phase to have succeeded, and the promotion to have +# run exactly when it was requested, so a skipped phase cannot report a +# successful partial deployment. +# Required env vars: PROMOTION_REQUESTED, ENTRYPOINT_RESULT, INTEGRATION_RESULT, +# AUTHENTICATED_TEST_RESULT, PROMOTION_RESULT (GitHub job results). + +set -euo pipefail + +test "${ENTRYPOINT_RESULT}" = "success" +test "${INTEGRATION_RESULT}" = "success" +test "${AUTHENTICATED_TEST_RESULT}" = "success" +if [[ "${PROMOTION_REQUESTED}" = "true" ]]; then + test "${PROMOTION_RESULT}" = "success" +else + test "${PROMOTION_RESULT}" = "skipped" +fi diff --git a/.github/workflows/simulation-deploy.reusable.yml b/.github/workflows/simulation-deploy.reusable.yml index 323f767b6..e8eb90ab9 100644 --- a/.github/workflows/simulation-deploy.reusable.yml +++ b/.github/workflows/simulation-deploy.reusable.yml @@ -777,15 +777,7 @@ jobs: INTEGRATION_RESULT: ${{ needs.integration.result }} AUTHENTICATED_TEST_RESULT: ${{ needs.authenticated_test.result }} PROMOTION_RESULT: ${{ needs.promote_entrypoint.result }} - run: | - test "${ENTRYPOINT_RESULT}" = "success" - test "${INTEGRATION_RESULT}" = "success" - test "${AUTHENTICATED_TEST_RESULT}" = "success" - if [[ "${PROMOTION_REQUESTED}" = "true" ]]; then - test "${PROMOTION_RESULT}" = "success" - else - test "${PROMOTION_RESULT}" = "skipped" - fi + run: .github/scripts/simulation-deployment-ready.sh cleanup_entrypoint_revisions: name: Remove superseded Cloud Run revisions diff --git a/.github/workflows/simulation-deploy.yml b/.github/workflows/simulation-deploy.yml index 1ff48f7ea..e54e04972 100644 --- a/.github/workflows/simulation-deploy.yml +++ b/.github/workflows/simulation-deploy.yml @@ -81,9 +81,4 @@ jobs: - uses: actions/checkout@v6 - name: Report promotion status - run: | - .github/scripts/simulation-deployment-summary.sh \ - "${{ needs.beta.result }}" \ - "${{ needs.beta.outputs.simulation_api_url }}" \ - "${{ needs.prod.result }}" \ - "${{ needs.prod.outputs.stable_url }}" + run: .github/scripts/simulation-deployment-summary.sh "${{ needs.beta.result }}" "${{ needs.beta.outputs.simulation_api_url }}" "${{ needs.prod.result }}" "${{ needs.prod.outputs.stable_url }}" diff --git a/docs/generated/rules.md b/docs/generated/rules.md index 3aba69567..334f82353 100644 --- a/docs/generated/rules.md +++ b/docs/generated/rules.md @@ -8,7 +8,7 @@ Every rule that governs `policyengine-sim-api` is listed here with how it is enf | Tier | enforced | ratcheted | advisory | doc-only | Total | | --- | ---: | ---: | ---: | ---: | ---: | -| mechanical | 28 | 1 | 1 | 14 | 44 | +| mechanical | 29 | 0 | 1 | 14 | 44 | | metric | 0 | 0 | 0 | 0 | 0 | | judgment | 0 | 0 | 0 | 0 | 0 | @@ -60,7 +60,7 @@ Every rule that governs `policyengine-sim-api` is listed here with how it is enf ### Ratcheted (existing violations still tolerated) -- `sim.ci.multiline-run-is-script`: 2 remaining in `rules/baselines/ci-multiline-run.txt` +- None. ### Exceptions and review dates @@ -264,13 +264,13 @@ flowchart LR | Rule | Statement | Tier | Status | Blocking | Enforced by | CI | Proof | | --- | --- | --- | --- | --- | --- | --- | --- | -| `sim.ci.multiline-run-is-script` | A GitHub Actions run: block holds one line of code and no chained commands (&&, \|\|, ;); anything more lives in a tested script under .github/scripts/, called from a single-line step. | mechanical | ratcheted | yes | quality guard `ci-scripts` | `pr.yml:quality-guards` | yes | +| `sim.ci.multiline-run-is-script` | A GitHub Actions run: block holds one line of code and no chained commands (&&, \|\|, ;); anything more lives in a tested script under .github/scripts/, called from a single-line step. | mechanical | enforced | yes | quality guard `ci-scripts` | `pr.yml:quality-guards` | yes | | `sim.ci.shell-scripts-pass-shellcheck` | Every shell script under .github/scripts/ passes shellcheck. | mechanical | enforced | yes | quality guard `ci-scripts` | `pr.yml:quality-guards` | yes | | `sim.ci.workflow-scripts-tested` | Every shell script a workflow calls is run by at least one test. | mechanical | enforced | yes | quality guard `ci-scripts` | `pr.yml:quality-guards` | yes | ```mermaid flowchart LR - r_sim_ci_multiline_run_is_script("sim.ci.multiline-run-is-script (ratcheted)") + r_sim_ci_multiline_run_is_script("sim.ci.multiline-run-is-script (enforced)") e_quality_guard_ci_scripts["quality guard ci-scripts"] j_CI_pr_yml_quality_guards["CI pr.yml:quality-guards"] r_sim_ci_shell_scripts_pass_shellcheck("sim.ci.shell-scripts-pass-shellcheck (enforced)") diff --git a/projects/policyengine-simulation-entry/tests/test_deploy_workflow_scripts.py b/projects/policyengine-simulation-entry/tests/test_deploy_workflow_scripts.py index 14247d7f8..14373c3b4 100644 --- a/projects/policyengine-simulation-entry/tests/test_deploy_workflow_scripts.py +++ b/projects/policyengine-simulation-entry/tests/test_deploy_workflow_scripts.py @@ -520,3 +520,54 @@ def test_stage12_ready_requires_all_or_none_of_the_v2_jobs( ) assert result.returncode == (0 if passes else 1) + + +# --- deployment_ready ------------------------------------------------------- + +DEPLOYMENT_READY = "simulation-deployment-ready.sh" + + +def test_deploy_workflow_checks_deployment_results_with_the_tested_script(): + workflow = REUSABLE_DEPLOY_WORKFLOW.read_text(encoding="utf-8") + step = _step(workflow, "Check deployment result") + + for declaration in ( + "PROMOTION_REQUESTED: ${{ inputs.promote_entrypoint }}", + "ENTRYPOINT_RESULT: ${{ needs.deploy_entrypoint.result }}", + "INTEGRATION_RESULT: ${{ needs.integration.result }}", + "AUTHENTICATED_TEST_RESULT: ${{ needs.authenticated_test.result }}", + "PROMOTION_RESULT: ${{ needs.promote_entrypoint.result }}", + f"run: .github/scripts/{DEPLOYMENT_READY}\n", + ): + assert declaration in step + + +@pytest.mark.parametrize( + ("promotion_requested", "results", "passes"), + [ + ("true", ("success", "success", "success", "success"), True), + ("false", ("success", "success", "success", "skipped"), True), + ("true", ("success", "success", "success", "skipped"), False), + ("false", ("success", "success", "success", "success"), False), + ("true", ("success", "success", "skipped", "success"), False), + ("false", ("success", "skipped", "success", "skipped"), False), + ("false", ("failure", "success", "success", "skipped"), False), + ], +) +def test_deployment_ready_rejects_unexpected_skipped_phases( + fakes, promotion_requested, results, passes +): + entrypoint, integration, authenticated_test, promotion = results + + result = run_script( + DEPLOYMENT_READY, + env=fakes.env( + PROMOTION_REQUESTED=promotion_requested, + ENTRYPOINT_RESULT=entrypoint, + INTEGRATION_RESULT=integration, + AUTHENTICATED_TEST_RESULT=authenticated_test, + PROMOTION_RESULT=promotion, + ), + ) + + assert result.returncode == (0 if passes else 1) diff --git a/projects/policyengine-simulation-entry/tests/test_deployment_assets.py b/projects/policyengine-simulation-entry/tests/test_deployment_assets.py index 49314062f..90eff9d32 100644 --- a/projects/policyengine-simulation-entry/tests/test_deployment_assets.py +++ b/projects/policyengine-simulation-entry/tests/test_deployment_assets.py @@ -26,6 +26,9 @@ CLOUD_RUN_DEPLOY_SCRIPT = ( REPOSITORY_ROOT / ".github" / "scripts" / "deploy-cloud-run-simulation-entry.sh" ) +DEPLOYMENT_READY_SCRIPT = ( + REPOSITORY_ROOT / ".github" / "scripts" / "simulation-deployment-ready.sh" +) DEPLOY_WORKFLOW = REPOSITORY_ROOT / ".github" / "workflows" / "simulation-deploy.yml" REUSABLE_DEPLOY_WORKFLOW = ( REPOSITORY_ROOT / ".github" / "workflows" / "simulation-deploy.reusable.yml" @@ -651,6 +654,15 @@ def test_reusable_deployment_rejects_unexpected_skipped_phases(): "promote_entrypoint", ): assert dependency in completion_body + assert "run: .github/scripts/simulation-deployment-ready.sh" in completion_body + for result in ( + "ENTRYPOINT_RESULT: ${{ needs.deploy_entrypoint.result }}", + "INTEGRATION_RESULT: ${{ needs.integration.result }}", + "AUTHENTICATED_TEST_RESULT: ${{ needs.authenticated_test.result }}", + "PROMOTION_RESULT: ${{ needs.promote_entrypoint.result }}", + ): + assert result in completion_body + readiness_check = DEPLOYMENT_READY_SCRIPT.read_text(encoding="utf-8") for expected_result in ( 'test "${ENTRYPOINT_RESULT}" = "success"', 'test "${INTEGRATION_RESULT}" = "success"', @@ -658,7 +670,7 @@ def test_reusable_deployment_rejects_unexpected_skipped_phases(): 'test "${PROMOTION_RESULT}" = "success"', 'test "${PROMOTION_RESULT}" = "skipped"', ): - assert expected_result in completion_body + assert expected_result in readiness_check def test_complete_integration_suite_is_configured_for_beta_only(): diff --git a/rules/baselines/ci-multiline-run.txt b/rules/baselines/ci-multiline-run.txt index 7e5aac418..924c0983a 100644 --- a/rules/baselines/ci-multiline-run.txt +++ b/rules/baselines/ci-multiline-run.txt @@ -1,3 +1 @@ # Tolerated existing violations. This list may only shrink. -.github/workflows/simulation-deploy.reusable.yml :: deployment_ready :: Check deployment result -.github/workflows/simulation-deploy.yml :: summary :: Report promotion status diff --git a/rules/registry.toml b/rules/registry.toml index c938df93b..9df1b8712 100644 --- a/rules/registry.toml +++ b/rules/registry.toml @@ -659,7 +659,7 @@ statement = "A GitHub Actions run: block holds one line of code and no chained c rationale = "docs/engineering/skills/ci-workflows.md#one-line-of-code-per-run-block" scope = [".github/workflows/*.yml"] tier = "mechanical" -status = "ratcheted" +status = "enforced" autofix = false [rule.enforcer] kind = "quality-guard" @@ -668,9 +668,6 @@ name = "ci-scripts" workflow = ".github/workflows/pr.yml" job = "quality-guards" invocation = "run_quality_guards.py" -[rule.baseline] -file = "rules/baselines/ci-multiline-run.txt" -count = 2 [rule.proof] bad_fixture = "scripts/rules/tests/fixtures/ci_scripts/multiline_bad" good_fixture = "scripts/rules/tests/fixtures/ci_scripts/multiline_good" From 4d3c341a529cbc07e87121115f4adfcd02c8b433 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 13 Oct 2026 00:27:18 +0400 Subject: [PATCH 5/5] Add a changelog entry for the deploy workflow scripts Co-Authored-By: Claude Opus 5.5 --- changelog_entry.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/changelog_entry.yaml b/changelog_entry.yaml index c3fa6bd88..a73b4fa61 100644 --- a/changelog_entry.yaml +++ b/changelog_entry.yaml @@ -4,6 +4,7 @@ - Made CLAUDE.md and the Copilot instructions thin pointers to AGENTS.md, required workflow logic longer than one line to live in tested and shellchecked scripts, and registered the tracking-issue rule - Added a rules registry (rules/registry.toml) with a standard-library tool that validates it, checks every rule is enforced where it claims, and renders docs/generated/rules.md and the Rules block in AGENTS.md changed: + - Moved the multi-line `run` blocks of the simulation deployment workflows into single-line steps and tested scripts without changing their behaviour or outputs, and enforced the one-line `run` rule with an empty baseline - Moved multi-line `run` blocks in the PR, image smoke and client publishing workflows into single-line steps and tested scripts, tested the dependency-update scripts, fixed the shellcheck finding in modal-precompute.sh, and enforced the CI-script testing and shellcheck rules with empty baselines - Added CODEOWNERS for the rule machinery, pinned CI's ruff to 0.15.22 with an explicit rule selection, and documented that agents never bypass branch rules - Ran the Modal image smoke for Stage 12 persistence changes and added policyengine-fastapi to the PR test matrix