From 14980891f4e2cf63c2a369c06ca3041b6163412c Mon Sep 17 00:00:00 2001 From: Yuji Yokoo Date: Tue, 27 Oct 2026 19:09:29 +0900 Subject: [PATCH 1/2] feat(ci): tests-python.yml for the Python SDK workspace (#1383) The Python workspace had no CI. The new root workflow runs, on pull requests touching the workspace or the crates it compiles and on every push to main: - `python`, a matrix on Python 3.9 (the requires-python floor) and 3.14: cargo fmt, clippy, `cargo test --locked` for both crates, and pytest, through the workspace's mise tasks. MISE_PYTHON_VERSION moves a leg off the local version, and a step fails the leg if that did not take. - `wheels`, one abi3-py39 release wheel per protect-ffi platform (manylinux 2.28, musllinux 1.2, two Darwin, Windows), each checked by filename, imported on 3.9 where the runner matches, uploaded as an artifact, with its build time in the job summary; `wheels-summary` requires six and writes one table. No job holds a CS_* credential. Swatinem/rust-cache is on both compiling jobs. scripts/__tests__/python-ci.test.mjs holds the workflow to the tree: the path filter to the workspace's Cargo.lock path dependencies plus the EQL codegen and catalog, the Python matrix to the floor and the local version, the wheel matrix to the six platform packages, and every workspace task to a step that runs it. Two things the workflow made necessary: - languages/python/Cargo.lock was stale against the root crates (clww-ore 0.6.0, vitaminc 0.8.0) and every `--locked` check refused it. Re-seeded from the root lock and pruned; no shared crate differs now. - eql-suite-ci.test.mjs read `mise run test` from any workflow as EQL's `test` task. Its scan now counts a `mise run` only from a step whose working directory is under packages/eql, with fixture tests. Also excludes languages/python/** from tests.yml's live gate, records the push/pull_request asymmetry in workflow-paths-filter-parity.test.mjs, and updates AGENTS.md and the Python mise.toml notes. No Python lockfile: the package has no runtime dependency, so only pytest could be locked. Co-Authored-By: Claude Fable 5.1 --- .github/workflows/tests-python.yml | 421 ++++++++++++++++++ .github/workflows/tests.yml | 4 +- AGENTS.md | 18 +- languages/python/Cargo.lock | 178 ++++---- languages/python/mise.toml | 21 +- scripts/__tests__/eql-suite-ci.test.mjs | 120 ++++- scripts/__tests__/python-ci.test.mjs | 255 +++++++++++ .../workflow-paths-filter-parity.test.mjs | 8 + 8 files changed, 904 insertions(+), 121 deletions(-) create mode 100644 .github/workflows/tests-python.yml create mode 100644 scripts/__tests__/python-ci.test.mjs diff --git a/.github/workflows/tests-python.yml b/.github/workflows/tests-python.yml new file mode 100644 index 000000000..21c4745ff --- /dev/null +++ b/.github/workflows/tests-python.yml @@ -0,0 +1,421 @@ +name: Tests (Python) + +# The Python SDK workspace at languages/python: its two crates +# (stackencrypt-core, stackencrypt-engine) and the pure-Python package +# `stackencrypt`. The fourth Cargo workspace, and the first one whose checks +# nothing else runs: tests-crates.yml compiles the stack-* crates but never +# links them into pyo3, and tests.yml's gate excludes those crates on purpose. +# So a stack-encrypt change that breaks the engine crate is caught here or +# nowhere. scripts/__tests__/python-ci.test.mjs derives the `pull_request` +# filter below from the workspace's Cargo.lock, so a new path dependency fails +# that test until the filter follows. +# +# Two jobs. `python` runs the Rust checks and the pure-Python suite on the +# oldest supported interpreter (the `requires-python` floor) and the one +# mise.toml names, since a wheel tagged abi3-py39 is a claim about the floor +# that only a run on the floor tests. `wheels` builds one abi3 wheel per platform the FFI +# ships for, and uploads them as artifacts; nothing is released. The two jobs +# are independent, so a pytest failure does not hide whether the wheels build. +# +# No job here holds a CipherStash credential. The live test job (credentials, +# Postgres) lands with the first `encrypt` ticket (#1389), as its own job. + +on: + pull_request: + paths: + - languages/python/** + # The crates the workspace compiles by path, in the order Cargo.lock + # lists them: stack-encrypt and its derive macro, stack-kms, and the + # stack-auth and stack-profile that stack-kms's `profile` feature pulls + # in. The root manifest comes with them: their requirements are + # `{ workspace = true }` and resolve from the root Cargo.toml even when + # built from this workspace. + - packages/stack-encrypt/** + - packages/stack-encrypt-derive/** + - packages/stack-kms/** + - packages/stack-auth/** + - packages/stack-profile/** + - Cargo.toml + # The EQL codegen and the catalog it reads: the SDK's marker names are + # generated from the catalog (#1385), so a change to either must rerun + # the suite that reads them. + - packages/eql/crates/eql-codegen/** + - packages/eql/crates/eql-domains/** + # The Rust version the workspace builds under; its own mise.toml + # is under languages/python/** above. + - mise.toml + - .github/workflows/tests-python.yml + # Unfiltered, like tests-rust.yml: every push to main runs the suite, so + # main runs a superset of what pull requests run. Recorded in + # scripts/__tests__/workflow-paths-filter-parity.test.mjs. + push: + branches: [main] + workflow_dispatch: {} + +permissions: + contents: read + +defaults: + run: + shell: bash + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + RUST_BACKTRACE: full + CARGO_TERM_COLOR: always + CARGO_NET_GIT_FETCH_WITH_CLI: true + +jobs: + python: + name: "Python ${{ matrix.python }}: fmt, clippy, cargo test, pytest" + runs-on: blacksmith-4vcpu-ubuntu-2404 + timeout-minutes: 45 + strategy: + # Both interpreters report: a failure on 3.9 alone is the abi3 floor + # breaking, and that reads differently from a failure on both. + fail-fast: false + matrix: + # Quoted on purpose: YAML reads a bare 3.10 as the number 3.1. + python: ["3.9", "3.14"] + env: + RUSTFLAGS: "-D warnings" + # languages/python/mise.toml names python = "3.14" for the local loop. + # mise reads MISE__VERSION ahead of any config file, so this one + # variable moves every step, the mise-action install included, to the + # leg's interpreter; `mise run build` and `test:cargo` take `python3` + # from PATH and so follow it. The step "The interpreter is the leg's" + # below is what proves it took. + MISE_PYTHON_VERSION: ${{ matrix.python }} + defaults: + run: + working-directory: languages/python + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + persist-credentials: false + + # In the workspace folder, so mise merges languages/python/mise.toml + # (python, uv, the tasks) over the root one (rust 1.94.1). Only the + # three tools this job uses: with no install_args, `install: true` + # also installs the root's Go and golangci-lint, which nothing here + # runs. The cache key is shared by both legs (it hashes the config, not + # the environment), so the leg that saves first holds its interpreter + # and the other installs its own, uncached. That costs the other leg a + # download, and nothing else: see the interpreter check below. + - uses: jdx/mise-action@1648a7812b9aeae629881980618f079932869151 # v4 + with: + version: 2026.4.0 + install: true + install_args: python uv rust + working_directory: languages/python + cache: true + # Must name runs-on's image; see scripts/__tests__/workflow-mise-cache-image.test.mjs + cache_key_prefix: mise-v1-ubuntu-2404 + + # mise caches its rust install as a symlink into ~/.rustup, which the + # cache does not hold, so on a cache hit the components mise.toml + # declares can be missing. A no-op when they are there. The same step + # as tests-crates.yml, for the same reason. + - name: Add rustfmt and clippy to the toolchain mise installed + run: rustup component add rustfmt clippy + + # The matrix is decorative unless the interpreter on PATH is the leg's. + # A version mise stopped reading from the environment would run both + # legs on 3.14 and both would go green. + - name: The interpreter is the leg's + env: + WANT: ${{ matrix.python }} + run: | + set -euo pipefail + have="$(mise x -- python3 -c 'import sys; print("%d.%d" % sys.version_info[:2])')" + if [ "$have" != "$WANT" ]; then + echo "::error::python3 on PATH is $have, this leg wants $WANT: MISE_PYTHON_VERSION was not honoured" + exit 1 + fi + echo "python3 is $have" + + - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 + with: + workspaces: languages/python + + - name: rustfmt + run: cargo fmt --all --check + + # pyo3's build script asks the `python3` on PATH for its configuration, + # which is why this runs after mise is installed and per interpreter. + - name: clippy + run: cargo clippy --locked --no-deps --workspace --all-targets -- -D warnings + + # `cargo test --locked` for both crates. The task puts the interpreter's + # library directory on the loader path, because the engine crate links + # libpython outside a wheel. + - name: cargo test + run: mise run test:cargo + + # Builds the extension with maturin (dev profile) into a virtualenv for + # the leg's interpreter and runs pytest in it. + - name: pytest + run: mise run test + + # One abi3-py39 wheel per platform the FFI ships for, by the same Rust + # triples (python-ci.test.mjs holds the list to the protect-ffi platform + # packages). The wheels are uploaded as artifacts and nothing publishes them: + # the spec puts PyPI out of scope. The build time of each is written to the + # leg's own job summary and, as one table, to `wheels-summary`'s. + # + # Release profile, deliberately: the local loop builds the dev profile for + # speed (languages/python/mise.toml), and the number this job records is the + # cost of the wheel a user would install. + wheels: + name: "wheel ${{ matrix.platform }}" + strategy: + # One platform failing must not cancel the other five: the failure is + # usually specific to a toolchain, and seeing which of the six are fine + # is most of the diagnosis. + fail-fast: false + matrix: + # `platform` is the FFI's name for the leg and `target` its triple, + # read from languages/typescript/packages/protect-ffi/platforms/*/package.json. + # `wheel` is the platform tag the wheel's filename must carry, and the + # one fact each leg checks about what it built. `native` marks the legs + # whose runner can import the wheel it built. + # + # Linux builds run inside maturin-action's manylinux containers, which + # is what gives the gnu wheels their glibc floor: 2.28, the same floor + # the FFI's zigbuild passes as `--target .2.28`. The musl wheel + # is built in the musllinux_1_2 container, a musl system, as the FFI's + # musl binary is built in Alpine. Both Darwin legs share macos-latest, + # which is arm64, so `target` is what makes darwin-x64 an x86_64 wheel + # and not a second arm64 one with the wrong name. + include: + - platform: linux-x64-gnu + target: x86_64-unknown-linux-gnu + os: blacksmith-4vcpu-ubuntu-2404 + manylinux: "2_28" + wheel: manylinux_2_28_x86_64 + native: true + - platform: linux-arm64-gnu + target: aarch64-unknown-linux-gnu + os: blacksmith-4vcpu-ubuntu-2404 + manylinux: "2_28" + wheel: manylinux_2_28_aarch64 + native: false + - platform: linux-x64-musl + target: x86_64-unknown-linux-musl + os: blacksmith-4vcpu-ubuntu-2404 + manylinux: musllinux_1_2 + wheel: musllinux_1_2_x86_64 + native: false + - platform: darwin-x64 + target: x86_64-apple-darwin + os: macos-latest + manylinux: auto + wheel: macosx_[0-9_]*_x86_64 + native: false + - platform: darwin-arm64 + target: aarch64-apple-darwin + os: macos-latest + manylinux: auto + wheel: macosx_[0-9_]*_arm64 + native: true + - platform: win32-x64-msvc + target: x86_64-pc-windows-msvc + os: windows-latest + manylinux: auto + wheel: win_amd64 + native: true + runs-on: ${{ matrix.os }} + # A release build of the stack-encrypt graph, cross-compiled on two legs, + # on a 4 vCPU runner with no registry cache inside the container. Wider + # than the other jobs on purpose; tighten once a few runs have shown the + # real number. + timeout-minutes: 90 + steps: + # Git for Windows defaults to CRLF on checkout. `cargo fmt` is not run + # here, but a CRLF pyproject.toml is what maturin would read. + - name: Keep LF line endings + run: git config --global core.autocrlf false + + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + persist-credentials: false + + # One source of truth for the Rust version: the root mise.toml the + # `python` job builds under. mise itself is not installed here: the + # Linux legs compile inside a container, where maturin-action installs + # the toolchain this names. If that line changes shape, fail here by name + # rather than hand maturin-action an empty toolchain, which it would + # read as "stable". + - name: Rust version from mise.toml + id: rust + run: | + version=$(sed -n 's/^rust = { version = "\([^"]*\)".*/\1/p' mise.toml) + if [ -z "$version" ]; then + echo 'no rust = { version = "..." } line in mise.toml; the Rust version line has changed shape' >&2 + exit 1 + fi + echo "version=$version" >> "$GITHUB_OUTPUT" + + # The floor, not the newest: an abi3-py39 wheel claims to load on 3.9, + # and the import below is that claim tested. maturin needs no + # interpreter for an abi3 build, so this is for the import alone. + # "3.9" resolves to the newest 3.9.x the runner has a build for: 3.9.13 + # on arm64 macOS and on Windows, where the later security releases + # shipped no binaries (actions/python-versions manifest, 2026-10-27), + # and 3.9.25 on Linux and x64 macOS. + - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 + with: + python-version: "3.9" + + # The toolchain the wheel is built with, installed before the cache step + # so that step keys on it: rust-cache reads `rustc -vV` from the default + # toolchain, and without this it would key on whatever the runner image + # ships and never follow the version mise.toml names. maturin-action + # installs the same version again inside the Linux containers, where + # this install is not visible; `rustup toolchain install` is a no-op + # where it already is. + - name: Install the Rust toolchain mise.toml names + env: + RUST_VERSION: ${{ steps.rust.outputs.version }} + run: | + set -euo pipefail + rustup toolchain install "$RUST_VERSION" --profile minimal + rustup default "$RUST_VERSION" + + # Keyed by target as well as by lockfile, so the six legs do not evict + # each other. On the three host legs this caches the registry and the + # workspace `target/` directory. On the three container legs it is weak: + # the container keeps its own registry, and writes `target/` as root + # through the bind mount, so the post step may only warn. It stays on + # every leg because every leg compiles Rust (#1383). + - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 + with: + workspaces: languages/python + key: ${{ matrix.target }} + + - name: Start the clock + id: clock + run: echo "start=$(date +%s)" >> "$GITHUB_OUTPUT" + + # `--locked`: the Cargo.lock in the tree is the one built, as in every + # other check of this workspace. `--out dist` puts the wheel where the + # steps below and the artifact upload look. + - name: Build the wheel + uses: PyO3/maturin-action@e83996d129638aa358a18fbd1dfb82f0b0fb5d3b # v1.51.0 + with: + working-directory: languages/python + command: build + args: --release --locked --out dist + target: ${{ matrix.target }} + manylinux: ${{ matrix.manylinux }} + rust-toolchain: ${{ steps.rust.outputs.version }} + + # The build time is the number this job exists to record (the spec's + # scorecard asks for it per platform). Written here, to this leg's + # summary, and to a file the `wheels-summary` job gathers into one table. + - name: Record the build time + env: + PLATFORM: ${{ matrix.platform }} + START: ${{ steps.clock.outputs.start }} + run: | + set -euo pipefail + seconds=$(( $(date +%s) - START )) + mkdir -p languages/python/timing + echo "$seconds" > "languages/python/timing/${PLATFORM}.seconds" + echo "Built the ${PLATFORM} wheel in ${seconds}s" >> "$GITHUB_STEP_SUMMARY" + + # Exactly one wheel, tagged for the floor (cp39-abi3) and for this leg's + # platform. A wrong `target` on the shared Darwin runner, or a container + # that produced a manylinux tag below the floor, both show up in the + # filename and nowhere else that this job reads. + - name: The wheel is one, abi3-py39, for this platform + env: + PLATFORM: ${{ matrix.platform }} + WHEEL_TAG: ${{ matrix.wheel }} + run: | + set -euo pipefail + shopt -s nullglob + wheels=(languages/python/dist/*.whl) + if [ "${#wheels[@]}" -ne 1 ]; then + echo "::error::expected one wheel in languages/python/dist, found ${#wheels[@]}: ${wheels[*]}" + exit 1 + fi + name=$(basename "${wheels[0]}") + echo "$name" + if ! [[ "$name" =~ ^stackencrypt-[^-]+-cp39-abi3-${WHEEL_TAG}\.whl$ ]]; then + echo "::error::${name} is not a stackencrypt cp39-abi3 wheel tagged ${WHEEL_TAG} (the ${PLATFORM} leg)" + exit 1 + fi + + # On the legs whose runner is the wheel's platform, install it into the + # floor interpreter and load both halves. The cross-compiled legs + # (linux-arm64-gnu, linux-x64-musl, darwin-x64) cannot run theirs here; + # for them the filename check above is the whole verification. + - name: Import the wheel on Python 3.9 + if: ${{ matrix.native }} + run: | + set -euo pipefail + python -m pip install --no-index --no-deps languages/python/dist/*.whl + python -c 'import stackencrypt as s; print("stackencrypt", s.__version__, "engine", s.__engine__)' + + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: wheel-${{ matrix.platform }} + path: languages/python/dist/*.whl + if-no-files-found: error + retention-days: 14 + + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: wheel-timing-${{ matrix.platform }} + path: languages/python/timing/*.seconds + if-no-files-found: error + retention-days: 14 + + # The six wheels in one place: counts them, and writes one table of build + # time per platform to the job summary. A missing wheel fails here by + # platform name, so a leg that uploaded nothing is not mistaken for one + # whose job was skipped. + wheels-summary: + name: six wheels, build time per platform + needs: [wheels] + runs-on: blacksmith-4vcpu-ubuntu-2404 + timeout-minutes: 10 + steps: + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + pattern: wheel-* + merge-multiple: true + path: wheels + + - name: Six wheels, one per platform, and their build times + run: | + set -euo pipefail + shopt -s nullglob + platforms="linux-x64-gnu linux-arm64-gnu linux-x64-musl darwin-x64 darwin-arm64 win32-x64-msvc" + wheels=(wheels/*.whl) + { + echo "## Python wheels" + echo "" + echo "| platform | wheel | build time |" + echo "|---|---|---:|" + } >> "$GITHUB_STEP_SUMMARY" + missing=0 + for platform in $platforms; do + timing="wheels/${platform}.seconds" + if [ ! -f "$timing" ]; then + echo "::error::no build time recorded for ${platform}" + missing=1 + continue + fi + echo "| ${platform} | see artifact wheel-${platform} | $(cat "$timing")s |" >> "$GITHUB_STEP_SUMMARY" + done + echo "" >> "$GITHUB_STEP_SUMMARY" + printf '%s\n' "${wheels[@]##*/}" | sed 's/^/- /' >> "$GITHUB_STEP_SUMMARY" + if [ "${#wheels[@]}" -ne 6 ] || [ "$missing" -ne 0 ]; then + echo "::error::expected six wheels with six build times, found ${#wheels[@]} wheels: ${wheels[*]##*/}" + exit 1 + fi diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 7939d29e4..564e738aa 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -96,10 +96,12 @@ jobs: # Plans and ADRs; plan-docs-unread-by-tests.test.mjs holds the # plans half for the `paths:` filter above. - '!docs/**' - # The Go module, and the Rust crates no binding these jobs build + # The Go module, the Python SDK (tests-python.yml runs its + # checks), and the Rust crates no binding these jobs build # depends on. The auth and profile bindings compile stack-auth and # stack-profile, which stay in. - '!languages/golang/**' + - '!languages/python/**' - '!packages/stack-encrypt/**' - '!packages/stack-encrypt-derive/**' - '!packages/stack-kms/**' diff --git a/AGENTS.md b/AGENTS.md index 00b7aa51d..f33972804 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -93,7 +93,7 @@ Every npm package except EQL lives under `languages/typescript/`: packages in `l - `packages/stack-auth`, `packages/stack-profile`, `packages/stack-kms`, `packages/stack-encrypt`, `packages/stack-encrypt-derive`, `packages/stack-guest-abi`: The Rust crates imported from `cipherstash/cipherstash-suite` with their history — `stack-auth` and `stack-profile` (published to crates.io, one version group), `stack-kms` (published to crates.io from 0.1.0, its own version group, re-exported by `stack-encrypt` as `stack_encrypt::kms`), `stack-encrypt` and `stack-encrypt-derive` (published to crates.io from 0.1.0, one version group; `eql-bindings`' `stack-encrypt` feature depends on them from the registry), and `stack-guest-abi` (`publish = false`). They are the members of the **root Cargo workspace**, with the three node binding crates below. See "Working on the Rust crates". - `languages/typescript/packages/auth`, `languages/typescript/packages/profile`, `languages/typescript/packages/stack-auth-wasm`: The node bindings of those crates. `@cipherstash/auth` (napi-rs v2) and its six `platforms/*` packages are published to npm from this repository by `release.yml` (`auth-artifacts`, `publish-auth`); a change to what it ships, the `stack-auth` crate included, needs an `@cipherstash/auth` changeset (`require-auth-npm-changeset.yml`). `@cipherstash/profile` and its platforms are private and never published; `@cipherstash/stack-auth-wasm` is private and builds the wasm that `@cipherstash/auth` ships. Their `build` and `test` scripts never invoke cargo; `build:native`, `build:debug` and `test:cargo` do. - `languages/golang`: The Go module — the SDK `encrypt` with its generated-code support `encrypt/gensupport` and the policy packages `encrypt/policy` and `encrypt/policy/protosource`; the credential package `auth`; the generator `stashgen` and its command `cmd/stashgen`; and `internal` (the shared guest plumbing and the `record` wire model). A wazero host with no cgo. Its two WASI guests (`encrypt/guest`, `auth/guest`) are detached Cargo workspaces built by `mise run wasm:guest:build` and `mise run wasm:auth-guest:build`; `mise run wasm:guest:build:deterministic` builds the seeded TEST build the hermetic Go tests use; the `.wasm` files they embed are gitignored. The stack-encrypt guest has a second build with the EQL types (`mise run wasm:guest:build:eql`, cargo feature `eql`, linking `eql-bindings` by path; `wasm:guest:build:eql:deterministic` is its test build), embedded by `encrypt/eql` and registered on import, so a program whose generated code names an EQL type runs it (ADR-0007, amended). `encrypt/eql`'s types and `Types` table are generated from the EQL catalog by `eql-codegen` (`eql_gen.go`, written by `mise run types:generate` in `packages/eql` and drift-gated by `types:check` and `cargo test -p eql-codegen`). Generated `*_stash.go` files are committed and CI fails when `go generate ./...` changes one. Releases are cut by hand with `.github/workflows/release-golang.yml` (`gh workflow run release-golang.yml -f version=X.Y.Z[-pre] --ref `): because the guests are gitignored on main and Go fetches a version as the files committed at its tag, the workflow builds them, commits them on a release-only commit no branch points at, tags it `languages/golang/vX.Y.Z` and pushes the tag alone, then attaches the EQL v4 SQL bundle to the GitHub release. The ref must carry the EQL v4 bundle, or the build stops at "Build the EQL SQL bundles". `scripts/go-release-version.mjs` refuses, before anything is built, every version Go would refuse (non-canonical semver, `+build` metadata, a v2+ major on a module path with no `/vN` suffix, a version shaped like a pseudo-version), a final version dispatched from anywhere but `main`, and a final version while `languages/golang/go.mod` pins a `github.com/cipherstash/vitaminc/…` module at a pseudo-version (vitaminc's Go bindings have no tags, and a TEMPORARY pin on an unreleased commit must be gone before the module is tagged); a pre-release may come from a branch and may carry such a pin. The build job also refuses a version the Go proxy already lists in `@v/list`, which catches a tag deleted after the proxy fetched it; it never asks the proxy for the version itself before the tag exists, because the proxy would cache "not found" for up to 30 minutes and `verify` would fail after the push. The build checks the `Cargo.lock` of the EQL workspace and both guest workspaces with `--locked` first, since none of those builds passes it. It runs in a job with no write token and hands the guests to a `publish` job that holds `contents: write` and runs only git and gh. `publish` fetches the commit with plain unauthenticated git rather than `actions/checkout`, whose `token` input defaults to the job's writable token, and refuses a release commit whose tree is not the one the build tested. `publish` takes the paths it commits from the workflow's fixed `GUESTS` list, never from the artifact, and refuses a `guests.sha256` that is not exactly one SHA-256 line for each of those three guests: the build job runs third-party code after writing that manifest, so a path read from it could put any file on the tag (`scripts/__tests__/go-release-artifact.test.mjs`). A version is immutable once the Go proxy has fetched it, so a tag is never moved or deleted: if `publish` or `verify` fails after the tag is pushed, use "Re-run failed jobs" on the same run, which keeps a tag that run pushed and creates or completes the release; a new dispatch of the same version is refused. Never tag a main commit by hand: `go get` would fetch a module with no guests. -- `languages/python`: The Python SDK, `stackencrypt` — a pyo3 extension over `stack-encrypt`, built by maturin. **The fourth Cargo workspace**, excluded from the root one, with two crates: `crates/core` (no pyo3; the engine-side logic, linked and tested without a Python interpreter) and `crates/engine` (the `_engine` cdylib, `abi3-py39`, one wheel per platform for Python 3.9 to 3.14). The `extension-module` feature is set only by maturin in `pyproject.toml`, never in a crate manifest, so `cargo test` links libpython. Its tasks live in its own `mise.toml`, run from that folder: `mise run build` (wheel into `.venv`), `mise run test` (pytest), `mise run test:cargo` (both crates, `--locked`, libpython on the loader path). The wheel's version is `crates/engine/Cargo.toml`'s; `stackencrypt.__engine__` is the commit of this checkout, recorded by `crates/core/build.rs`. Its `Cargo.lock` was seeded from the root one because `stack-encrypt` requires a yanked `cllw-ore` pre-release that a fresh resolve cannot select. Nothing is published; a lockfile for the Python side and the CI workflow are the next tickets (#1383). Design: Python ADR-0001 under `languages/python/docs/adr/`, the spec in #1341. +- `languages/python`: The Python SDK, `stackencrypt`, a pyo3 extension over `stack-encrypt`, built by maturin. **The fourth Cargo workspace**, excluded from the root one. It has two crates. `crates/core` has no pyo3: the engine-side logic, linked and tested without a Python interpreter. `crates/engine` is the `_engine` cdylib, `abi3-py39`, one wheel per platform for Python 3.9 to 3.14. The `extension-module` feature is set only by maturin in `pyproject.toml`, never in a crate manifest, so `cargo test` links libpython. Its tasks live in its own `mise.toml` and run from that folder. `mise run build` builds the wheel into `.venv`, `mise run test` runs pytest, and `mise run test:cargo` tests both crates with `--locked` and libpython on the loader path. The wheel's version is `crates/engine/Cargo.toml`'s. `stackencrypt.__engine__` is the commit of this checkout, recorded by `crates/core/build.rs`. Its `Cargo.lock` is seeded from the root one, so every crate it shares with the root workspace resolves to the version the root tests. After a root dependency moves, re-seed it the same way: copy the root `Cargo.lock` over it and run `cargo metadata --offline` there. That command prunes the copy to what the workspace uses. Until then, `tests-python.yml` fails on `--locked`. Nothing is published. CI is `tests-python.yml`. It runs the Rust checks and pytest on Python 3.9 and 3.14, where `MISE_PYTHON_VERSION` moves a leg off the local version. It then builds one abi3 wheel per FFI platform, uploads them as artifacts, and writes each build time to the job summary. `scripts/__tests__/python-ci.test.mjs` compares its path filter with the workspace's `Cargo.lock` and its wheel matrix with the protect-ffi platform packages. There is no Python lockfile yet: the package has no runtime dependency, so only pytest could be locked. Design: Python ADR-0001 under `languages/python/docs/adr/`, the spec in #1341. - `e2e/*`: Cross-package end-to-end tests (package managers, supply chain, Prisma example README) - `languages/typescript/examples/*`: Working apps (basic, prisma, supabase-worker) - `docs/plans/*`: Internal design plans. User-facing documentation lives at https://cipherstash.com/docs (not in this repo). @@ -750,13 +750,15 @@ exact pin would stop the suite sharing one `cts-common` with it. typings to the committed `native.d.ts`; `index.d.ts` is hand-written. After a `#[napi]` change, rebuild and commit `native.d.ts`, or CI's drift guard fails. -- **CI:** `tests-crates.yml` (fmt, clippy, nextest including the trybuild - `tests/ui` binary, doctests, rustdoc, the node bindings and - `stack-auth-wasm`), `tests-golang.yml` (WASI checks, the guests, the Go - module on Linux, macOS and Windows, its PostgreSQL tests against the EQL v4 - bundle built from the tree — `mise run go:test:postgres` locally — and the - live tests), `crap-crates.yml`, - `mutants.yml`, `fuzz.yml`, `miri.yml` and `udeps.yml`. +- **CI:** `tests-crates.yml` runs fmt, clippy, nextest including the trybuild + `tests/ui` binary, doctests, rustdoc, the node bindings, and + `stack-auth-wasm`. `tests-golang.yml` runs the WASI checks, the guests, the + Go module on Linux, macOS, and Windows, and the live tests. It also runs the + Go PostgreSQL tests against the EQL v4 bundle built from the tree. Locally + that is `mise run go:test:postgres`. `tests-python.yml` runs the Python + SDK's two crates and pytest on both interpreters, and builds the six + wheels. The others are `crap-crates.yml`, `mutants.yml`, `fuzz.yml`, + `miri.yml`, and `udeps.yml`. `scripts/__tests__/crates-ci.test.mjs` fails when a mise task in the root `mise.toml` or the five crate `tasks.toml` files loses its last CI caller. diff --git a/languages/python/Cargo.lock b/languages/python/Cargo.lock index c9c64a681..3987b4b5d 100644 --- a/languages/python/Cargo.lock +++ b/languages/python/Cargo.lock @@ -168,6 +168,12 @@ version = "1.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" +[[package]] +name = "base85-simd" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbd16ef96137ca89cc38b0acb5bee36fada9707c044bc9c753f55c816bfa719" + [[package]] name = "bitflags" version = "2.10.0" @@ -317,9 +323,9 @@ dependencies = [ [[package]] name = "chrono" -version = "0.4.43" +version = "0.4.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fac4744fb15ae8337dc853fee7fb3f4e48c0fbaa23d0afe49c447b4fab126118" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" dependencies = [ "iana-time-zone", "num-traits", @@ -350,18 +356,17 @@ dependencies = [ ] [[package]] -name = "cllw-ore" -version = "0.5.2-rc.1" +name = "clww-ore" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24eb4a774d070a6c2b52175b98273cd7193c61e1bda39c1a98b50b7a2d91375a" +checksum = "b95747f5724666cb973b2ca0659e10b6083c11f99877c4727b4c9f534a591c74" dependencies = [ "blake3", "hex", "subtle", "thiserror 1.0.69", - "unicode-normalization", "vitaminc-ore", - "vitaminc-protected 0.6.1", + "vitaminc-protected 0.8.0", "zeroize", ] @@ -1432,32 +1437,26 @@ dependencies = [ "libc", ] -[[package]] -name = "orderable-bytes" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c7d30b72ffca73d9936fe5fc80a9c9520c73f5b4f64839a810211903d5ace3c" - [[package]] name = "ore-rs" -version = "0.9.0" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e1d6cb8c1249643266bd60269547c93d3cd263d5385f1262902bbb295ff9a5e" +checksum = "8bd65962ec1409c09d20e1e016f4fe1b4ae2e2e58dba38c155538b2c0801a185" dependencies = [ "aes", "block-modes", "byteorder", "hex", "num", - "orderable-bytes", - "rand 0.8.6", - "rand_chacha 0.3.1", + "rand 0.10.1", "subtle-ng", "thiserror 1.0.69", + "vitaminc-context 0.8.0", "vitaminc-hmac", "vitaminc-ore", "vitaminc-prf", - "vitaminc-protected 0.6.1", + "vitaminc-protected 0.8.0", + "vitaminc-random 0.8.0", "zeroize", ] @@ -2128,8 +2127,8 @@ dependencies = [ "tracing", "url", "uuid", - "vitaminc 0.6.1", - "vitaminc-protected 0.6.1", + "vitaminc 0.8.0", + "vitaminc-protected 0.8.0", "web-time", "zeroize", "zerokms-protocol", @@ -2140,23 +2139,29 @@ name = "stack-encrypt" version = "0.3.0" dependencies = [ "base64ct", - "cllw-ore", + "base85-simd", + "clww-ore", + "futures", "miette", "ore-rs", + "rand 0.10.1", "serde", "serde_json", + "serdect", + "sha2 0.11.0", "stack-encrypt-derive", "stack-kms", "thiserror 1.0.69", "uuid", - "vitaminc-aead 0.6.1", + "vitaminc 0.8.0", + "vitaminc-aead 0.8.0", "vitaminc-aead-frame", - "vitaminc-encrypt 0.6.1", + "vitaminc-encrypt 0.8.0", "vitaminc-hmac", "vitaminc-normalize", "vitaminc-ore", "vitaminc-prf", - "vitaminc-protected 0.6.1", + "vitaminc-protected 0.8.0", "zeroize", ] @@ -2191,8 +2196,8 @@ dependencies = [ "tracing", "url", "uuid", - "vitaminc 0.6.1", - "vitaminc-protected 0.6.1", + "vitaminc 0.8.0", + "vitaminc-protected 0.8.0", "zeroize", "zerokms-protocol", ] @@ -2658,16 +2663,16 @@ dependencies = [ [[package]] name = "vitaminc" -version = "0.6.1" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac5c15b6c25efcb9a6111f7108ab8101947a7bf7e9af3ff6d6a73358a1ace0da" +checksum = "ea37168a603d605732d0bf0672586139ab91736f767d7b85e173c84dcee19054" dependencies = [ - "vitaminc-aead 0.6.1", - "vitaminc-context 0.6.1", - "vitaminc-encrypt 0.6.1", - "vitaminc-protected 0.6.1", - "vitaminc-random 0.6.1", - "vitaminc-traits 0.6.1", + "vitaminc-aead 0.8.0", + "vitaminc-context 0.8.0", + "vitaminc-encrypt 0.8.0", + "vitaminc-protected 0.8.0", + "vitaminc-random 0.8.0", + "vitaminc-traits 0.8.0", ] [[package]] @@ -2687,18 +2692,18 @@ dependencies = [ [[package]] name = "vitaminc-aead" -version = "0.6.1" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71b8c5c4ced92545dce5fd7cf04581d08210def174cc81475cf227fa2506c5b7" +checksum = "e9bb97fd543ee6a2b168e257ec69d0d274a775b1eaa3f856b5ac2941b01891c1" dependencies = [ "bytes", "futures-util", "serde", "thiserror 2.0.18", - "vitaminc-aead-derive 0.6.1", - "vitaminc-context 0.6.1", - "vitaminc-protected 0.6.1", - "vitaminc-random 0.6.1", + "vitaminc-aead-derive 0.8.0", + "vitaminc-context 0.8.0", + "vitaminc-protected 0.8.0", + "vitaminc-random 0.8.0", "zeroize", ] @@ -2715,9 +2720,9 @@ dependencies = [ [[package]] name = "vitaminc-aead-derive" -version = "0.6.1" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c546497a972d0f0498f2f8d5f537c514bad82df8662951c63d0c80023808900" +checksum = "0845a44c07ddc882ed47a0e92e04fc83496220f054b589ad0cbd215c5009e5b7" dependencies = [ "proc-macro2", "quote", @@ -2726,12 +2731,12 @@ dependencies = [ [[package]] name = "vitaminc-aead-frame" -version = "0.6.1" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e35992a269bcaa5b201f1938e54bfbf167bfa6fe975d83cbdbdb54c2a363c5ef" +checksum = "3b0299777255171712a296d59164691dc1e93d10bddaab6e248fede1790dfd2e" dependencies = [ "serde", - "vitaminc-aead 0.6.1", + "vitaminc-aead 0.8.0", "zeroize", ] @@ -2747,12 +2752,12 @@ dependencies = [ [[package]] name = "vitaminc-context" -version = "0.6.1" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa4c4bc2bd9811253eb427657565a45bd4826e2379e93ee1c4d00c808a769882" +checksum = "a630079ba2d50680ee5048addd430eda30613559809dd6d383551e6d3d299826" dependencies = [ "mutants", - "vitaminc-protected 0.6.1", + "vitaminc-protected 0.8.0", ] [[package]] @@ -2771,71 +2776,72 @@ dependencies = [ [[package]] name = "vitaminc-encrypt" -version = "0.6.1" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0923694602855b33a296d9af473e282e8096c1cfe1b30b8695d3b6651717fc5" +checksum = "db45e073f92ae8ca27f8fe14270bc34c75e767559c2347098dacca95180d1ddd" dependencies = [ "aes-gcm", "aws-lc-rs", - "vitaminc-aead 0.6.1", - "vitaminc-protected 0.6.1", - "vitaminc-random 0.6.1", + "vitaminc-aead 0.8.0", + "vitaminc-protected 0.8.0", + "vitaminc-random 0.8.0", "zeroize", ] [[package]] name = "vitaminc-hmac" -version = "0.6.1" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd361f626e95cbe770b9d012e4a86c43cfefe4aefb3d5b3ed9339a83085bdc53" +checksum = "b5fa0ae8bbad85ae30fb5c91b0b72b694d59bf2c2c1e7ad799eda66fdbd67e7c" dependencies = [ "hmac", "sha2 0.11.0", "vitaminc-prf", - "vitaminc-protected 0.6.1", + "vitaminc-protected 0.8.0", "zeroize", ] [[package]] name = "vitaminc-normalize" -version = "0.6.1" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "214b2a9d085924c3bc3c91f3ba74238e8848bf54990e5f48f170c71fbacea07a" +checksum = "f5a2d8875e0b8b35b7a4f9dd698bbd13c989837cea564b5a4cbb5371b78fa937" dependencies = [ "thiserror 2.0.18", "unicode-general-category", "unicode-normalization", - "vitaminc-aead 0.6.1", - "vitaminc-protected 0.6.1", + "vitaminc-aead 0.8.0", + "vitaminc-protected 0.8.0", ] [[package]] name = "vitaminc-ore" -version = "0.6.1" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed88c54b5f06f2177ecbfb7666812adf645406f6fb555f4e2f0fd7fa218e71c4" +checksum = "f05518b4d35cb0c592dad59afb11b8e883ea252449b9ae47ff5ddeff0ed49f75" dependencies = [ "thiserror 2.0.18", "unicode-general-category", "unicode-normalization", - "vitaminc-aead 0.6.1", - "vitaminc-context 0.6.1", + "vitaminc-aead 0.8.0", + "vitaminc-context 0.8.0", "vitaminc-normalize", - "vitaminc-protected 0.6.1", + "vitaminc-protected 0.8.0", + "zeroize", ] [[package]] name = "vitaminc-prf" -version = "0.6.1" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2715cca25006b4656223bc932ab4fd7f70d387aa4ec63a4e2ef33aa8cd2b3a5" +checksum = "916688478273ca3e5c6805b355534ce9263688391cd4ab23b08bef760039d325" dependencies = [ "mutants", "thiserror 2.0.18", - "vitaminc-aead 0.6.1", - "vitaminc-context 0.6.1", + "vitaminc-aead 0.8.0", + "vitaminc-context 0.8.0", "vitaminc-normalize", - "vitaminc-protected 0.6.1", + "vitaminc-protected 0.8.0", ] [[package]] @@ -2857,9 +2863,9 @@ dependencies = [ [[package]] name = "vitaminc-protected" -version = "0.6.1" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1bb76dd473f5aa89b844f92018ebe3041deb8e5390e7061cca4283d959ff855" +checksum = "a6edc22562a9bbff0ba9704a2c43670b2c7cb1637524c045c3ba6436ed00917f" dependencies = [ "bitvec", "digest 0.11.3", @@ -2868,7 +2874,7 @@ dependencies = [ "serde_bytes", "subtle", "thiserror 2.0.18", - "vitaminc-protected-derive 0.6.1", + "vitaminc-protected-derive 0.8.0", "zeroize", ] @@ -2885,9 +2891,9 @@ dependencies = [ [[package]] name = "vitaminc-protected-derive" -version = "0.6.1" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7478ed160c019059746e75fc613563dcdcd5266d155a34601d29377a1c4afc3" +checksum = "a7cce3a7d0ce6f57c0e82345f20e87f3737cccd3a70aec2a2ab49f68a3a5e24a" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -2912,16 +2918,16 @@ dependencies = [ [[package]] name = "vitaminc-random" -version = "0.6.1" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b477af7c01c3d9ef2647e46952ad3df7f246b3f806b1d4009ee83beecec613a" +checksum = "8bb253f783ce6f31a55a92f5a9f0526109b8caa6a465edb94244fd344a078c11" dependencies = [ "chacha20", "getrandom 0.4.2", "rand 0.10.1", "thiserror 2.0.18", - "vitaminc-protected 0.6.1", - "vitaminc-random-derives 0.6.1", + "vitaminc-protected 0.8.0", + "vitaminc-random-derives 0.8.0", "zeroize", ] @@ -2938,9 +2944,9 @@ dependencies = [ [[package]] name = "vitaminc-random-derives" -version = "0.6.1" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab5a0de8da428c6789dc067a578a0462e411099721d616736cf29e1622230c35" +checksum = "df0b3615936d6dded4b775856f4b0fa229d745d183f0b3754ac80035f16c27d9" dependencies = [ "proc-macro2", "quote", @@ -2965,17 +2971,17 @@ dependencies = [ [[package]] name = "vitaminc-traits" -version = "0.6.1" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a900f0edcacd49c8557ab54722175869fec66926054a6fbdc53a05069db3c922" +checksum = "1e9098ee4b13446ef959583ee2f3c384df571b453179104f7b4960ffddf65ee9" dependencies = [ "anyhow", "bytes", "rmp-serde", "serde", "thiserror 2.0.18", - "vitaminc-protected 0.6.1", - "vitaminc-random 0.6.1", + "vitaminc-protected 0.8.0", + "vitaminc-random 0.8.0", "zeroize", ] diff --git a/languages/python/mise.toml b/languages/python/mise.toml index 381082ea2..6b94d4e57 100644 --- a/languages/python/mise.toml +++ b/languages/python/mise.toml @@ -4,13 +4,14 @@ # # Tasks live here and not in the root mise.toml on purpose: # scripts/__tests__/crates-ci.test.mjs fails a root task no workflow runs, -# and the CI ticket (#1383) is what gives these a caller. Run them from this +# and .github/workflows/tests-python.yml is their caller. Run them from this # directory: `mise run build`, `mise run test`, `mise run test:cargo`. [tools] -# The interpreter the build task links and tests against; the oldest -# supported (3.9) is covered in CI (#1383). uv makes the virtualenv and -# installs the package. +# The interpreter the build task links and tests against locally. CI +# (tests-python.yml) runs the same tasks on this version and on the oldest +# supported one, 3.9, by setting MISE_PYTHON_VERSION over this version. uv makes +# the virtualenv and installs the package. python = "3.14" uv = "0.8.24" @@ -22,9 +23,12 @@ set -euo pipefail # The interpreter is named: uv prefers its own managed interpreters over the # one on PATH, and the one on PATH is the python3 `[tools]` names, the one # the Rust tests link. A `.venv` made for another interpreter version is -# recreated, so a change to `[tools]` cannot leave the tests on the old one. No lockfile is written: `uv pip install` resolves against the -# index each time. The lockfile, and the Dependabot entry a tracked lockfile -# needs, come with the CI ticket. +# recreated, so a change to `[tools]` cannot leave the tests on the old one. +# No lockfile is written: `uv pip install` resolves against the index each +# time. The package has no runtime dependency, and the only thing resolved +# here is the `test` extra (pytest) plus maturin as the build backend, so a +# lockfile would fix test tooling versions alone. One comes, with its Dependabot +# entry, when the package gains a runtime dependency. want="$(python3 -c 'import sys; print("%d.%d" % sys.version_info[:2])')" have="$([ -x .venv/bin/python ] && .venv/bin/python -c 'import sys; print("%d.%d" % sys.version_info[:2])' || true)" if [ "$have" != "$want" ]; then @@ -33,7 +37,8 @@ if [ "$have" != "$want" ]; then fi # The dev profile for the local loop: maturin builds release by default, and # a release build of the stack-encrypt graph takes minutes per rebuild. The -# CI ticket decides the release profile for the wheel it builds and times. +# `wheels` job in tests-python.yml builds release, because the time it records +# is the cost of the wheel a user would install. export MATURIN_PEP517_ARGS="${MATURIN_PEP517_ARGS:---profile dev}" # The package itself, built by maturin through the PEP 517 backend, plus the # `test` extra. Always reinstall the package so a Rust change is picked up. diff --git a/scripts/__tests__/eql-suite-ci.test.mjs b/scripts/__tests__/eql-suite-ci.test.mjs index 1c9dc60fe..33ef5569a 100644 --- a/scripts/__tests__/eql-suite-ci.test.mjs +++ b/scripts/__tests__/eql-suite-ci.test.mjs @@ -68,21 +68,40 @@ const SUITE_TASKS = [ ] /** - * The part of a workflow that actually does something: `jobs:` and everything - * under it, with comment lines stripped. + * The part of a workflow that runs an EQL task: the `run:` bodies of the steps + * that execute inside `packages/eql`, with shell comment lines stripped. * - * Lifted from `integrationSuiteCi.test.ts`, and for the reason recorded there — - * a claim satisfied by prose rather than by a step. Every assertion below is a - * substring search, and this file's own header names all three suite tasks. Cut - * at `jobs:` and the `on:`/`paths:` block goes too, which is what makes "a - * `paths:` filter naming the suite is a trigger, not a run" true rather than - * merely intended. + * Only `run:` bodies, so a claim is satisfied by a step and not by prose (the + * reason `integrationSuiteCi.test.ts` cuts at `jobs:`); this file's own header + * names all three suite tasks, and so does a `paths:` filter, which is a + * trigger and not a run. + * + * Only the steps whose working directory is under `packages/eql`, because a + * mise task name is scoped to the config mise finds from the current + * directory, and nothing else. `mise run test` from `languages/python` runs + * THAT workspace's `test` task; read without the directory it counted as + * EQL's `test`, which depends on `test:sqlx` and `test:lint`, and the two + * exemptions below went stale the day `tests-python.yml` landed — a false + * "CI runs this" in the direction this file exists to rule out. The + * directory is the step's own, else the job's `defaults.run`, else the + * workflow's; AGENTS.md records that every EQL task must run that way, and + * every EQL workflow does. A composite action's bodies are appended by the + * caller unscoped, as before: an action is reached by `uses:`, which has no + * working directory of its own. */ -function executablePart(body) { - const jobsAt = body.search(/^jobs:/m) - // No `jobs:` key means nothing runs, so nothing can match. Returning the - // whole body here would hand every assertion the header comments instead. - return jobsAt === -1 ? '' : body.slice(jobsAt).replace(/^[ \t]*#.*$/gm, '') +const EQL_DIR = EQL_PREFIX.slice(0, -1) +const insideEql = (dir) => + dir !== undefined && + (String(dir) === EQL_DIR || String(dir).startsWith(EQL_PREFIX)) +function eqlRunBodies(wf) { + const workflowDefault = wf?.defaults?.run?.['working-directory'] + return Object.values(wf?.jobs ?? {}).flatMap((job) => { + const jobDefault = + job?.defaults?.run?.['working-directory'] ?? workflowDefault + return (Array.isArray(job?.steps) ? job.steps : []) + .filter((step) => insideEql(step?.['working-directory'] ?? jobDefault)) + .map((step) => String(step?.run ?? '').replace(/^[ \t]*#.*$/gm, '')) + }) } /** The two spellings GitHub accepts for a composite action's manifest. */ @@ -157,18 +176,83 @@ const workflowSteps = (wf) => * makes the mutation test below report a workflow as its own sole caller. */ const rootWorkflows = workflowFiles().map((relPath) => { - const actions = workflowSteps(readWorkflow(relPath)) + const wf = readWorkflow(relPath) + const actions = workflowSteps(wf) .map((step) => String(step?.uses ?? '')) .map((uses) => actionRunBodies(uses)) return { relPath, - body: [ - executablePart(readFileSync(join(REPO_ROOT, relPath), 'utf8')), - ...actions, - ].join('\n'), + body: [...eqlRunBodies(wf), ...actions].join('\n'), } }) +describe('a `mise run` counts as an EQL task only from inside packages/eql', () => { + const step = (run, dir) => + dir === undefined ? { run } : { run, 'working-directory': dir } + const workflow = (jobs, defaults) => ({ defaults, jobs }) + + it('reads the step, the job default and the workflow default, in that order', () => { + const wf = workflow( + { + own: { steps: [step('mise run test:schema', 'packages/eql')] }, + job: { + defaults: { run: { 'working-directory': 'packages/eql' } }, + steps: [step('mise run test:crates')], + }, + nested: { + steps: [step('mise run build', 'packages/eql/packages/eql')], + }, + overridden: { + defaults: { run: { 'working-directory': 'packages/eql' } }, + steps: [step('mise run test:cargo', 'languages/python')], + }, + }, + { run: { 'working-directory': 'packages/eql' } }, + ) + expect(eqlRunBodies(wf)).toEqual([ + 'mise run test:schema', + 'mise run test:crates', + 'mise run build', + ]) + }) + + it("does not read another workspace's task of the same name as EQL's", () => { + // tests-python.yml's `python` job: `mise run test` and `mise run + // test:cargo` under languages/python. EQL has a `test` task too, and it + // depends on `test:sqlx` and `test:lint`; counted here, both read as run + // by CI and their exemptions below as stale. + const wf = workflow({ + python: { + defaults: { run: { 'working-directory': 'languages/python' } }, + steps: [step('mise run test:cargo'), step('mise run test')], + }, + root: { steps: [step('mise run test:doc')] }, + }) + expect(eqlRunBodies(wf)).toEqual([]) + }) + + it('strips shell comment lines, so a mention in a comment is not a run', () => { + const wf = workflow({ + eql: { + steps: [ + step( + '# run mise run test:sqlx first\nmise run test:schema', + 'packages/eql', + ), + ], + }, + }) + expect(eqlRunBodies(wf)).toEqual(['\nmise run test:schema']) + }) + + it('finds a real step this way, so the scan is not fixture-only', () => { + const real = rootWorkflows.find( + ({ relPath }) => relPath === `${WORKFLOW_DIR}/test-eql.yml`, + ) + expect(real?.body).toContain('test:sqlx:partition') + }) +}) + describe('a root workflow runs the EQL SQLx suite', () => { it('finds workflows at the repository root at all', () => { // Guards the guard. An empty scan satisfies every "some workflow contains diff --git a/scripts/__tests__/python-ci.test.mjs b/scripts/__tests__/python-ci.test.mjs new file mode 100644 index 000000000..208ca8c51 --- /dev/null +++ b/scripts/__tests__/python-ci.test.mjs @@ -0,0 +1,255 @@ +import { existsSync, readdirSync, readFileSync } from 'node:fs' +import { join } from 'node:path' +import { describe, expect, it } from 'vitest' +import { filterCovers } from './lib/paths-filter.mjs' +import { REPO_ROOT } from './lib/repo-root.mjs' +import { readWorkflow } from './lib/workflows.mjs' + +/** + * `tests-python.yml` is held to the Python workspace it tests, with every + * expectation derived from the tree rather than copied into this file. + * + * The workspace at `languages/python` is the fourth Cargo workspace, and the + * first whose checks a path filter can skip without anything else noticing: + * `tests-crates.yml` compiles the stack-* crates but never links them into + * pyo3, and `tests.yml`'s gate excludes those crates on purpose. So a + * stack-encrypt change that breaks the Python engine crate is caught here or + * nowhere, and "here" depends on the `pull_request` filter naming every tree + * the workspace compiles. That list is the workspace's path dependencies, + * which `Cargo.lock` records as packages with no `source`: a new path + * dependency lands in the lockfile, and this file fails until the filter + * follows. + * + * The other facts are read the same way: the two Python versions from + * `pyproject.toml`'s floor and `mise.toml`'s own version, and the six wheel targets + * from the protect-ffi platform packages (the issue names "the six platforms + * the FFI already ships for"). A seventh platform or a moved floor changes the + * source and this file says so. + */ + +const WORKFLOW = '.github/workflows/tests-python.yml' +const PYTHON_ROOT = 'languages/python' + +const read = (rel) => readFileSync(join(REPO_ROOT, rel), 'utf8') + +/** `wf.on`, under YAML 1.1's reading of the bare key `on` as `true`. */ +function triggers() { + const wf = readWorkflow(WORKFLOW) + return wf.on ?? wf[true] +} + +/** + * The directories outside `languages/python` whose crates the workspace + * compiles: every `[[package]]` in its Cargo.lock without a `source` line is a + * path dependency, and the stack-* ones live at `packages/`. + */ +function pathDependencyTrees() { + const lock = read(`${PYTHON_ROOT}/Cargo.lock`) + const trees = new Set() + for (const block of lock.split('[[package]]').slice(1)) { + if (/^source = /m.test(block)) continue + const name = /^name = "([^"]+)"/m.exec(block)?.[1] + if (!name) throw new Error(`Cargo.lock package with no name:\n${block}`) + if (name.startsWith('stackencrypt-')) continue + const dir = `packages/${name}` + if (!existsSync(join(REPO_ROOT, dir, 'Cargo.toml'))) { + throw new Error( + `${PYTHON_ROOT}/Cargo.lock lists ${name} as a path dependency, and ${dir}/Cargo.toml does not exist; teach this test where it lives`, + ) + } + trees.add(dir) + } + return [...trees].sort() +} + +/** + * A `paths:` glob of the form `dir/**` selects a file under `dir`. Only that + * form: a narrower glob (`dir/src/**`) would leave part of the tree unwatched, + * and GitHub's full glob grammar is not needed to read this one filter. + */ +const coversTree = (pattern, file) => + pattern.endsWith('/**') && file.startsWith(pattern.slice(0, -2)) + +describe('tests-python.yml', () => { + it('runs on pull requests touching every tree the Python workspace compiles', () => { + const on = triggers() + const paths = on.pull_request?.paths ?? [] + const wanted = [ + PYTHON_ROOT, + ...pathDependencyTrees(), + // The issue names the EQL codegen: the marker names the SDK ships are + // generated by eql-codegen from the eql-domains catalog (#1385), and a + // change to either must rerun the suite that reads the generated names. + 'packages/eql/crates/eql-codegen', + 'packages/eql/crates/eql-domains', + ] + const missing = wanted.filter( + (dir) => !filterCovers(paths, `${dir}/x`, coversTree), + ) + expect( + missing, + `pull_request.paths does not select these trees, which the Python workspace compiles or generates from:\n${missing.join('\n')}`, + ).toEqual([]) + }) + + it('runs the pure-Python suite on the oldest supported Python and the one mise.toml names', () => { + // The floor is `requires-python` in pyproject.toml; the local version is the + // interpreter `mise run test` links locally. A wheel tagged abi3-py39 is a + // claim about the floor, and only a run on the floor tests it. + const floor = /^requires-python = ">=(\d+\.\d+)"/m.exec( + read(`${PYTHON_ROOT}/pyproject.toml`), + )?.[1] + const local = /^python = "(\d+\.\d+)"/m.exec( + read(`${PYTHON_ROOT}/mise.toml`), + )?.[1] + expect(floor, 'pyproject.toml requires-python floor').toBeTruthy() + expect(local, 'languages/python/mise.toml python version').toBeTruthy() + + const wf = readWorkflow(WORKFLOW) + const matrices = Object.entries(wf.jobs) + .map(([id, job]) => [id, job.strategy?.matrix?.python]) + .filter(([, versions]) => Array.isArray(versions)) + expect(matrices.length, 'one job carries a `python:` matrix').toBe(1) + const [[id, versions]] = matrices + // Strings, not floats: YAML reads a bare `3.10` as the number 3.1. + expect( + versions.map(String).sort(), + `jobs.${id}.strategy.matrix.python must be exactly the floor and the local version`, + ).toEqual([floor, local].sort()) + for (const v of versions) { + expect(typeof v, `matrix.python entry ${v} must be a quoted string`).toBe( + 'string', + ) + } + }) + + it('builds one wheel per platform the FFI ships for, by the same Rust triples', () => { + // The six platform packages each carry `neon.rust`, the triple their + // binary is built for. The issue names "the six platforms the FFI already + // ships for", so the wheel matrix is held to that list rather than to a + // copy of it: a seventh platform package, or a moved triple, fails here. + const platformsDir = 'languages/typescript/packages/protect-ffi/platforms' + const ffi = readdirSync(join(REPO_ROOT, platformsDir)) + .map((name) => { + const pkg = JSON.parse(read(`${platformsDir}/${name}/package.json`)) + return { platform: name, target: pkg.neon?.rust } + }) + .filter((p) => p.target) + .sort((a, b) => a.platform.localeCompare(b.platform)) + expect(ffi.length, 'protect-ffi platform packages').toBe(6) + + const wf = readWorkflow(WORKFLOW) + const wheelJobs = Object.entries(wf.jobs).filter( + ([, job]) => + Array.isArray(job.strategy?.matrix?.include) && + job.strategy.matrix.include.some((leg) => leg.target), + ) + expect(wheelJobs.length, 'one job carries a matrix of `target` legs').toBe( + 1, + ) + const [[id, job]] = wheelJobs + const legs = job.strategy.matrix.include + .map(({ platform, target }) => ({ platform, target })) + .sort((a, b) => a.platform.localeCompare(b.platform)) + expect( + legs, + `jobs.${id}.strategy.matrix.include (platform, target)`, + ).toEqual(ffi) + }) + + it('restores a Rust cache in every job that compiles Rust', () => { + // The acceptance criterion, and the repo-wide rule eql-suite-ci.test.mjs + // holds the EQL workflows to. A job compiles Rust when a step runs cargo + // directly, through a mise task of the Python workspace (every one of + // them builds), or through maturin-action. + const wf = readWorkflow(WORKFLOW) + const compiles = (step) => + /\bcargo\b|\bmise run\b/.test(step.run ?? '') || + /^PyO3\/maturin-action@/.test(step.uses ?? '') + const missing = Object.entries(wf.jobs) + .filter(([, job]) => job.steps.some(compiles)) + .filter( + ([, job]) => + !job.steps.some((step) => + /^Swatinem\/rust-cache@/.test(step.uses ?? ''), + ), + ) + .map(([id]) => id) + expect( + missing, + 'jobs that compile Rust with no Swatinem/rust-cache step', + ).toEqual([]) + }) + + it('runs every task of the Python workspace, and the two cargo lints, from that folder', () => { + // "A check nothing invokes reads exactly like a check that passes" + // (crates-ci.test.mjs). That file scans the root mise.toml and the crate + // task files; the Python tasks live in languages/python/mise.toml, which + // it does not read, so this is their caller check. Every task defined + // there is run by a step whose working directory is the workspace, except + // one another task depends on. fmt and clippy are plain cargo commands, + // not tasks, so they are named here. + const miseToml = read(`${PYTHON_ROOT}/mise.toml`) + const tasks = [ + ...miseToml.matchAll(/^\[tasks\.(?:"([^"]+)"|([\w:-]+))\]/gm), + ].map((m) => m[1] ?? m[2]) + const dependedOn = new Set( + [...miseToml.matchAll(/^depends = \[([^\]]*)\]/gm)].flatMap((m) => + [...m[1].matchAll(/"([^"]+)"/g)].map((d) => d[1]), + ), + ) + expect(tasks, 'tasks read from languages/python/mise.toml').toContain( + 'test', + ) + + const wf = readWorkflow(WORKFLOW) + const workflowDefault = wf.defaults?.run?.['working-directory'] + const runs = Object.values(wf.jobs).flatMap((job) => { + const jobDefault = + job.defaults?.run?.['working-directory'] ?? workflowDefault + return job.steps + .filter( + (step) => (step['working-directory'] ?? jobDefault) === PYTHON_ROOT, + ) + .map((step) => step.run ?? '') + }) + const invoked = (command) => + runs.some((run) => + new RegExp(`(^|[\\s;&|])${command}(\\s|$)`, 'm').test(run), + ) + + const wanted = [ + ...tasks.filter((t) => !dependedOn.has(t)).map((t) => `mise run ${t}`), + 'cargo fmt --all --check', + 'cargo clippy', + ] + const missing = wanted.filter((command) => !invoked(command)) + expect( + missing, + `no step with working-directory ${PYTHON_ROOT} runs these:\n${missing.join('\n')}`, + ).toEqual([]) + }) + + it('lists in the summary job the same platforms the wheel matrix builds', () => { + // `wheels-summary` cannot see the matrix, so it carries the list a second + // time. This holds the copy to the original. + const wf = readWorkflow(WORKFLOW) + const matrix = wf.jobs.wheels.strategy.matrix.include.map( + (leg) => leg.platform, + ) + const summary = wf.jobs['wheels-summary'].steps + .map((s) => s.run ?? '') + .join('\n') + const listed = /^\s*platforms="([^"]+)"/m.exec(summary)?.[1]?.split(/\s+/) + expect(listed, 'a `platforms="…"` line in wheels-summary').toBeTruthy() + expect([...listed].sort()).toEqual([...matrix].sort()) + }) + + it('gives no job a CipherStash credential', () => { + // The live job (credentials, Postgres) is #1389's and lands as its own + // job. Until then nothing in this file may read a CS_* variable or + // secret: a credentialed job is one ffi-binding-step-order.test.mjs + // holds to a pre-flight and a binding build this workflow has no use for. + expect(read(WORKFLOW)).not.toMatch(/(vars|secrets)\.CS_/) + }) +}) diff --git a/scripts/__tests__/workflow-paths-filter-parity.test.mjs b/scripts/__tests__/workflow-paths-filter-parity.test.mjs index 5e719237c..83c9ad5e2 100644 --- a/scripts/__tests__/workflow-paths-filter-parity.test.mjs +++ b/scripts/__tests__/workflow-paths-filter-parity.test.mjs @@ -122,6 +122,14 @@ const EXPECTED_ASYMMETRIES = new Map([ // unchecked — is unreachable when `push` is unfiltered. 'push is deliberately unfiltered (branches: [main]), so main runs a superset of what PRs run', ], + [ + '.github/workflows/tests-python.yml', + // The same shape as tests-rust.yml: `push:` is `branches: [main]` with no + // `paths:`, as #1383 asks ("on every push to main"), and only pull + // requests are filtered. The pull_request list is held to the Python + // workspace's Cargo.lock by python-ci.test.mjs, which is the second copy. + 'push is deliberately unfiltered (branches: [main]), so main runs a superset of what PRs run', + ], [ '.github/workflows/tests.yml', // `push: branches: [main]` has no `paths:`, so main runs everything and only From a6212a7047463814136a47e193367e73d95f5647 Mon Sep 17 00:00:00 2001 From: Yuji Yokoo Date: Tue, 27 Oct 2026 19:47:39 +0900 Subject: [PATCH 2/2] fix(ci): address the review of tests-python.yml (#1383) Plain-language pass over the workflow comments, the test comments, and the AGENTS.md bullet: short sentences, no idioms, and the interpreter check step named as a sentence. The timeout comment states a fact and no longer carries an open task. AGENTS.md now says the Python workspace runs plain `cargo test`, not nextest, and why: no test there sets an environment variable, and nextest comes from mise.test.toml, which its CI job does not load. `triggers(wf)` and `runWorkingDirectory(wf, job, step)` move into scripts/__tests__/lib/workflows.mjs. shared-test-helpers.test.mjs rejects a local copy of an exported helper, so the six guards that carried their own `triggers` switch to the shared one in this commit. The wheel count is written once, in python-ci.test.mjs. wheels-summary derives its expected count from its platform list. The clock comment says what the recorded build time includes. Co-Authored-By: Claude Fable 5.1 --- .github/workflows/tests-python.yml | 243 +++++++++--------- AGENTS.md | 2 +- .../__tests__/claude-review-workflow.test.mjs | 10 +- .../__tests__/code-review-workflow.test.mjs | 8 +- scripts/__tests__/crates-ci.test.mjs | 9 +- scripts/__tests__/eql-suite-ci.test.mjs | 42 +-- .../__tests__/eql-workflow-filters.test.mjs | 8 +- scripts/__tests__/lib/workflows.mjs | 30 ++- scripts/__tests__/lint-release-scope.test.mjs | 12 +- scripts/__tests__/python-ci.test.mjs | 45 ++-- .../workflow-dispatch-job-conditions.test.mjs | 12 +- 11 files changed, 222 insertions(+), 199 deletions(-) diff --git a/.github/workflows/tests-python.yml b/.github/workflows/tests-python.yml index 21c4745ff..69380addd 100644 --- a/.github/workflows/tests-python.yml +++ b/.github/workflows/tests-python.yml @@ -1,51 +1,51 @@ name: Tests (Python) -# The Python SDK workspace at languages/python: its two crates -# (stackencrypt-core, stackencrypt-engine) and the pure-Python package -# `stackencrypt`. The fourth Cargo workspace, and the first one whose checks -# nothing else runs: tests-crates.yml compiles the stack-* crates but never -# links them into pyo3, and tests.yml's gate excludes those crates on purpose. -# So a stack-encrypt change that breaks the engine crate is caught here or -# nowhere. scripts/__tests__/python-ci.test.mjs derives the `pull_request` -# filter below from the workspace's Cargo.lock, so a new path dependency fails +# The Python SDK workspace at languages/python: the crates stackencrypt-core +# and stackencrypt-engine, and the pure-Python package `stackencrypt`. It is +# the fourth Cargo workspace. No other workflow runs its checks. +# tests-crates.yml compiles the stack-* crates but never links them into +# pyo3. tests.yml's gate excludes those crates on purpose. So this workflow +# is the only one that catches a stack-encrypt change that breaks the engine +# crate. scripts/__tests__/python-ci.test.mjs derives the `pull_request` +# filter below from the workspace's Cargo.lock. A new path dependency fails # that test until the filter follows. # -# Two jobs. `python` runs the Rust checks and the pure-Python suite on the -# oldest supported interpreter (the `requires-python` floor) and the one -# mise.toml names, since a wheel tagged abi3-py39 is a claim about the floor -# that only a run on the floor tests. `wheels` builds one abi3 wheel per platform the FFI -# ships for, and uploads them as artifacts; nothing is released. The two jobs -# are independent, so a pytest failure does not hide whether the wheels build. +# Two jobs. `python` runs the Rust checks and the pure-Python suite on two +# interpreters: the oldest supported one (the `requires-python` floor) and +# the one mise.toml names. A wheel tagged abi3-py39 claims to work on the +# floor. Only a run on the floor tests that claim. `wheels` builds one abi3 +# wheel per platform the FFI ships for and uploads them as artifacts. Nothing +# is released. The two jobs are independent, so a pytest failure does not +# hide whether the wheels build. # # No job here holds a CipherStash credential. The live test job (credentials, -# Postgres) lands with the first `encrypt` ticket (#1389), as its own job. +# Postgres) arrives with the first `encrypt` ticket, #1389, as its own job. on: pull_request: paths: - languages/python/** # The crates the workspace compiles by path, in the order Cargo.lock - # lists them: stack-encrypt and its derive macro, stack-kms, and the - # stack-auth and stack-profile that stack-kms's `profile` feature pulls - # in. The root manifest comes with them: their requirements are - # `{ workspace = true }` and resolve from the root Cargo.toml even when - # built from this workspace. + # lists them. The last two are what stack-kms's `profile` feature pulls + # in. The root manifest + # comes with them. Their requirements are `{ workspace = true }`. Those + # resolve from the root Cargo.toml even when built from this workspace. - packages/stack-encrypt/** - packages/stack-encrypt-derive/** - packages/stack-kms/** - packages/stack-auth/** - packages/stack-profile/** - Cargo.toml - # The EQL codegen and the catalog it reads: the SDK's marker names are - # generated from the catalog (#1385), so a change to either must rerun - # the suite that reads them. + # The EQL codegen and the catalog it reads. The SDK's marker names are + # generated from the catalog (#1385). A change to either must rerun the + # suite that reads them. - packages/eql/crates/eql-codegen/** - packages/eql/crates/eql-domains/** - # The Rust version the workspace builds under; its own mise.toml - # is under languages/python/** above. + # The Rust version the workspace builds under. The workspace's own + # mise.toml is under languages/python/** above. - mise.toml - .github/workflows/tests-python.yml - # Unfiltered, like tests-rust.yml: every push to main runs the suite, so + # Unfiltered, like tests-rust.yml. Every push to main runs the suite, so # main runs a superset of what pull requests run. Recorded in # scripts/__tests__/workflow-paths-filter-parity.test.mjs. push: @@ -74,8 +74,8 @@ jobs: runs-on: blacksmith-4vcpu-ubuntu-2404 timeout-minutes: 45 strategy: - # Both interpreters report: a failure on 3.9 alone is the abi3 floor - # breaking, and that reads differently from a failure on both. + # Both interpreters report. A failure on 3.9 alone means the abi3 floor + # is broken. That reads differently from a failure on both. fail-fast: false matrix: # Quoted on purpose: YAML reads a bare 3.10 as the number 3.1. @@ -83,11 +83,11 @@ jobs: env: RUSTFLAGS: "-D warnings" # languages/python/mise.toml names python = "3.14" for the local loop. - # mise reads MISE__VERSION ahead of any config file, so this one - # variable moves every step, the mise-action install included, to the - # leg's interpreter; `mise run build` and `test:cargo` take `python3` - # from PATH and so follow it. The step "The interpreter is the leg's" - # below is what proves it took. + # mise reads MISE__VERSION before any config file. So this one + # variable moves every step to the leg's interpreter, the mise-action + # install included. `mise run build` and `test:cargo` take `python3` + # from PATH, so they follow it. The step "Check that python3 is the + # leg's interpreter" below fails the leg when that did not happen. MISE_PYTHON_VERSION: ${{ matrix.python }} defaults: run: @@ -97,14 +97,14 @@ jobs: with: persist-credentials: false - # In the workspace folder, so mise merges languages/python/mise.toml + # Run in the workspace folder, so mise merges languages/python/mise.toml # (python, uv, the tasks) over the root one (rust 1.94.1). Only the - # three tools this job uses: with no install_args, `install: true` - # also installs the root's Go and golangci-lint, which nothing here - # runs. The cache key is shared by both legs (it hashes the config, not - # the environment), so the leg that saves first holds its interpreter - # and the other installs its own, uncached. That costs the other leg a - # download, and nothing else: see the interpreter check below. + # three tools this job uses are installed. With no install_args, + # `install: true` also installs the root's Go and golangci-lint, and + # nothing here runs them. The cache key hashes the config, not the + # environment, so both legs share it. The leg that saves first holds + # its interpreter. The other leg downloads its own, uncached. That + # costs a download and nothing else: see the interpreter check below. - uses: jdx/mise-action@1648a7812b9aeae629881980618f079932869151 # v4 with: version: 2026.4.0 @@ -115,17 +115,17 @@ jobs: # Must name runs-on's image; see scripts/__tests__/workflow-mise-cache-image.test.mjs cache_key_prefix: mise-v1-ubuntu-2404 - # mise caches its rust install as a symlink into ~/.rustup, which the - # cache does not hold, so on a cache hit the components mise.toml - # declares can be missing. A no-op when they are there. The same step - # as tests-crates.yml, for the same reason. + # mise caches its rust install as a symlink into ~/.rustup. The cache + # does not hold ~/.rustup, so on a cache hit the components mise.toml + # declares can be missing. The command does nothing when they are + # there. tests-crates.yml has the same step for the same reason. - name: Add rustfmt and clippy to the toolchain mise installed run: rustup component add rustfmt clippy - # The matrix is decorative unless the interpreter on PATH is the leg's. - # A version mise stopped reading from the environment would run both - # legs on 3.14 and both would go green. - - name: The interpreter is the leg's + # The matrix tests nothing unless the interpreter on PATH is the leg's. + # If mise stopped reading the version from the environment, both legs + # would run on 3.14 and both would pass. + - name: Check that python3 is the leg's interpreter env: WANT: ${{ matrix.python }} run: | @@ -144,8 +144,8 @@ jobs: - name: rustfmt run: cargo fmt --all --check - # pyo3's build script asks the `python3` on PATH for its configuration, - # which is why this runs after mise is installed and per interpreter. + # pyo3's build script asks the `python3` on PATH for its configuration. + # That is why this runs after mise is installed, once per interpreter. - name: clippy run: cargo clippy --locked --no-deps --workspace --all-targets -- -D warnings @@ -156,40 +156,41 @@ jobs: run: mise run test:cargo # Builds the extension with maturin (dev profile) into a virtualenv for - # the leg's interpreter and runs pytest in it. + # the leg's interpreter, then runs pytest in it. - name: pytest run: mise run test # One abi3-py39 wheel per platform the FFI ships for, by the same Rust - # triples (python-ci.test.mjs holds the list to the protect-ffi platform - # packages). The wheels are uploaded as artifacts and nothing publishes them: - # the spec puts PyPI out of scope. The build time of each is written to the - # leg's own job summary and, as one table, to `wheels-summary`'s. + # triples. python-ci.test.mjs holds the list to the protect-ffi platform + # packages. The wheels are uploaded as artifacts. Nothing publishes them: + # the spec puts PyPI out of scope. Each build time is written to the leg's + # own job summary and, as one table, to `wheels-summary`'s. # - # Release profile, deliberately: the local loop builds the dev profile for - # speed (languages/python/mise.toml), and the number this job records is the + # Release profile, on purpose. The local loop builds the dev profile for + # speed (languages/python/mise.toml). The number this job records is the # cost of the wheel a user would install. wheels: name: "wheel ${{ matrix.platform }}" strategy: - # One platform failing must not cancel the other five: the failure is - # usually specific to a toolchain, and seeing which of the six are fine - # is most of the diagnosis. + # One platform failing must not cancel the others. The failure is + # usually specific to a toolchain. Seeing which platforms pass is most + # of the diagnosis. fail-fast: false matrix: # `platform` is the FFI's name for the leg and `target` its triple, # read from languages/typescript/packages/protect-ffi/platforms/*/package.json. - # `wheel` is the platform tag the wheel's filename must carry, and the - # one fact each leg checks about what it built. `native` marks the legs - # whose runner can import the wheel it built. + # `wheel` is the platform tag the wheel's filename must carry. It is + # the one fact each leg checks about what it built. `native` marks + # the legs whose runner can import the wheel it built. # - # Linux builds run inside maturin-action's manylinux containers, which - # is what gives the gnu wheels their glibc floor: 2.28, the same floor - # the FFI's zigbuild passes as `--target .2.28`. The musl wheel - # is built in the musllinux_1_2 container, a musl system, as the FFI's - # musl binary is built in Alpine. Both Darwin legs share macos-latest, - # which is arm64, so `target` is what makes darwin-x64 an x86_64 wheel - # and not a second arm64 one with the wrong name. + # Linux builds run inside maturin-action's manylinux containers. That + # gives the gnu wheels their glibc floor, 2.28. The FFI's zigbuild + # passes the same floor as `--target .2.28`. The musl wheel is + # built in the musllinux_1_2 container, a musl system. The FFI's musl + # binary is built in Alpine for the same reason. Both Darwin legs + # share macos-latest, an arm64 runner. So `target` is what makes + # darwin-x64 an x86_64 wheel, not a second arm64 wheel with the wrong + # name. include: - platform: linux-x64-gnu target: x86_64-unknown-linux-gnu @@ -229,13 +230,12 @@ jobs: native: true runs-on: ${{ matrix.os }} # A release build of the stack-encrypt graph, cross-compiled on two legs, - # on a 4 vCPU runner with no registry cache inside the container. Wider - # than the other jobs on purpose; tighten once a few runs have shown the - # real number. + # on a 4 vCPU runner with no registry cache inside the container. No run + # has measured it yet, so this is wider than the other jobs on purpose. timeout-minutes: 90 steps: - # Git for Windows defaults to CRLF on checkout. `cargo fmt` is not run - # here, but a CRLF pyproject.toml is what maturin would read. + # Git for Windows defaults to CRLF on checkout. `cargo fmt` does not + # run here, but maturin would read a CRLF pyproject.toml. - name: Keep LF line endings run: git config --global core.autocrlf false @@ -244,11 +244,11 @@ jobs: persist-credentials: false # One source of truth for the Rust version: the root mise.toml the - # `python` job builds under. mise itself is not installed here: the + # `python` job builds under. mise itself is not installed here. The # Linux legs compile inside a container, where maturin-action installs - # the toolchain this names. If that line changes shape, fail here by name - # rather than hand maturin-action an empty toolchain, which it would - # read as "stable". + # the toolchain this step names. If that line changes shape, this step + # fails by name. Otherwise maturin-action would receive an empty + # toolchain and read it as "stable". - name: Rust version from mise.toml id: rust run: | @@ -259,24 +259,24 @@ jobs: fi echo "version=$version" >> "$GITHUB_OUTPUT" - # The floor, not the newest: an abi3-py39 wheel claims to load on 3.9, - # and the import below is that claim tested. maturin needs no - # interpreter for an abi3 build, so this is for the import alone. - # "3.9" resolves to the newest 3.9.x the runner has a build for: 3.9.13 - # on arm64 macOS and on Windows, where the later security releases - # shipped no binaries (actions/python-versions manifest, 2026-10-27), - # and 3.9.25 on Linux and x64 macOS. + # The floor, not the newest. An abi3-py39 wheel claims to load on 3.9, + # and the import below tests that claim. maturin needs no interpreter + # for an abi3 build, so this is for the import alone. "3.9" resolves to + # the newest 3.9.x the runner has a build for. That is 3.9.13 on arm64 + # macOS and on Windows, where the later security releases shipped no + # binaries (actions/python-versions manifest, 2026-10-27). It is 3.9.25 + # on Linux and x64 macOS. - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 with: python-version: "3.9" - # The toolchain the wheel is built with, installed before the cache step - # so that step keys on it: rust-cache reads `rustc -vV` from the default - # toolchain, and without this it would key on whatever the runner image - # ships and never follow the version mise.toml names. maturin-action - # installs the same version again inside the Linux containers, where - # this install is not visible; `rustup toolchain install` is a no-op - # where it already is. + # The toolchain the wheel is built with. It is installed before the + # cache step so that step keys on it. rust-cache reads `rustc -vV` from + # the default toolchain. Without this step it would key on whatever the + # runner image ships and never follow the version mise.toml names. + # maturin-action installs the same version again inside the Linux + # containers, where this install is not visible. `rustup toolchain + # install` does nothing where the version is already installed. - name: Install the Rust toolchain mise.toml names env: RUST_VERSION: ${{ steps.rust.outputs.version }} @@ -285,18 +285,22 @@ jobs: rustup toolchain install "$RUST_VERSION" --profile minimal rustup default "$RUST_VERSION" - # Keyed by target as well as by lockfile, so the six legs do not evict - # each other. On the three host legs this caches the registry and the - # workspace `target/` directory. On the three container legs it is weak: - # the container keeps its own registry, and writes `target/` as root - # through the bind mount, so the post step may only warn. It stays on - # every leg because every leg compiles Rust (#1383). + # Keyed by target as well as by lockfile, so the legs do not evict each + # other. On the three host legs this caches the registry and the + # workspace `target/` directory. On the three container legs it caches + # little. The container keeps its own registry, and it writes `target/` + # as root through the bind mount, so the post step may only warn. It + # stays on every leg because every leg compiles Rust (#1383). - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 with: workspaces: languages/python key: ${{ matrix.target }} - - name: Start the clock + # The build time below is measured from here, so it includes everything + # maturin-action does. On the three Linux legs that is the container + # pull and the toolchain install inside it. On every leg it is the + # registry fetch on a cache miss, and the compile itself. + - name: Record the start time id: clock run: echo "start=$(date +%s)" >> "$GITHUB_OUTPUT" @@ -313,8 +317,8 @@ jobs: manylinux: ${{ matrix.manylinux }} rust-toolchain: ${{ steps.rust.outputs.version }} - # The build time is the number this job exists to record (the spec's - # scorecard asks for it per platform). Written here, to this leg's + # The build time is the number this job exists to record. The spec's + # scorecard asks for it per platform. It is written here, to this leg's # summary, and to a file the `wheels-summary` job gathers into one table. - name: Record the build time env: @@ -328,10 +332,10 @@ jobs: echo "Built the ${PLATFORM} wheel in ${seconds}s" >> "$GITHUB_STEP_SUMMARY" # Exactly one wheel, tagged for the floor (cp39-abi3) and for this leg's - # platform. A wrong `target` on the shared Darwin runner, or a container - # that produced a manylinux tag below the floor, both show up in the - # filename and nowhere else that this job reads. - - name: The wheel is one, abi3-py39, for this platform + # platform. A wrong `target` on the shared Darwin runner shows up in the + # filename. So does a container that produced a manylinux tag below the + # floor. This job reads the filename and nothing else about the wheel. + - name: Check that the wheel is one, abi3-py39, for this platform env: PLATFORM: ${{ matrix.platform }} WHEEL_TAG: ${{ matrix.wheel }} @@ -352,8 +356,8 @@ jobs: # On the legs whose runner is the wheel's platform, install it into the # floor interpreter and load both halves. The cross-compiled legs - # (linux-arm64-gnu, linux-x64-musl, darwin-x64) cannot run theirs here; - # for them the filename check above is the whole verification. + # (linux-arm64-gnu, linux-x64-musl, darwin-x64) cannot run theirs here. + # For them the filename check above is the whole verification. - name: Import the wheel on Python 3.9 if: ${{ matrix.native }} run: | @@ -375,12 +379,12 @@ jobs: if-no-files-found: error retention-days: 14 - # The six wheels in one place: counts them, and writes one table of build - # time per platform to the job summary. A missing wheel fails here by - # platform name, so a leg that uploaded nothing is not mistaken for one - # whose job was skipped. + # All the wheels in one place. The job counts them against the platform + # list and writes one table of build time per platform to the job summary. + # A missing wheel fails here by platform name. So a leg that uploaded + # nothing is not mistaken for a leg whose job was skipped. wheels-summary: - name: six wheels, build time per platform + name: one wheel per platform, build time per platform needs: [wheels] runs-on: blacksmith-4vcpu-ubuntu-2404 timeout-minutes: 10 @@ -391,11 +395,16 @@ jobs: merge-multiple: true path: wheels - - name: Six wheels, one per platform, and their build times + - name: One wheel per platform, and their build times run: | set -euo pipefail shopt -s nullglob + # A copy of the wheel matrix's `platform` values. This job cannot + # read the matrix. python-ci.test.mjs holds the copy to the + # original. The expected count comes from this list alone. platforms="linux-x64-gnu linux-arm64-gnu linux-x64-musl darwin-x64 darwin-arm64 win32-x64-msvc" + read -ra platform_list <<<"$platforms" + want=${#platform_list[@]} wheels=(wheels/*.whl) { echo "## Python wheels" @@ -404,7 +413,7 @@ jobs: echo "|---|---|---:|" } >> "$GITHUB_STEP_SUMMARY" missing=0 - for platform in $platforms; do + for platform in "${platform_list[@]}"; do timing="wheels/${platform}.seconds" if [ ! -f "$timing" ]; then echo "::error::no build time recorded for ${platform}" @@ -415,7 +424,7 @@ jobs: done echo "" >> "$GITHUB_STEP_SUMMARY" printf '%s\n' "${wheels[@]##*/}" | sed 's/^/- /' >> "$GITHUB_STEP_SUMMARY" - if [ "${#wheels[@]}" -ne 6 ] || [ "$missing" -ne 0 ]; then - echo "::error::expected six wheels with six build times, found ${#wheels[@]} wheels: ${wheels[*]##*/}" + if [ "${#wheels[@]}" -ne "$want" ] || [ "$missing" -ne 0 ]; then + echo "::error::expected ${want} wheels with ${want} build times, found ${#wheels[@]} wheels: ${wheels[*]##*/}" exit 1 fi diff --git a/AGENTS.md b/AGENTS.md index f33972804..2b732895c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -93,7 +93,7 @@ Every npm package except EQL lives under `languages/typescript/`: packages in `l - `packages/stack-auth`, `packages/stack-profile`, `packages/stack-kms`, `packages/stack-encrypt`, `packages/stack-encrypt-derive`, `packages/stack-guest-abi`: The Rust crates imported from `cipherstash/cipherstash-suite` with their history — `stack-auth` and `stack-profile` (published to crates.io, one version group), `stack-kms` (published to crates.io from 0.1.0, its own version group, re-exported by `stack-encrypt` as `stack_encrypt::kms`), `stack-encrypt` and `stack-encrypt-derive` (published to crates.io from 0.1.0, one version group; `eql-bindings`' `stack-encrypt` feature depends on them from the registry), and `stack-guest-abi` (`publish = false`). They are the members of the **root Cargo workspace**, with the three node binding crates below. See "Working on the Rust crates". - `languages/typescript/packages/auth`, `languages/typescript/packages/profile`, `languages/typescript/packages/stack-auth-wasm`: The node bindings of those crates. `@cipherstash/auth` (napi-rs v2) and its six `platforms/*` packages are published to npm from this repository by `release.yml` (`auth-artifacts`, `publish-auth`); a change to what it ships, the `stack-auth` crate included, needs an `@cipherstash/auth` changeset (`require-auth-npm-changeset.yml`). `@cipherstash/profile` and its platforms are private and never published; `@cipherstash/stack-auth-wasm` is private and builds the wasm that `@cipherstash/auth` ships. Their `build` and `test` scripts never invoke cargo; `build:native`, `build:debug` and `test:cargo` do. - `languages/golang`: The Go module — the SDK `encrypt` with its generated-code support `encrypt/gensupport` and the policy packages `encrypt/policy` and `encrypt/policy/protosource`; the credential package `auth`; the generator `stashgen` and its command `cmd/stashgen`; and `internal` (the shared guest plumbing and the `record` wire model). A wazero host with no cgo. Its two WASI guests (`encrypt/guest`, `auth/guest`) are detached Cargo workspaces built by `mise run wasm:guest:build` and `mise run wasm:auth-guest:build`; `mise run wasm:guest:build:deterministic` builds the seeded TEST build the hermetic Go tests use; the `.wasm` files they embed are gitignored. The stack-encrypt guest has a second build with the EQL types (`mise run wasm:guest:build:eql`, cargo feature `eql`, linking `eql-bindings` by path; `wasm:guest:build:eql:deterministic` is its test build), embedded by `encrypt/eql` and registered on import, so a program whose generated code names an EQL type runs it (ADR-0007, amended). `encrypt/eql`'s types and `Types` table are generated from the EQL catalog by `eql-codegen` (`eql_gen.go`, written by `mise run types:generate` in `packages/eql` and drift-gated by `types:check` and `cargo test -p eql-codegen`). Generated `*_stash.go` files are committed and CI fails when `go generate ./...` changes one. Releases are cut by hand with `.github/workflows/release-golang.yml` (`gh workflow run release-golang.yml -f version=X.Y.Z[-pre] --ref `): because the guests are gitignored on main and Go fetches a version as the files committed at its tag, the workflow builds them, commits them on a release-only commit no branch points at, tags it `languages/golang/vX.Y.Z` and pushes the tag alone, then attaches the EQL v4 SQL bundle to the GitHub release. The ref must carry the EQL v4 bundle, or the build stops at "Build the EQL SQL bundles". `scripts/go-release-version.mjs` refuses, before anything is built, every version Go would refuse (non-canonical semver, `+build` metadata, a v2+ major on a module path with no `/vN` suffix, a version shaped like a pseudo-version), a final version dispatched from anywhere but `main`, and a final version while `languages/golang/go.mod` pins a `github.com/cipherstash/vitaminc/…` module at a pseudo-version (vitaminc's Go bindings have no tags, and a TEMPORARY pin on an unreleased commit must be gone before the module is tagged); a pre-release may come from a branch and may carry such a pin. The build job also refuses a version the Go proxy already lists in `@v/list`, which catches a tag deleted after the proxy fetched it; it never asks the proxy for the version itself before the tag exists, because the proxy would cache "not found" for up to 30 minutes and `verify` would fail after the push. The build checks the `Cargo.lock` of the EQL workspace and both guest workspaces with `--locked` first, since none of those builds passes it. It runs in a job with no write token and hands the guests to a `publish` job that holds `contents: write` and runs only git and gh. `publish` fetches the commit with plain unauthenticated git rather than `actions/checkout`, whose `token` input defaults to the job's writable token, and refuses a release commit whose tree is not the one the build tested. `publish` takes the paths it commits from the workflow's fixed `GUESTS` list, never from the artifact, and refuses a `guests.sha256` that is not exactly one SHA-256 line for each of those three guests: the build job runs third-party code after writing that manifest, so a path read from it could put any file on the tag (`scripts/__tests__/go-release-artifact.test.mjs`). A version is immutable once the Go proxy has fetched it, so a tag is never moved or deleted: if `publish` or `verify` fails after the tag is pushed, use "Re-run failed jobs" on the same run, which keeps a tag that run pushed and creates or completes the release; a new dispatch of the same version is refused. Never tag a main commit by hand: `go get` would fetch a module with no guests. -- `languages/python`: The Python SDK, `stackencrypt`, a pyo3 extension over `stack-encrypt`, built by maturin. **The fourth Cargo workspace**, excluded from the root one. It has two crates. `crates/core` has no pyo3: the engine-side logic, linked and tested without a Python interpreter. `crates/engine` is the `_engine` cdylib, `abi3-py39`, one wheel per platform for Python 3.9 to 3.14. The `extension-module` feature is set only by maturin in `pyproject.toml`, never in a crate manifest, so `cargo test` links libpython. Its tasks live in its own `mise.toml` and run from that folder. `mise run build` builds the wheel into `.venv`, `mise run test` runs pytest, and `mise run test:cargo` tests both crates with `--locked` and libpython on the loader path. The wheel's version is `crates/engine/Cargo.toml`'s. `stackencrypt.__engine__` is the commit of this checkout, recorded by `crates/core/build.rs`. Its `Cargo.lock` is seeded from the root one, so every crate it shares with the root workspace resolves to the version the root tests. After a root dependency moves, re-seed it the same way: copy the root `Cargo.lock` over it and run `cargo metadata --offline` there. That command prunes the copy to what the workspace uses. Until then, `tests-python.yml` fails on `--locked`. Nothing is published. CI is `tests-python.yml`. It runs the Rust checks and pytest on Python 3.9 and 3.14, where `MISE_PYTHON_VERSION` moves a leg off the local version. It then builds one abi3 wheel per FFI platform, uploads them as artifacts, and writes each build time to the job summary. `scripts/__tests__/python-ci.test.mjs` compares its path filter with the workspace's `Cargo.lock` and its wheel matrix with the protect-ffi platform packages. There is no Python lockfile yet: the package has no runtime dependency, so only pytest could be locked. Design: Python ADR-0001 under `languages/python/docs/adr/`, the spec in #1341. +- `languages/python`: The Python SDK, `stackencrypt`, a pyo3 extension over `stack-encrypt`, built by maturin. **The fourth Cargo workspace**, excluded from the root one. It has two crates. `crates/core` has no pyo3: the engine-side logic, linked and tested without a Python interpreter. `crates/engine` is the `_engine` cdylib, `abi3-py39`, one wheel per platform for Python 3.9 to 3.14. The `extension-module` feature is set only by maturin in `pyproject.toml`, never in a crate manifest, so `cargo test` links libpython. Its tasks live in its own `mise.toml` and run from that folder. `mise run build` builds the wheel into `.venv`. `mise run test` runs pytest. `mise run test:cargo` runs plain `cargo test --locked` for both crates, with libpython on the loader path. It does not use nextest, and its CI job sets no `MISE_ENV`. The nextest rule under "Working on the Rust crates" is for the root workspace, whose `stack-kms` tests set environment variables. No test in this workspace does, and nextest comes from `mise.test.toml`, which this workspace's CI does not load. The wheel's version is `crates/engine/Cargo.toml`'s. `stackencrypt.__engine__` is the commit of this checkout, recorded by `crates/core/build.rs`. Its `Cargo.lock` is seeded from the root one, so every crate it shares with the root workspace resolves to the version the root tests. After a root dependency moves, re-seed it the same way: copy the root `Cargo.lock` over it and run `cargo metadata --offline` there. That command prunes the copy to what the workspace uses. Until then, `tests-python.yml` fails on `--locked`. Nothing is published. CI is `tests-python.yml`. It runs the Rust checks and pytest on Python 3.9 and 3.14, where `MISE_PYTHON_VERSION` moves a leg off the local version. It then builds one abi3 wheel per FFI platform, uploads them as artifacts, and writes each build time to the job summary. `scripts/__tests__/python-ci.test.mjs` compares its path filter with the workspace's `Cargo.lock` and its wheel matrix with the protect-ffi platform packages. There is no Python lockfile yet: the package has no runtime dependency, so only pytest could be locked. Design: Python ADR-0001 under `languages/python/docs/adr/`, the spec in #1341. - `e2e/*`: Cross-package end-to-end tests (package managers, supply chain, Prisma example README) - `languages/typescript/examples/*`: Working apps (basic, prisma, supabase-worker) - `docs/plans/*`: Internal design plans. User-facing documentation lives at https://cipherstash.com/docs (not in this repo). diff --git a/scripts/__tests__/claude-review-workflow.test.mjs b/scripts/__tests__/claude-review-workflow.test.mjs index 631a9be21..7e88f78d7 100644 --- a/scripts/__tests__/claude-review-workflow.test.mjs +++ b/scripts/__tests__/claude-review-workflow.test.mjs @@ -1,13 +1,13 @@ import { readFileSync } from 'node:fs' import { describe, expect, it } from 'vitest' -import { readWorkflow } from './lib/workflows.mjs' +import { readWorkflow, triggers } from './lib/workflows.mjs' const WORKFLOW = '.github/workflows/claude-review.yml' const ACTION_SHA = 'bf38e86e58df9ebf3420326d019f955bb3be64dd' const gha = (expression) => `\${{ ${expression} }}` const workflow = readWorkflow(WORKFLOW) -const triggers = workflow.on ?? workflow[true] +const on = triggers(workflow) const review = workflow.jobs.review const claude = review.steps.find((step) => String(step.uses ?? '').startsWith('anthropics/claude-code-action@'), @@ -19,14 +19,14 @@ describe('Claude pull-request review', () => { }) it('reviews every agreed pull-request lifecycle event', () => { - expect(Object.keys(triggers)).toEqual(['pull_request']) - expect(triggers.pull_request.types).toEqual([ + expect(Object.keys(on)).toEqual(['pull_request']) + expect(on.pull_request.types).toEqual([ 'opened', 'synchronize', 'ready_for_review', 'reopened', ]) - expect(triggers.pull_request['paths-ignore']).toEqual([ + expect(on.pull_request['paths-ignore']).toEqual([ '.changeset/**', '**/__snapshots__/**', '**/*.snap', diff --git a/scripts/__tests__/code-review-workflow.test.mjs b/scripts/__tests__/code-review-workflow.test.mjs index 3b09436be..49164e438 100644 --- a/scripts/__tests__/code-review-workflow.test.mjs +++ b/scripts/__tests__/code-review-workflow.test.mjs @@ -11,13 +11,13 @@ import { tmpdir } from 'node:os' import { join } from 'node:path' import { afterEach, beforeEach, describe, expect, it } from 'vitest' import { runsWhen } from './lib/expressions.mjs' -import { readWorkflow } from './lib/workflows.mjs' +import { readWorkflow, triggers } from './lib/workflows.mjs' const WORKFLOW = '.github/workflows/code-review.yml' const gha = (expression) => `\${{ ${expression} }}` const workflow = readWorkflow(WORKFLOW) -const triggers = workflow.on ?? workflow[true] +const on = triggers(workflow) const request = workflow.jobs.request const fire = request.steps.find( (step) => step.name === 'Fire the code-review routine', @@ -32,8 +32,8 @@ describe('Code-review routine trigger', () => { it('fires once per lifecycle event, never on every push', () => { // No `synchronize`: each push would start another review session. Mark the // pull request draft and ready again for a fresh review. - expect(Object.keys(triggers)).toEqual(['pull_request']) - expect(triggers.pull_request.types).toEqual([ + expect(Object.keys(on)).toEqual(['pull_request']) + expect(on.pull_request.types).toEqual([ 'opened', 'reopened', 'ready_for_review', diff --git a/scripts/__tests__/crates-ci.test.mjs b/scripts/__tests__/crates-ci.test.mjs index 8b3a64bb3..6946d3fa1 100644 --- a/scripts/__tests__/crates-ci.test.mjs +++ b/scripts/__tests__/crates-ci.test.mjs @@ -3,7 +3,12 @@ import { existsSync, readdirSync, readFileSync } from 'node:fs' import { join } from 'node:path' import { describe, expect, it } from 'vitest' import { REPO_ROOT } from './lib/repo-root.mjs' -import { readWorkflow, WORKFLOW_DIR, workflowFiles } from './lib/workflows.mjs' +import { + readWorkflow, + triggers, + WORKFLOW_DIR, + workflowFiles, +} from './lib/workflows.mjs' /** * Guards that the stack-* crates, their node bindings and the Go module are @@ -380,8 +385,6 @@ describe('the imported workflow directory is gone', () => { // The silent failures specific to this set // --------------------------------------------------------------------------- -const triggers = (wf) => wf?.on ?? wf?.[true] ?? {} - /** Every step of every root workflow, with its job and effective env. */ const STEPS = ROOT_WORKFLOWS.flatMap(({ relPath, wf }) => Object.entries(wf?.jobs ?? {}).flatMap(([jobName, job]) => diff --git a/scripts/__tests__/eql-suite-ci.test.mjs b/scripts/__tests__/eql-suite-ci.test.mjs index 33ef5569a..c0e14ad30 100644 --- a/scripts/__tests__/eql-suite-ci.test.mjs +++ b/scripts/__tests__/eql-suite-ci.test.mjs @@ -4,7 +4,12 @@ import { dirname, join, relative, sep } from 'node:path' import yaml from 'js-yaml' import { describe, expect, it } from 'vitest' import { REPO_ROOT } from './lib/repo-root.mjs' -import { readWorkflow, WORKFLOW_DIR, workflowFiles } from './lib/workflows.mjs' +import { + readWorkflow, + runWorkingDirectory, + WORKFLOW_DIR, + workflowFiles, +} from './lib/workflows.mjs' /** * Guards that the EQL suite is RUN, not merely present. @@ -76,17 +81,17 @@ const SUITE_TASKS = [ * names all three suite tasks, and so does a `paths:` filter, which is a * trigger and not a run. * - * Only the steps whose working directory is under `packages/eql`, because a - * mise task name is scoped to the config mise finds from the current - * directory, and nothing else. `mise run test` from `languages/python` runs - * THAT workspace's `test` task; read without the directory it counted as - * EQL's `test`, which depends on `test:sqlx` and `test:lint`, and the two - * exemptions below went stale the day `tests-python.yml` landed — a false - * "CI runs this" in the direction this file exists to rule out. The - * directory is the step's own, else the job's `defaults.run`, else the - * workflow's; AGENTS.md records that every EQL task must run that way, and - * every EQL workflow does. A composite action's bodies are appended by the - * caller unscoped, as before: an action is reached by `uses:`, which has no + * Only the steps whose working directory is under `packages/eql`. A mise + * task name is scoped to the config mise finds from the current directory, + * and to nothing else. `mise run test` from `languages/python` runs THAT + * workspace's `test` task. Read without the directory, it counted as EQL's + * `test`, and EQL's `test` depends on `test:sqlx` and `test:lint`. So the + * two exemptions below were wrong from the first commit of + * `tests-python.yml`: a false "CI runs this", in the direction this file + * exists to rule out. The directory is resolved by `runWorkingDirectory`. + * AGENTS.md records that every EQL task must run that way, and every EQL + * workflow does. A composite action's bodies are appended by the caller + * unscoped, as before. An action is reached by `uses:`, and `uses:` has no * working directory of its own. */ const EQL_DIR = EQL_PREFIX.slice(0, -1) @@ -94,14 +99,11 @@ const insideEql = (dir) => dir !== undefined && (String(dir) === EQL_DIR || String(dir).startsWith(EQL_PREFIX)) function eqlRunBodies(wf) { - const workflowDefault = wf?.defaults?.run?.['working-directory'] - return Object.values(wf?.jobs ?? {}).flatMap((job) => { - const jobDefault = - job?.defaults?.run?.['working-directory'] ?? workflowDefault - return (Array.isArray(job?.steps) ? job.steps : []) - .filter((step) => insideEql(step?.['working-directory'] ?? jobDefault)) - .map((step) => String(step?.run ?? '').replace(/^[ \t]*#.*$/gm, '')) - }) + return Object.values(wf?.jobs ?? {}).flatMap((job) => + (Array.isArray(job?.steps) ? job.steps : []) + .filter((step) => insideEql(runWorkingDirectory(wf, job, step))) + .map((step) => String(step?.run ?? '').replace(/^[ \t]*#.*$/gm, '')), + ) } /** The two spellings GitHub accepts for a composite action's manifest. */ diff --git a/scripts/__tests__/eql-workflow-filters.test.mjs b/scripts/__tests__/eql-workflow-filters.test.mjs index 5f230772d..c47b242f6 100644 --- a/scripts/__tests__/eql-workflow-filters.test.mjs +++ b/scripts/__tests__/eql-workflow-filters.test.mjs @@ -4,7 +4,7 @@ import { join } from 'node:path' import yaml from 'js-yaml' import { describe, expect, it } from 'vitest' import { REPO_ROOT } from './lib/repo-root.mjs' -import { readWorkflow } from './lib/workflows.mjs' +import { readWorkflow, triggers } from './lib/workflows.mjs' /** * The two EQL workflows decide what to run from a list of path globs, and both @@ -59,12 +59,6 @@ import { readWorkflow } from './lib/workflows.mjs' const TEST_EQL = '.github/workflows/test-eql.yml' const BENCH_EQL = '.github/workflows/bench-eql.yml' -/** `on:` parses as the boolean `true` under YAML 1.1 — the "Norway problem". */ -function triggers(wf) { - const on = wf?.on ?? wf?.[true] - return on && typeof on === 'object' ? on : {} -} - /** A workflow's `on.push.paths` list, or `[]`. */ function pushPaths(relPath) { const paths = triggers(readWorkflow(relPath))?.push?.paths diff --git a/scripts/__tests__/lib/workflows.mjs b/scripts/__tests__/lib/workflows.mjs index e8f933176..6cfef5281 100644 --- a/scripts/__tests__/lib/workflows.mjs +++ b/scripts/__tests__/lib/workflows.mjs @@ -48,8 +48,36 @@ export function workflowFiles() { * both kinds with the same reader. * * Note for callers: `on:` parses as the boolean `true` under YAML 1.1 (the - * "Norway problem"), so read triggers as `wf.on ?? wf[true]`. + * "Norway problem"), so read triggers with `triggers(wf)` below. */ export function readWorkflow(relPath) { return yaml.load(readFileSync(join(REPO_ROOT, relPath), 'utf8')) } + +/** + * The triggers of a parsed workflow, as an object. `on:` parses as the boolean + * `true` under YAML 1.1 (the "Norway problem"), so the key is `true` unless + * the file quoted it. A workflow with no usable `on:` gives `{}`, so a caller + * can read `.pull_request` or test `'workflow_dispatch' in` without a guard. + * `workflow_dispatch:` with no value parses as null, so membership, not + * truthiness, is what says a trigger is declared. Before this helper, eight + * guards each wrote their own `wf.on ?? wf[true]`. + */ +export function triggers(wf) { + const on = wf?.on ?? wf?.[true] + return on && typeof on === 'object' ? on : {} +} + +/** + * The directory a step's `run:` executes in. GitHub takes the step's own + * `working-directory`, else the job's `defaults.run.working-directory`, else + * the workflow's. The result is `undefined` when none is set, and GitHub then + * runs the step at the checkout root. + */ +export function runWorkingDirectory(wf, job, step) { + return ( + step?.['working-directory'] ?? + job?.defaults?.run?.['working-directory'] ?? + wf?.defaults?.run?.['working-directory'] + ) +} diff --git a/scripts/__tests__/lint-release-scope.test.mjs b/scripts/__tests__/lint-release-scope.test.mjs index fa9f67142..32b3330fc 100644 --- a/scripts/__tests__/lint-release-scope.test.mjs +++ b/scripts/__tests__/lint-release-scope.test.mjs @@ -2,7 +2,12 @@ import { readFileSync } from 'node:fs' import { join } from 'node:path' import { describe, expect, it } from 'vitest' import { REPO_ROOT } from './lib/repo-root.mjs' -import { readWorkflow, WORKFLOW_DIR, workflowFiles } from './lib/workflows.mjs' +import { + readWorkflow, + triggers, + WORKFLOW_DIR, + workflowFiles, +} from './lib/workflows.mjs' /** * `lint-release.yml` states its scope TWICE — once as the `pull_request` paths @@ -23,10 +28,7 @@ import { readWorkflow, WORKFLOW_DIR, workflowFiles } from './lib/workflows.mjs' const LINT_RELEASE = `${WORKFLOW_DIR}/lint-release.yml` const workflow = readWorkflow(LINT_RELEASE) -// `on:` parses as the boolean `true` under YAML 1.1 — the Norway problem. -const triggers = workflow.on ?? workflow[true] - -const filtered = triggers.pull_request.paths.filter((path) => +const filtered = triggers(workflow).pull_request.paths.filter((path) => path.startsWith(`${WORKFLOW_DIR}/`), ) diff --git a/scripts/__tests__/python-ci.test.mjs b/scripts/__tests__/python-ci.test.mjs index 208ca8c51..a7903f580 100644 --- a/scripts/__tests__/python-ci.test.mjs +++ b/scripts/__tests__/python-ci.test.mjs @@ -3,7 +3,11 @@ import { join } from 'node:path' import { describe, expect, it } from 'vitest' import { filterCovers } from './lib/paths-filter.mjs' import { REPO_ROOT } from './lib/repo-root.mjs' -import { readWorkflow } from './lib/workflows.mjs' +import { + readWorkflow, + runWorkingDirectory, + triggers, +} from './lib/workflows.mjs' /** * `tests-python.yml` is held to the Python workspace it tests, with every @@ -15,9 +19,9 @@ import { readWorkflow } from './lib/workflows.mjs' * pyo3, and `tests.yml`'s gate excludes those crates on purpose. So a * stack-encrypt change that breaks the Python engine crate is caught here or * nowhere, and "here" depends on the `pull_request` filter naming every tree - * the workspace compiles. That list is the workspace's path dependencies, - * which `Cargo.lock` records as packages with no `source`: a new path - * dependency lands in the lockfile, and this file fails until the filter + * the workspace compiles. That list is the workspace's path dependencies. + * `Cargo.lock` records those as packages with no `source`. A new path + * dependency appears in the lockfile, and this file fails until the filter * follows. * * The other facts are read the same way: the two Python versions from @@ -32,12 +36,6 @@ const PYTHON_ROOT = 'languages/python' const read = (rel) => readFileSync(join(REPO_ROOT, rel), 'utf8') -/** `wf.on`, under YAML 1.1's reading of the bare key `on` as `true`. */ -function triggers() { - const wf = readWorkflow(WORKFLOW) - return wf.on ?? wf[true] -} - /** * The directories outside `languages/python` whose crates the workspace * compiles: every `[[package]]` in its Cargo.lock without a `source` line is a @@ -72,7 +70,7 @@ const coversTree = (pattern, file) => describe('tests-python.yml', () => { it('runs on pull requests touching every tree the Python workspace compiles', () => { - const on = triggers() + const on = triggers(readWorkflow(WORKFLOW)) const paths = on.pull_request?.paths ?? [] const wanted = [ PYTHON_ROOT, @@ -124,10 +122,12 @@ describe('tests-python.yml', () => { }) it('builds one wheel per platform the FFI ships for, by the same Rust triples', () => { - // The six platform packages each carry `neon.rust`, the triple their - // binary is built for. The issue names "the six platforms the FFI already - // ships for", so the wheel matrix is held to that list rather than to a - // copy of it: a seventh platform package, or a moved triple, fails here. + // Each platform package carries `neon.rust`, the triple its binary is + // built for. The issue names "the six platforms the FFI already ships + // for", so the wheel matrix is held to that list, not to a copy of it. A + // seventh platform package, or a moved triple, fails here. The count six + // is written here and nowhere else: the workflow derives its own count + // from the platform list in `wheels-summary`. const platformsDir = 'languages/typescript/packages/protect-ffi/platforms' const ffi = readdirSync(join(REPO_ROOT, platformsDir)) .map((name) => { @@ -203,16 +203,11 @@ describe('tests-python.yml', () => { ) const wf = readWorkflow(WORKFLOW) - const workflowDefault = wf.defaults?.run?.['working-directory'] - const runs = Object.values(wf.jobs).flatMap((job) => { - const jobDefault = - job.defaults?.run?.['working-directory'] ?? workflowDefault - return job.steps - .filter( - (step) => (step['working-directory'] ?? jobDefault) === PYTHON_ROOT, - ) - .map((step) => step.run ?? '') - }) + const runs = Object.values(wf.jobs).flatMap((job) => + job.steps + .filter((step) => runWorkingDirectory(wf, job, step) === PYTHON_ROOT) + .map((step) => step.run ?? ''), + ) const invoked = (command) => runs.some((run) => new RegExp(`(^|[\\s;&|])${command}(\\s|$)`, 'm').test(run), diff --git a/scripts/__tests__/workflow-dispatch-job-conditions.test.mjs b/scripts/__tests__/workflow-dispatch-job-conditions.test.mjs index 3b02b86e7..33c6854ac 100644 --- a/scripts/__tests__/workflow-dispatch-job-conditions.test.mjs +++ b/scripts/__tests__/workflow-dispatch-job-conditions.test.mjs @@ -5,7 +5,7 @@ import { UnsupportedExpression, unwrap, } from './lib/expressions.mjs' -import { readWorkflow, workflowFiles } from './lib/workflows.mjs' +import { readWorkflow, triggers, workflowFiles } from './lib/workflows.mjs' /** * A workflow that declares `workflow_dispatch:` must actually run when someone @@ -299,16 +299,6 @@ const EVENT_OF = { // Discovery // --------------------------------------------------------------------------- -/** - * `on:` parses as the boolean `true` under YAML 1.1 (the "Norway problem"), - * hence the two keys. `workflow_dispatch:` with no value parses as null, so - * membership — not truthiness — is what says the trigger is declared. - */ -function triggers(wf) { - const on = wf?.on ?? wf?.[true] - return on && typeof on === 'object' ? on : {} -} - function declaresDispatch(wf) { return 'workflow_dispatch' in triggers(wf) }