From 82656acfb8015679d0f3c46a66f77557963337d8 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Mon, 12 Oct 2026 19:12:42 +0400 Subject: [PATCH 1/6] Add rules registry tool Standard-library tool (scripts/rules) that loads rules/registry.toml, checks each rule's enforcer and CI wiring, finds unregistered checks, compares ratchet baselines with the base branch, and renders the human and agent views. Byte-identical with policyengine-api; see scripts/rules/SYNC.md. Co-Authored-By: Claude Opus 5.5 --- scripts/rules/SYNC.md | 19 + scripts/rules/__init__.py | 12 + scripts/rules/__main__.py | 5 + scripts/rules/baseline.py | 82 +++ scripts/rules/cli.py | 54 ++ scripts/rules/discover.py | 164 ++++++ scripts/rules/guard.py | 124 +++++ scripts/rules/load.py | 353 ++++++++++++ scripts/rules/model.py | 192 +++++++ scripts/rules/render.py | 271 +++++++++ scripts/rules/resolve.py | 402 ++++++++++++++ scripts/rules/tests/__init__.py | 0 .../tests/fixtures/policyengine_api_pr.yml | 201 +++++++ .../fixtures/policyengine_sim_api_pr.yml | 179 ++++++ scripts/rules/tests/support.py | 232 ++++++++ scripts/rules/tests/test_guard.py | 518 ++++++++++++++++++ scripts/rules/tests/test_workflows.py | 80 +++ scripts/rules/validate.py | 319 +++++++++++ scripts/rules/workflows.py | 191 +++++++ 19 files changed, 3398 insertions(+) create mode 100644 scripts/rules/SYNC.md create mode 100644 scripts/rules/__init__.py create mode 100644 scripts/rules/__main__.py create mode 100644 scripts/rules/baseline.py create mode 100644 scripts/rules/cli.py create mode 100644 scripts/rules/discover.py create mode 100644 scripts/rules/guard.py create mode 100644 scripts/rules/load.py create mode 100644 scripts/rules/model.py create mode 100644 scripts/rules/render.py create mode 100644 scripts/rules/resolve.py create mode 100644 scripts/rules/tests/__init__.py create mode 100644 scripts/rules/tests/fixtures/policyengine_api_pr.yml create mode 100644 scripts/rules/tests/fixtures/policyengine_sim_api_pr.yml create mode 100644 scripts/rules/tests/support.py create mode 100644 scripts/rules/tests/test_guard.py create mode 100644 scripts/rules/tests/test_workflows.py create mode 100644 scripts/rules/validate.py create mode 100644 scripts/rules/workflows.py diff --git a/scripts/rules/SYNC.md b/scripts/rules/SYNC.md new file mode 100644 index 000000000..4a3cf44ef --- /dev/null +++ b/scripts/rules/SYNC.md @@ -0,0 +1,19 @@ +# Keeping `scripts/rules/` in sync + +This directory is byte-identical in two repositories: + +- `PolicyEngine/policyengine-api` (canonical copy) +- `PolicyEngine/policyengine-sim-api` + +When you change it: + +1. Make the change in policyengine-api first. +2. Copy the whole directory to policyengine-sim-api unchanged. +3. Bump `TOOL_VERSION` in `__init__.py`, and `tool_version` in each repository's + `rules/registry.toml`. +4. Run `python -m scripts.rules render` in both repositories. The header of + `docs/generated/rules.md` shows a hash of the tool's source, so reviewers can + confirm both copies match. + +Repository-specific content belongs in `rules/registry.toml`, never in this +directory. diff --git a/scripts/rules/__init__.py b/scripts/rules/__init__.py new file mode 100644 index 000000000..09bd68688 --- /dev/null +++ b/scripts/rules/__init__.py @@ -0,0 +1,12 @@ +"""Rules registry tooling. + +The registry in ``rules/registry.toml`` is the single list of rules that govern +this repository. This package validates it, checks that every rule is really +enforced where it claims to be, and renders the human- and agent-facing views. + +This directory is kept byte-identical across policyengine-api and +policyengine-sim-api; see ``SYNC.md``. It must stay standard-library only so it +can run in CI jobs that install no dependencies. +""" + +TOOL_VERSION = "0.1.0" diff --git a/scripts/rules/__main__.py b/scripts/rules/__main__.py new file mode 100644 index 000000000..96e63b9b7 --- /dev/null +++ b/scripts/rules/__main__.py @@ -0,0 +1,5 @@ +"""Entry point for ``python -m scripts.rules``.""" + +from scripts.rules.cli import main + +raise SystemExit(main()) diff --git a/scripts/rules/baseline.py b/scripts/rules/baseline.py new file mode 100644 index 000000000..9728f904b --- /dev/null +++ b/scripts/rules/baseline.py @@ -0,0 +1,82 @@ +"""Compare ratchets against the base branch: baselines may only shrink.""" + +from __future__ import annotations + +import os +import subprocess +from pathlib import Path + +from scripts.rules.load import REGISTRY_PATH, load_text +from scripts.rules.model import Registry, Status, Violation + + +class BaseUnavailable(Exception): + """The base ref could not be read.""" + + +def default_base_ref() -> str | None: + base = os.environ.get("GITHUB_BASE_REF") + return f"origin/{base}" if base else None + + +def load_base_registry(root: Path, ref: str) -> Registry | None: + """The registry as it is on ``ref``, or None if the base has no registry yet.""" + verify = subprocess.run( + ["git", "rev-parse", "--verify", "--quiet", f"{ref}^{{commit}}"], + cwd=root, + capture_output=True, + text=True, + ) + if verify.returncode != 0: + raise BaseUnavailable(ref) + shown = subprocess.run( + ["git", "show", f"{ref}:{REGISTRY_PATH.as_posix()}"], + cwd=root, + capture_output=True, + text=True, + ) + if shown.returncode != 0: + return None + registry, _ = load_text(shown.stdout) + return registry + + +def check_baselines( + current: Registry, base: Registry +) -> tuple[list[Violation], list[str]]: + """Return (violations, notes).""" + violations: list[Violation] = [] + notes: list[str] = [] + current_ids = {rule.id for rule in current.rules} + for old in base.rules: + new = current.rule(old.id) + if new is None: + notes.append(f"rule {old.id} was removed") + continue + if old.baseline and new.baseline and new.baseline.count > old.baseline.count: + violations.append( + Violation( + new.id, + "baseline", + f"baseline grew from {old.baseline.count} to {new.baseline.count}", + "fix the new violations instead of adding them to the baseline", + ) + ) + weaker = (Status.ADVISORY, Status.DOC_ONLY) + if old.status in (Status.ENFORCED, Status.RATCHETED) and new.status in weaker: + if new.downgrade_reason: + notes.append( + f"rule {new.id} downgraded to {new.status}: {new.downgrade_reason}" + ) + else: + violations.append( + Violation( + new.id, + "baseline", + f"status weakened from {old.status} to {new.status}", + 'add [rule.downgrade] reason = "..." so the downgrade is explicit and reviewable', + ) + ) + for rule_id in sorted(current_ids - {rule.id for rule in base.rules}): + notes.append(f"rule {rule_id} is new") + return violations, notes diff --git a/scripts/rules/cli.py b/scripts/rules/cli.py new file mode 100644 index 000000000..e5fd94fda --- /dev/null +++ b/scripts/rules/cli.py @@ -0,0 +1,54 @@ +"""Command line: ``python -m scripts.rules {validate,render,guard}``.""" + +from __future__ import annotations + +import argparse +from datetime import date +from pathlib import Path + +from scripts.rules.guard import REPO_ROOT, run_guard +from scripts.rules.load import load_registry +from scripts.rules.render import write_outputs +from scripts.rules.resolve import clear_caches +from scripts.rules.validate import validate + + +def _print(violations: list, notes: list[str] | None = None) -> int: + for note in notes or []: + print(f"note: {note}") + for violation in violations: + print(f" - {violation}") + print("rules: failed" if violations else "rules: passed") + return 1 if violations else 0 + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + prog="python -m scripts.rules", description=__doc__ + ) + parser.add_argument("command", choices=["validate", "render", "guard"]) + parser.add_argument("--root", type=Path, default=REPO_ROOT, help="repository root") + parser.add_argument( + "--base-ref", + help="git ref to compare baselines against (default: origin/$GITHUB_BASE_REF)", + ) + args = parser.parse_args(argv) + root = args.root.resolve() + + if args.command == "guard": + result = run_guard(root, base_ref=args.base_ref) + return _print(result.violations, result.notes) + + clear_caches() + registry, violations = load_registry(root) + if registry is None: + return _print(violations) + violations += validate(registry, root, date.today()) + if args.command == "render": + if violations: + print("render refused: fix the registry first") + return _print(violations) + for path in write_outputs(registry, root): + print(f"wrote {path.relative_to(root)}") + return 0 + return _print(violations) diff --git a/scripts/rules/discover.py b/scripts/rules/discover.py new file mode 100644 index 000000000..c7c7e42d9 --- /dev/null +++ b/scripts/rules/discover.py @@ -0,0 +1,164 @@ +"""Find rules that are enforced somewhere but missing from the registry. + +A check that isn't registered is a hidden rule: nobody can see it in +``rules.md`` and agents aren't told about it. This scans the places checks live +today and requires each one to be referenced by a rule, or ignored with a +reason in ``[discovery]``. +""" + +from __future__ import annotations + +from pathlib import Path + +from scripts.rules.model import Kind, Registry, Violation +from scripts.rules.resolve import ( + DEFAULT_RUNNER, + guard_names, + mypy_files, + root_relative_nodes, + ruff_lint_config, + split_node, + test_names, + workflow_jobs, +) + + +def _hidden(message: str, fix: str) -> Violation: + return Violation(None, "hidden-rule", message, fix) + + +def _registered_nodes(registry: Registry) -> set[str]: + nodes = set() + for rule in registry.rules: + if rule.enforcer.kind is Kind.PYTEST: + for node in root_relative_nodes(rule): + path, name = split_node(node) + nodes.add(f"{path}::{name}") + if rule.proof is not None and isinstance(rule.proof.fields.get("test"), str): + project = rule.proof.fields.get("project") + prefix = f"{project.rstrip('/')}/" if isinstance(project, str) else "" + path, name = split_node(prefix + rule.proof.fields["test"]) + nodes.add(f"{path}::{name}") + return nodes + + +def _discovered_tests(registry: Registry, root: Path) -> set[str]: + found = set() + for pattern in registry.discovery.guard_test_globs: + for path in sorted(root.glob(pattern)): + names = test_names(path) + relative = path.relative_to(root).as_posix() + for name in names or (): + found.add(f"{relative}::{name}") + for node in registry.discovery.extra_guard_tests: + path, name = split_node(node) + found.add(f"{path}::{name}") + return found + + +def discover_hidden(registry: Registry, root: Path) -> list[Violation]: + violations = [] + rules = registry.rules + + # Quality guards registered in the runner. + names = guard_names(root, DEFAULT_RUNNER) or [] + registered = { + r.enforcer.get("name") for r in rules if r.enforcer.kind is Kind.QUALITY_GUARD + } + for name in names: + if name not in registered: + violations.append( + _hidden( + f"quality guard '{name}' in {DEFAULT_RUNNER} has no rule", + 'add a [[rule]] with enforcer.kind = "quality-guard"', + ) + ) + + # Ruff: format checks in CI, explicitly selected codes, per-file ignores. + ruff_rules = [r for r in rules if r.enforcer.kind is Kind.RUFF] + runs_format = any( + "ruff format" in job.run_text + for workflow in registry.discovery.workflows + for job in workflow_jobs(root / workflow)[0].values() + ) + if runs_format and not any(r.enforcer.get("mode") == "format" for r in ruff_rules): + violations.append( + _hidden( + "CI runs 'ruff format' but no rule describes it", + "add a ruff format rule", + ) + ) + lint = ruff_lint_config(root) + lint_codes = { + c + for r in ruff_rules + if r.enforcer.get("mode") == "lint" + for c in r.enforcer.get("codes", []) + } + for code in list(lint.get("select", [])) + list(lint.get("extend-select", [])): + if code not in lint_codes: + violations.append( + _hidden( + f"ruff selects '{code}' but no rule describes it", + "add the code to a ruff lint rule", + ) + ) + ignore_patterns = { + r.enforcer.get("pattern") + for r in ruff_rules + if r.enforcer.get("mode") == "per-file-ignore" + } + for pattern in lint.get("per-file-ignores", {}): + if pattern not in ignore_patterns: + violations.append( + _hidden( + f"ruff per-file-ignores '{pattern}' has no rule", + "add a ruff per-file-ignore rule explaining the exemption", + ) + ) + + # mypy scope. + covered = { + p + for r in rules + if r.enforcer.kind is Kind.MYPY + for p in r.enforcer.get("paths", []) + } + for path in mypy_files(root): + if path not in covered: + violations.append( + _hidden( + f"mypy checks '{path}' but no rule describes it", + "add it to a mypy rule's paths", + ) + ) + + # Guard-style tests. + registered_nodes = _registered_nodes(registry) + ignored_tests = { + split_node(i.target)[0] + "::" + split_node(i.target)[1] + for i in registry.discovery.ignore_tests + } + for node in sorted( + _discovered_tests(registry, root) - registered_nodes - ignored_tests + ): + violations.append( + _hidden( + f"guard test {node} is not referenced by any rule", + "add it to a pytest rule's nodes, or to discovery.ignore_tests with a reason", + ) + ) + + # CI jobs. + referenced_jobs = {(ref.workflow, ref.job) for r in rules for ref in r.ci} + ignored_jobs = {i.target for i in registry.discovery.ignore_jobs} + for workflow in registry.discovery.workflows: + for job_id in workflow_jobs(root / workflow)[0]: + if (workflow, job_id) not in referenced_jobs and job_id not in ignored_jobs: + violations.append( + _hidden( + f"CI job '{job_id}' in {workflow} enforces no registered rule", + "reference it from a rule's [[rule.ci]], or add it to discovery.ignore_jobs with a reason", + ) + ) + return violations diff --git a/scripts/rules/guard.py b/scripts/rules/guard.py new file mode 100644 index 000000000..aee74163a --- /dev/null +++ b/scripts/rules/guard.py @@ -0,0 +1,124 @@ +"""The registry guard: every check in one pass.""" + +from __future__ import annotations + +import os +from dataclasses import dataclass, field +from datetime import date +from pathlib import Path + +from scripts.rules.baseline import ( + BaseUnavailable, + check_baselines, + default_base_ref, + load_base_registry, +) +from scripts.rules.discover import discover_hidden +from scripts.rules.load import load_registry +from scripts.rules.model import Violation +from scripts.rules.render import ( + AGENTS_MD, + RULES_MD, + extract_block, + render_agents_block, + render_rules_md, +) +from scripts.rules.resolve import clear_caches, resolve_ci, resolve_enforcer +from scripts.rules.validate import validate + +REPO_ROOT = Path(__file__).resolve().parents[2] +RENDER_FIX = "run python -m scripts.rules render and commit the result" + + +@dataclass +class GuardResult: + violations: list[Violation] = field(default_factory=list) + notes: list[str] = field(default_factory=list) + + +def _stale_outputs(registry, root: Path) -> list[Violation]: + violations = [] + rules_md = root / RULES_MD + if not rules_md.exists(): + violations.append( + Violation(None, "render", f"{RULES_MD} is missing", RENDER_FIX) + ) + elif rules_md.read_text() != render_rules_md(registry): + violations.append(Violation(None, "render", f"{RULES_MD} is stale", RENDER_FIX)) + agents = root / AGENTS_MD + block = extract_block(agents.read_text()) if agents.exists() else None + if block is None: + violations.append( + Violation( + None, "render", f"{AGENTS_MD} has no generated rules block", RENDER_FIX + ) + ) + elif block != render_agents_block(registry): + violations.append( + Violation( + None, "render", f"the rules block in {AGENTS_MD} is stale", RENDER_FIX + ) + ) + return violations + + +def run_guard( + root: Path = REPO_ROOT, + *, + base_ref: str | None = None, + today: date | None = None, + compare_base: bool = True, +) -> GuardResult: + clear_caches() + result = GuardResult() + registry, schema_violations = load_registry(root) + result.violations += schema_violations + if registry is None: + return result + + result.violations += validate(registry, root, today or date.today()) + for rule in registry.rules: + result.violations += resolve_enforcer(rule, root) + result.violations += resolve_ci(rule, root) + result.violations += discover_hidden(registry, root) + result.violations += _stale_outputs(registry, root) + + ref = base_ref or default_base_ref() + if not compare_base: + return result + if ref is None: + result.notes.append( + "no base ref (set --base-ref or GITHUB_BASE_REF); baseline comparison skipped" + ) + else: + try: + base = load_base_registry(root, ref) + except BaseUnavailable: + if os.environ.get("GITHUB_ACTIONS"): + result.violations.append( + Violation( + None, + "baseline", + f"base ref {ref} is not available", + "check out with fetch-depth: 0", + ) + ) + else: + result.notes.append( + f"base ref {ref} is not available; baseline comparison skipped" + ) + else: + if base is None: + result.notes.append( + f"{ref} has no registry yet; baseline comparison skipped" + ) + else: + violations, notes = check_baselines(registry, base) + result.violations += violations + result.notes += notes + return result + + +def check() -> list[str]: + """Adapter for scripts/run_quality_guards.py.""" + return [str(v) for v in run_guard(REPO_ROOT).violations] diff --git a/scripts/rules/load.py b/scripts/rules/load.py new file mode 100644 index 000000000..10b6faa60 --- /dev/null +++ b/scripts/rules/load.py @@ -0,0 +1,353 @@ +"""Load ``rules/registry.toml`` into the typed model. + +Parsing reports structural problems (wrong types, unknown keys, bad enum values) +as violations instead of raising, so one run shows every problem at once. +""" + +from __future__ import annotations + +import tomllib +from datetime import date +from pathlib import Path +from typing import Any + +from scripts.rules.model import ( + Baseline, + Category, + CiRef, + Discovery, + Enforcer, + Ignored, + Kind, + Proof, + Registry, + Rule, + RuleException, + Status, + Tier, + Violation, +) + +REGISTRY_PATH = Path("rules/registry.toml") + +_TOP_KEYS = {"schema_version", "repo", "tool_version", "category", "discovery", "rule"} +_RULE_KEYS = { + "id", + "category", + "statement", + "rationale", + "scope", + "tier", + "status", + "autofix", + "review_by", + "enforcer", + "ci", + "proof", + "baseline", + "exception", + "downgrade", +} +_DISCOVERY_KEYS = { + "workflows", + "guard_test_globs", + "extra_guard_tests", + "ignore_jobs", + "ignore_tests", +} + + +class _Collector: + def __init__(self) -> None: + self.violations: list[Violation] = [] + + def add(self, rule_id: str | None, message: str, fix: str) -> None: + self.violations.append(Violation(rule_id, "schema", message, fix)) + + def unknown_keys( + self, rule_id: str | None, table: dict[str, Any], allowed: set[str], where: str + ) -> None: + for key in sorted(set(table) - allowed): + self.add( + rule_id, + f"unknown key '{key}' in {where}", + f"remove it or use one of {sorted(allowed)}", + ) + + def string( + self, rule_id: str | None, table: dict[str, Any], key: str, where: str + ) -> str: + value = table.get(key) + if isinstance(value, str) and value.strip(): + return value + self.add(rule_id, f"{where}.{key} must be a non-empty string", f"set {key}") + return "" + + def string_list( + self, + rule_id: str | None, + table: dict[str, Any], + key: str, + where: str, + *, + required: bool = True, + ) -> tuple[str, ...]: + value = table.get(key) + if value is None and not required: + return () + if ( + isinstance(value, list) + and all(isinstance(item, str) and item for item in value) + and (value or not required) + ): + return tuple(value) + self.add( + rule_id, + f"{where}.{key} must be a non-empty list of strings", + f'set {key} = ["..."]', + ) + return () + + def enum(self, rule_id: str | None, enum_type: Any, value: Any, where: str) -> Any: + try: + return enum_type(value) + except ValueError: + allowed = [member.value for member in enum_type] + self.add(rule_id, f"{where} is {value!r}", f"use one of {allowed}") + return None + + def day(self, rule_id: str | None, value: Any, where: str) -> date | None: + if isinstance(value, date): + return value + if isinstance(value, str): + try: + return date.fromisoformat(value) + except ValueError: + pass + self.add(rule_id, f"{where} must be a date", "write it as YYYY-MM-DD") + return None + + +def _ignored(c: _Collector, raw: Any, key: str, field: str) -> tuple[Ignored, ...]: + if raw is None: + return () + items = [] + for entry in raw if isinstance(raw, list) else [raw]: + if ( + isinstance(entry, dict) + and isinstance(entry.get(field), str) + and isinstance(entry.get("reason"), str) + and entry["reason"].strip() + ): + items.append(Ignored(entry[field], entry["reason"])) + else: + c.add( + None, + f"discovery.{key} entries need '{field}' and a non-empty 'reason'", + f'write {{ {field} = "...", reason = "..." }}', + ) + return tuple(items) + + +def _discovery(c: _Collector, raw: Any) -> Discovery: + if raw is None: + return Discovery() + if not isinstance(raw, dict): + c.add(None, "discovery must be a table", "write [discovery]") + return Discovery() + c.unknown_keys(None, raw, _DISCOVERY_KEYS, "discovery") + workflows = c.string_list(None, raw, "workflows", "discovery", required=False) + return Discovery( + workflows=workflows or Discovery().workflows, + guard_test_globs=c.string_list( + None, raw, "guard_test_globs", "discovery", required=False + ), + extra_guard_tests=c.string_list( + None, raw, "extra_guard_tests", "discovery", required=False + ), + ignore_jobs=_ignored(c, raw.get("ignore_jobs"), "ignore_jobs", "job"), + ignore_tests=_ignored(c, raw.get("ignore_tests"), "ignore_tests", "node"), + ) + + +def _rule(c: _Collector, raw: Any, index: int) -> Rule | None: + if not isinstance(raw, dict): + c.add(None, f"rule #{index + 1} must be a table", "write [[rule]]") + return None + rule_id = raw.get("id") if isinstance(raw.get("id"), str) else None + label = rule_id or f"rule #{index + 1}" + c.unknown_keys(rule_id, raw, _RULE_KEYS, label) + + enforcer_raw = raw.get("enforcer") + enforcer = Enforcer(Kind.NONE) + if isinstance(enforcer_raw, dict): + kind = c.enum(rule_id, Kind, enforcer_raw.get("kind"), f"{label}.enforcer.kind") + fields = {k: v for k, v in enforcer_raw.items() if k != "kind"} + enforcer = Enforcer(kind or Kind.NONE, fields) + else: + c.add(rule_id, f"{label}.enforcer must be a table", "add [rule.enforcer]") + + ci_refs = [] + ci_raw = raw.get("ci", []) + for entry in ci_raw if isinstance(ci_raw, list) else [ci_raw]: + if not isinstance(entry, dict): + c.add(rule_id, f"{label}.ci entries must be tables", "write [[rule.ci]]") + continue + c.unknown_keys(rule_id, entry, {"workflow", "job", "invocation"}, f"{label}.ci") + ci_refs.append( + CiRef( + c.string(rule_id, entry, "workflow", f"{label}.ci"), + c.string(rule_id, entry, "job", f"{label}.ci"), + c.string(rule_id, entry, "invocation", f"{label}.ci"), + ) + ) + + proof = None + if "proof" in raw: + if isinstance(raw["proof"], dict): + proof = Proof(dict(raw["proof"])) + else: + c.add(rule_id, f"{label}.proof must be a table", "write [rule.proof]") + + baseline = None + if "baseline" in raw: + braw = raw["baseline"] + if isinstance(braw, dict) and isinstance(braw.get("count"), int): + c.unknown_keys(rule_id, braw, {"file", "count"}, f"{label}.baseline") + baseline = Baseline( + c.string(rule_id, braw, "file", f"{label}.baseline"), braw["count"] + ) + else: + c.add( + rule_id, + f"{label}.baseline needs 'file' and an integer 'count'", + "write [rule.baseline] with file and count", + ) + + exceptions = [] + exc_raw = raw.get("exception", []) + for entry in exc_raw if isinstance(exc_raw, list) else [exc_raw]: + if not isinstance(entry, dict): + c.add( + rule_id, + f"{label}.exception entries must be tables", + "write [[rule.exception]]", + ) + continue + c.unknown_keys( + rule_id, entry, {"path", "reason", "expires"}, f"{label}.exception" + ) + expires = c.day(rule_id, entry.get("expires"), f"{label}.exception.expires") + exceptions.append( + RuleException( + c.string(rule_id, entry, "path", f"{label}.exception"), + c.string(rule_id, entry, "reason", f"{label}.exception"), + expires or date.max, + ) + ) + + downgrade_reason = None + if "downgrade" in raw: + draw = raw["downgrade"] + if isinstance(draw, dict) and isinstance(draw.get("reason"), str): + downgrade_reason = draw["reason"] + else: + c.add( + rule_id, + f"{label}.downgrade needs a 'reason'", + 'write [rule.downgrade] reason = "..."', + ) + + autofix = raw.get("autofix") + if not isinstance(autofix, bool): + c.add(rule_id, f"{label}.autofix must be true or false", "set autofix") + autofix = False + + review_by = None + if "review_by" in raw: + review_by = c.day(rule_id, raw["review_by"], f"{label}.review_by") + + tier = c.enum(rule_id, Tier, raw.get("tier"), f"{label}.tier") + status = c.enum(rule_id, Status, raw.get("status"), f"{label}.status") + if tier is None or status is None: + return None + + return Rule( + id=c.string(rule_id, raw, "id", label), + category=c.string(rule_id, raw, "category", label), + statement=c.string(rule_id, raw, "statement", label), + rationale=c.string(rule_id, raw, "rationale", label), + scope=c.string_list(rule_id, raw, "scope", label), + tier=tier, + status=status, + autofix=autofix, + enforcer=enforcer, + ci=tuple(ci_refs), + review_by=review_by, + proof=proof, + baseline=baseline, + exceptions=tuple(exceptions), + downgrade_reason=downgrade_reason, + ) + + +def parse(raw: dict[str, Any]) -> tuple[Registry | None, list[Violation]]: + """Turn raw TOML data into a ``Registry``, collecting every schema problem.""" + c = _Collector() + c.unknown_keys(None, raw, _TOP_KEYS, "registry") + + schema_version = raw.get("schema_version") + if schema_version != 1: + c.add(None, f"schema_version is {schema_version!r}", "set schema_version = 1") + + categories = [] + for entry in raw.get("category", []): + if isinstance(entry, dict) and isinstance(entry.get("id"), str): + c.unknown_keys(None, entry, {"id", "title"}, f"category {entry['id']}") + categories.append( + Category(entry["id"], c.string(None, entry, "title", "category")) + ) + else: + c.add( + None, + "each [[category]] needs an id and title", + "fix the category table", + ) + + rules = [_rule(c, entry, i) for i, entry in enumerate(raw.get("rule", []))] + + registry = Registry( + schema_version=1, + repo=c.string(None, raw, "repo", "registry"), + tool_version=c.string(None, raw, "tool_version", "registry"), + categories=tuple(categories), + discovery=_discovery(c, raw.get("discovery")), + rules=tuple(rule for rule in rules if rule is not None), + ) + return registry, c.violations + + +def load_text(text: str) -> tuple[Registry | None, list[Violation]]: + try: + raw = tomllib.loads(text) + except tomllib.TOMLDecodeError as error: + return None, [ + Violation( + None, + "schema", + f"registry is not valid TOML ({error})", + "fix the TOML syntax", + ) + ] + return parse(raw) + + +def load_registry(root: Path) -> tuple[Registry | None, list[Violation]]: + path = root / REGISTRY_PATH + if not path.exists(): + return None, [ + Violation( + None, "schema", f"{REGISTRY_PATH} is missing", f"create {REGISTRY_PATH}" + ) + ] + return load_text(path.read_text()) diff --git a/scripts/rules/model.py b/scripts/rules/model.py new file mode 100644 index 000000000..38247b9d4 --- /dev/null +++ b/scripts/rules/model.py @@ -0,0 +1,192 @@ +"""Data model for the rules registry.""" + +from __future__ import annotations + +from collections.abc import Mapping +from dataclasses import dataclass, field +from datetime import date +from enum import StrEnum +from typing import Any + + +class Tier(StrEnum): + MECHANICAL = "mechanical" + METRIC = "metric" + JUDGMENT = "judgment" + + +class Status(StrEnum): + ENFORCED = "enforced" + RATCHETED = "ratcheted" + ADVISORY = "advisory" + DOC_ONLY = "doc-only" + + +class Kind(StrEnum): + PYTEST = "pytest" + QUALITY_GUARD = "quality-guard" + RUFF = "ruff" + MYPY = "mypy" + PYRIGHT = "pyright" + IMPORT_LINTER = "import-linter" + WORKFLOW_JOB = "workflow-job" + DOCS_GUARD = "docs-guard" + LLM_REVIEW = "llm-review" + NONE = "none" + + +@dataclass(frozen=True) +class Violation: + """One guard failure. ``fix`` always tells the reader what to do next.""" + + rule_id: str | None + check: str + message: str + fix: str + + def __str__(self) -> str: + subject = self.rule_id or "registry" + return f"[{subject}] {self.check}: {self.message}. Fix: {self.fix}." + + +@dataclass(frozen=True) +class Category: + id: str + title: str + + +@dataclass(frozen=True) +class Ignored: + """A discovered item deliberately left out of the registry.""" + + target: str + reason: str + + +@dataclass(frozen=True) +class Discovery: + workflows: tuple[str, ...] = (".github/workflows/pr.yml",) + guard_test_globs: tuple[str, ...] = () + extra_guard_tests: tuple[str, ...] = () + ignore_jobs: tuple[Ignored, ...] = () + ignore_tests: tuple[Ignored, ...] = () + + +@dataclass(frozen=True) +class Enforcer: + kind: Kind + fields: Mapping[str, Any] = field(default_factory=dict) + + def get(self, key: str, default: Any = None) -> Any: + return self.fields.get(key, default) + + @property + def description(self) -> str: + """Short human-readable description of what enforces the rule.""" + kind, fields = self.kind, self.fields + if kind is Kind.PYTEST: + prefix = f"{fields['project'].rstrip('/')}/" if "project" in fields else "" + files = sorted({node.split("::", 1)[0] for node in fields.get("nodes", [])}) + return "pytest " + ", ".join(f"`{prefix}{path}`" for path in files) + if kind is Kind.QUALITY_GUARD: + return f"quality guard `{fields.get('name', '')}`" + if kind is Kind.RUFF: + codes = ", ".join(fields.get("codes", [])) + mode = fields.get("mode", "") + if mode == "format": + return "ruff format" + if mode == "per-file-ignore": + return ( + f"ruff per-file ignore of {codes} for `{fields.get('pattern', '')}`" + ) + return f"ruff check ({codes})" + if kind is Kind.MYPY: + return "mypy" + if kind is Kind.WORKFLOW_JOB: + return "CI job" + if kind in (Kind.PYRIGHT, Kind.IMPORT_LINTER): + return f"{kind.value} `{fields.get('config', '')}`" + if kind is Kind.DOCS_GUARD: + return f"docs guard on `{fields.get('doc', '')}`" + if kind is Kind.LLM_REVIEW: + return f"LLM review with `{fields.get('prompt', '')}`" + return "nothing" + + @property + def expects_proof(self) -> bool: + """Custom checkers need proof; third-party tools and CI jobs are the check.""" + return self.kind in ( + Kind.PYTEST, + Kind.QUALITY_GUARD, + Kind.DOCS_GUARD, + Kind.IMPORT_LINTER, + Kind.LLM_REVIEW, + ) + + +@dataclass(frozen=True) +class CiRef: + workflow: str + job: str + invocation: str + + +@dataclass(frozen=True) +class Proof: + fields: Mapping[str, Any] + + +@dataclass(frozen=True) +class Baseline: + file: str + count: int + + +@dataclass(frozen=True) +class RuleException: + path: str + reason: str + expires: date + + +@dataclass(frozen=True) +class Rule: + id: str + category: str + statement: str + rationale: str + scope: tuple[str, ...] + tier: Tier + status: Status + autofix: bool + enforcer: Enforcer + ci: tuple[CiRef, ...] = () + review_by: date | None = None + proof: Proof | None = None + baseline: Baseline | None = None + exceptions: tuple[RuleException, ...] = () + downgrade_reason: str | None = None + + @property + def blocking(self) -> bool: + """Derived, never declared: whether a violation can fail a merge.""" + return ( + self.status in (Status.ENFORCED, Status.RATCHETED) + and self.tier is not Tier.JUDGMENT + ) + + +@dataclass(frozen=True) +class Registry: + schema_version: int + repo: str + tool_version: str + categories: tuple[Category, ...] + discovery: Discovery + rules: tuple[Rule, ...] + + def rule(self, rule_id: str) -> Rule | None: + for rule in self.rules: + if rule.id == rule_id: + return rule + return None diff --git a/scripts/rules/render.py b/scripts/rules/render.py new file mode 100644 index 000000000..9458349b2 --- /dev/null +++ b/scripts/rules/render.py @@ -0,0 +1,271 @@ +"""Render the registry into ``docs/generated/rules.md`` and the AGENTS.md block. + +Output is deterministic (fixed ordering, no timestamps, no dependence on today's +date) so the guard can regenerate it and compare. +""" + +from __future__ import annotations + +import hashlib +import re +from pathlib import Path + +from scripts.rules import TOOL_VERSION +from scripts.rules.model import Kind, Registry, Rule, Status, Tier + +RULES_MD = Path("docs/generated/rules.md") +AGENTS_MD = Path("AGENTS.md") +SKILL_DOC = "docs/engineering/skills/rules-registry.md" +BEGIN = "" +END = "" +MERMAID_NODE_LIMIT = 40 +MERMAID_CHUNK = 15 +AGENTS_COLLAPSE_AT = 60 + + +def tool_hash() -> str: + digest = hashlib.sha256() + for path in sorted(Path(__file__).parent.glob("*.py")): + digest.update(path.name.encode()) + digest.update(path.read_bytes()) + return digest.hexdigest()[:12] + + +def _cell(text: str) -> str: + return text.replace("|", "\\|").replace("\n", " ") + + +def _ordered(registry: Registry) -> list[tuple[str, str, list[Rule]]]: + """Categories in declared order, each with its rules sorted by id.""" + groups = [] + for category in registry.categories: + rules = sorted( + (r for r in registry.rules if r.category == category.id), key=lambda r: r.id + ) + if rules: + groups.append((category.id, category.title, rules)) + return groups + + +def _ci_label(rule: Rule) -> str: + return ", ".join(f"`{Path(ref.workflow).name}:{ref.job}`" for ref in rule.ci) or "—" + + +def _unproven(rule: Rule) -> bool: + return rule.blocking and rule.proof is None and rule.enforcer.expects_proof + + +def _proof_label(rule: Rule) -> str: + if rule.proof is not None: + return "yes" + return "**unproven**" if _unproven(rule) else "—" + + +def _enforced_by(rule: Rule) -> str: + """What enforces a rule, naming the CI job when the job itself is the check.""" + if rule.enforcer.kind is Kind.WORKFLOW_JOB: + jobs = ", ".join(f"`{ref.job}`" for ref in rule.ci) + return f"CI job {jobs}" + return rule.enforcer.description + + +def _mermaid_id(prefix: str, text: str) -> str: + return prefix + "_" + re.sub(r"[^A-Za-z0-9]", "_", text) + + +def _mermaid(rules: list[Rule]) -> list[str]: + lines = ["```mermaid", "flowchart LR"] + declared: set[str] = set() + edges: list[str] = [] + + def node(node_id: str, label: str, shape: str = "box") -> str: + if node_id not in declared: + declared.add(node_id) + safe = label.replace('"', "'") + if shape == "round": + lines.append(f' {node_id}("{safe}")') + elif shape == "flag": + lines.append(f' {node_id}>"{safe}"]') + else: + lines.append(f' {node_id}["{safe}"]') + return node_id + + for rule in rules: + rule_node = node( + _mermaid_id("r", rule.id), f"{rule.id} ({rule.status})", "round" + ) + if rule.enforcer.kind is Kind.NONE: + gap = node("gap_unenforced", "not enforced", "flag") + edges.append(f" {rule_node} -.-> {gap}") + continue + label = _enforced_by(rule).replace("`", "") + enforcer_node = node(_mermaid_id("e", label), label) + edges.append(f" {rule_node} --> {enforcer_node}") + for ref in rule.ci: + job_label = f"CI {Path(ref.workflow).name}:{ref.job}" + job_node = node(_mermaid_id("j", job_label), job_label) + edge = f" {enforcer_node} --> {job_node}" + if edge not in edges: + edges.append(edge) + if not rule.ci: + local = node("local_only", "not run in CI", "flag") + edge = f" {enforcer_node} -.-> {local}" + if edge not in edges: + edges.append(edge) + return lines + edges + ["```"] + + +def _mermaid_chunks(rules: list[Rule]) -> list[list[str]]: + graph = _mermaid(rules) + node_count = sum(1 for line in graph if re.match(r"^ \w+[\[(>]", line)) + if node_count <= MERMAID_NODE_LIMIT: + return [graph] + return [ + _mermaid(rules[i : i + MERMAID_CHUNK]) + for i in range(0, len(rules), MERMAID_CHUNK) + ] + + +def _bullets(items: list[str]) -> list[str]: + return items or ["- None."] + + +def render_rules_md(registry: Registry) -> str: + rules = sorted(registry.rules, key=lambda r: r.id) + out = [ + "# Rules", + "", + f"Generated by `python -m scripts.rules render` (scripts/rules v{TOOL_VERSION}, " + f"tool sha256 `{tool_hash()}`) from `rules/registry.toml`. Do not edit by hand.", + "", + f"Every rule that governs `{registry.repo}` is listed here with how it is enforced. " + "Blocking rules fail CI. The Gaps section lists what is written down but not yet " + f"enforced or proven. To add or change a rule, see `{SKILL_DOC}`.", + "", + "## Summary", + "", + "| Tier | " + " | ".join(s.value for s in Status) + " | Total |", + "| --- | " + " | ".join("---:" for _ in Status) + " | ---: |", + ] + for tier in Tier: + counts = [ + sum(1 for r in rules if r.tier is tier and r.status is s) for s in Status + ] + out.append( + f"| {tier.value} | " + + " | ".join(str(c) for c in counts) + + f" | {sum(counts)} |" + ) + blocking = sum(1 for r in rules if r.blocking) + out += ["", f"**Blocking:** {blocking} of {len(rules)} rules.", "", "## Gaps", ""] + + def section(title: str, items: list[str]) -> None: + out.extend([f"### {title}", "", *_bullets(items), ""]) + + section( + "Not enforced (doc-only)", + [f"- `{r.id}`: {r.statement}" for r in rules if r.status is Status.DOC_ONLY], + ) + section( + "Advisory (checked but not blocking)", + [f"- `{r.id}`: {r.statement}" for r in rules if r.status is Status.ADVISORY], + ) + section( + "Unproven (no proof that the checker rejects a bad example)", + [f"- `{r.id}`" for r in rules if _unproven(r)], + ) + section( + "Ratcheted (existing violations still tolerated)", + [ + f"- `{r.id}`: {r.baseline.count} remaining in `{r.baseline.file}`" + for r in rules + if r.baseline + ], + ) + dated = [f"- `{r.id}`: review by {r.review_by}" for r in rules if r.review_by] + [ + f"- `{r.id}`: exception for `{e.path}` until {e.expires} ({e.reason})" + for r in rules + for e in r.exceptions + ] + section("Exceptions and review dates", sorted(dated)) + + for _, title, group in _ordered(registry): + out += [ + f"## {title}", + "", + "| Rule | Statement | Tier | Status | Blocking | Enforced by | CI | Proof |", + "| --- | --- | --- | --- | --- | --- | --- | --- |", + ] + for r in group: + out.append( + f"| `{r.id}` | {_cell(r.statement)} | {r.tier} | {r.status} | " + f"{'yes' if r.blocking else 'no'} | {_cell(_enforced_by(r))} | " + f"{_ci_label(r)} | {_proof_label(r)} |" + ) + out.append("") + for chunk in _mermaid_chunks(group): + out += chunk + [""] + return "\n".join(out).rstrip() + "\n" + + +def _agents_line(rule: Rule) -> str: + if rule.status is Status.DOC_ONLY: + how = "doc-only, not checked: follow it yourself" + elif rule.status is Status.ADVISORY: + how = f"advisory: {_enforced_by(rule)}, not blocking" + elif rule.blocking: + how = f"blocking: {_enforced_by(rule)}" + else: + how = f"{rule.status}: {_enforced_by(rule)}" + return f"- `{rule.id}`: {rule.statement} ({how})" + + +def render_agents_block(registry: Registry) -> str: + out = [ + BEGIN, + "", + "Every rule for this repository lives in `rules/registry.toml` and is rendered,", + "with its enforcement and gaps, in `docs/generated/rules.md`. Blocking rules", + "fail CI. Doc-only rules are not checked automatically, so follow them yourself.", + f"To add or change a rule, read `{SKILL_DOC}`.", + "", + ] + collapse = len(registry.rules) > AGENTS_COLLAPSE_AT + for _, title, group in _ordered(registry): + out.append(f"### {title}") + out.append("") + if collapse: + enforced = [f"`{r.id}`" for r in group if r.status is not Status.DOC_ONLY] + if enforced: + out.append("- Checked: " + ", ".join(enforced)) + out += [_agents_line(r) for r in group if r.status is Status.DOC_ONLY] + else: + out += [_agents_line(r) for r in group] + out.append("") + out.append(END) + return "\n".join(out) + + +def splice_block(text: str, block: str) -> str: + """Replace the generated block in AGENTS.md, appending a Rules section if absent.""" + start, end = text.find(BEGIN), text.find(END) + if start != -1 and end != -1 and end > start: + return text[:start] + block + text[end + len(END) :] + return text.rstrip() + "\n\n## Rules\n\n" + block + "\n" + + +def extract_block(text: str) -> str | None: + start, end = text.find(BEGIN), text.find(END) + if start == -1 or end == -1 or end < start: + return None + return text[start : end + len(END)] + + +def write_outputs(registry: Registry, root: Path) -> list[Path]: + rules_md = root / RULES_MD + rules_md.parent.mkdir(parents=True, exist_ok=True) + rules_md.write_text(render_rules_md(registry)) + agents = root / AGENTS_MD + current = agents.read_text() if agents.exists() else "# Agent Instructions\n" + agents.write_text(splice_block(current, render_agents_block(registry))) + return [rules_md, agents] diff --git a/scripts/rules/resolve.py b/scripts/rules/resolve.py new file mode 100644 index 000000000..596557e9b --- /dev/null +++ b/scripts/rules/resolve.py @@ -0,0 +1,402 @@ +"""Check that each rule's enforcer and CI wiring really exist. + +Everything here reads files statically (``ast``, ``tomllib`` and the workflow +subset reader) so it needs no project dependencies and never imports project +code. +""" + +from __future__ import annotations + +import ast +import re +import tomllib +from collections.abc import Callable +from functools import cache +from pathlib import Path +from typing import Any + +from scripts.rules.model import Kind, Rule, Violation +from scripts.rules.workflows import JobBlock, parse_jobs + +DEFAULT_RUNNER = "scripts/run_quality_guards.py" +RUFF_DEFAULT_SELECT = ("E4", "E7", "E9", "F") + + +# --- shared readers ------------------------------------------------------- + + +@cache +def _read(path: Path) -> str | None: + return path.read_text() if path.is_file() else None + + +def clear_caches() -> None: + _read.cache_clear() + workflow_jobs.cache_clear() + test_names.cache_clear() + + +@cache +def workflow_jobs(path: Path) -> tuple[dict[str, JobBlock], tuple[str, ...]]: + text = _read(path) + if text is None: + return {}, (f"{path.name} is missing",) + jobs, problems = parse_jobs(text) + return jobs, tuple(problems) + + +@cache +def test_names(path: Path) -> frozenset[str] | None: + """Pytest-collectable names in a file: ``func`` and ``Class::method``.""" + text = _read(path) + if text is None: + return None + names = set() + for node in ast.parse(text).body: + if isinstance( + node, (ast.FunctionDef, ast.AsyncFunctionDef) + ) and node.name.startswith("test"): + names.add(node.name) + elif isinstance(node, ast.ClassDef) and node.name.startswith("Test"): + for item in node.body: + if isinstance( + item, (ast.FunctionDef, ast.AsyncFunctionDef) + ) and item.name.startswith("test"): + names.add(f"{node.name}::{item.name}") + return frozenset(names) + + +def split_node(node: str) -> tuple[str, str]: + """``path::name[param]`` -> (``path``, ``name``).""" + path, _, name = node.partition("::") + return path, re.sub(r"\[.*\]$", "", name) + + +def root_relative_nodes(rule: Rule) -> list[str]: + """Pytest nodes of a rule, prefixed with the project directory if any.""" + project = rule.enforcer.get("project") + nodes = rule.enforcer.get("nodes", []) + prefix = f"{project.rstrip('/')}/" if project else "" + return [f"{prefix}{node}" for node in nodes] + + +def node_exists(root: Path, node: str) -> str | None: + """Return a problem description, or None if the node resolves.""" + path, name = split_node(node) + if not name: + return f"'{node}' has no '::test_name' part" + names = test_names(root / path) + if names is None: + return f"test file {path} does not exist" + if name not in names: + return f"{path} has no test '{name}'" + return None + + +def guard_names(root: Path, runner: str = DEFAULT_RUNNER) -> list[str] | None: + """String names in the ``GUARDS`` tuple of the quality-guard runner.""" + text = _read(root / runner) + if text is None: + return None + for node in ast.parse(text).body: + target = None + if isinstance(node, ast.Assign) and len(node.targets) == 1: + target, value = node.targets[0], node.value + elif isinstance(node, ast.AnnAssign): + target, value = node.target, node.value + if ( + isinstance(target, ast.Name) + and target.id == "GUARDS" + and isinstance(value, ast.Tuple) + ): + names = [] + for element in value.elts: + if ( + isinstance(element, ast.Tuple) + and element.elts + and isinstance(element.elts[0], ast.Constant) + and isinstance(element.elts[0].value, str) + ): + names.append(element.elts[0].value) + return names + return [] + + +def _toml(path: Path) -> dict[str, Any]: + text = _read(path) + return tomllib.loads(text) if text else {} + + +def ruff_lint_config(root: Path) -> dict[str, Any]: + """The ``lint`` table from ruff.toml, or ``[tool.ruff.lint]`` in pyproject.""" + ruff_toml = _toml(root / "ruff.toml") + if ruff_toml: + return ruff_toml.get("lint", {}) + return ( + _toml(root / "pyproject.toml").get("tool", {}).get("ruff", {}).get("lint", {}) + ) + + +def ruff_selected(root: Path) -> tuple[str, ...]: + lint = ruff_lint_config(root) + selected = tuple(lint.get("select", RUFF_DEFAULT_SELECT)) + return selected + tuple(lint.get("extend-select", ())) + + +def mypy_files(root: Path) -> list[str]: + files = ( + _toml(root / "pyproject.toml").get("tool", {}).get("mypy", {}).get("files", []) + ) + return [files] if isinstance(files, str) else list(files) + + +def make_target_paths(root: Path, target: str) -> list[str] | None: + """Pytest path arguments in a Makefile target's recipe.""" + text = _read(root / "Makefile") + if text is None: + return None + lines = text.splitlines() + for index, line in enumerate(lines): + if re.match(rf"^{re.escape(target)}\s*:", line): + recipe = [] + for body in lines[index + 1 :]: + if not body.startswith("\t"): + break + recipe.append(body) + paths = [] + for body in recipe: + if "pytest" not in body: + continue + tokens = body.split("pytest", 1)[1].split() + paths += [ + t + for t in tokens + if not t.startswith("-") and ("/" in t or t.endswith(".py")) + ] + return paths + return None + + +# --- per-kind resolvers --------------------------------------------------- + + +def _v(rule: Rule, message: str, fix: str) -> Violation: + return Violation(rule.id, "enforcer", message, fix) + + +def _resolve_pytest(rule: Rule, root: Path) -> list[Violation]: + violations = [] + project = rule.enforcer.get("project") + if project and not (root / project).is_dir(): + violations.append( + _v( + rule, + f"project directory {project} does not exist", + "fix enforcer.project", + ) + ) + for node in root_relative_nodes(rule): + problem = node_exists(root, node) + if problem: + violations.append( + _v( + rule, + problem, + "point enforcer.nodes at an existing test, or restore the test", + ) + ) + return violations + + +def _resolve_quality_guard(rule: Rule, root: Path) -> list[Violation]: + runner = rule.enforcer.get("runner", DEFAULT_RUNNER) + names = guard_names(root, runner) + if names is None: + return [ + _v(rule, f"guard runner {runner} does not exist", "fix enforcer.runner") + ] + name = rule.enforcer.get("name") + if name not in names: + return [ + _v( + rule, + f"no guard named '{name}' in GUARDS of {runner}", + f"use one of {names}", + ) + ] + return [] + + +def _resolve_ruff(rule: Rule, root: Path) -> list[Violation]: + mode = rule.enforcer.get("mode") + codes = rule.enforcer.get("codes", []) + if mode == "lint": + selected = ruff_selected(root) + missing = [ + code for code in codes if not any(code.startswith(s) for s in selected) + ] + if missing: + return [ + _v( + rule, + f"ruff does not select {missing} (selected: {list(selected)})", + "add them to the ruff select list or fix enforcer.codes", + ) + ] + elif mode == "per-file-ignore": + ignores = ruff_lint_config(root).get("per-file-ignores", {}) + pattern = rule.enforcer.get("pattern") + if sorted(ignores.get(pattern, [])) != sorted(codes): + return [ + _v( + rule, + f"ruff per-file-ignores has no entry {pattern!r} = {codes}", + "match enforcer.pattern/codes to the ruff config", + ) + ] + return [] + + +def _resolve_mypy(rule: Rule, root: Path) -> list[Violation]: + configured = mypy_files(root) + missing = [p for p in rule.enforcer.get("paths", []) if p not in configured] + if missing: + return [ + _v( + rule, + f"{missing} not in [tool.mypy].files", + "add them to [tool.mypy].files or fix enforcer.paths", + ) + ] + return [] + + +def _resolve_file_field(field_name: str) -> Callable[[Rule, Path], list[Violation]]: + def resolve(rule: Rule, root: Path) -> list[Violation]: + value = rule.enforcer.get(field_name) + if value and not (root / value).exists(): + return [ + _v( + rule, + f"{field_name} file {value} does not exist", + f"create it or fix enforcer.{field_name}", + ) + ] + return [] + + return resolve + + +def _resolve_workflow_job(rule: Rule, root: Path) -> list[Violation]: + step = rule.enforcer.get("step") + if not step: + return [] + violations = [] + for ref in rule.ci: + job = workflow_jobs(root / ref.workflow)[0].get(ref.job) + if job and not any(step in name for name in job.step_names): + violations.append( + _v( + rule, + f"job '{ref.job}' has no step named like '{step}'", + "fix enforcer.step", + ) + ) + return violations + + +RESOLVERS: dict[Kind, Callable[[Rule, Path], list[Violation]]] = { + Kind.PYTEST: _resolve_pytest, + Kind.QUALITY_GUARD: _resolve_quality_guard, + Kind.RUFF: _resolve_ruff, + Kind.MYPY: _resolve_mypy, + Kind.PYRIGHT: _resolve_file_field("config"), + Kind.IMPORT_LINTER: _resolve_file_field("config"), + Kind.WORKFLOW_JOB: _resolve_workflow_job, + Kind.DOCS_GUARD: _resolve_file_field("doc"), + Kind.LLM_REVIEW: _resolve_file_field("prompt"), + Kind.NONE: lambda rule, root: [], +} + + +def resolve_enforcer(rule: Rule, root: Path) -> list[Violation]: + return RESOLVERS[rule.enforcer.kind](rule, root) + + +# --- CI wiring ------------------------------------------------------------ + + +def resolve_ci(rule: Rule, root: Path) -> list[Violation]: + """Each CI reference names a real job that really runs the enforcer.""" + violations = [] + for ref in rule.ci: + + def v(message: str, fix: str) -> Violation: + return Violation(rule.id, "ci", message, fix) + + jobs, problems = workflow_jobs(root / ref.workflow) + if not (root / ref.workflow).is_file(): + violations.append( + v(f"workflow {ref.workflow} does not exist", "fix rule.ci.workflow") + ) + continue + for problem in problems: + violations.append( + v( + f"{ref.workflow}: {problem}", + "keep workflows within the supported YAML subset", + ) + ) + job = jobs.get(ref.job) + if job is None: + violations.append( + v(f"no job '{ref.job}' in {ref.workflow}", f"use one of {sorted(jobs)}") + ) + continue + if not job.invokes(ref.invocation): + violations.append( + v( + f"job '{ref.job}' in {ref.workflow} does not run '{ref.invocation}'", + "correct rule.ci.invocation, or make the job run the enforcer", + ) + ) + continue + if rule.enforcer.kind is Kind.PYTEST: + violations += _pytest_coverage(rule, root, ref.invocation, job) + return violations + + +def _pytest_coverage( + rule: Rule, root: Path, invocation: str, job: JobBlock +) -> list[Violation]: + """The job's test command actually collects the rule's tests.""" + violations = [] + if invocation.startswith("make "): + target = invocation.split()[1] + paths = make_target_paths(root, target) + if paths: + for node in root_relative_nodes(rule): + path = split_node(node)[0] + if not any( + path == p or path.startswith(p.rstrip("/") + "/") for p in paths + ): + violations.append( + Violation( + rule.id, + "ci", + f"{path} is not collected by 'make {target}' (collects {paths})", + f"add it to the Makefile '{target}' target or move the test", + ) + ) + project = rule.enforcer.get("project") + if project and job.matrix: + if not any(project in values for values in job.matrix.values()): + violations.append( + Violation( + rule.id, + "ci", + f"project {project} is not in the matrix of job '{job.id}'", + "add the project to the job matrix", + ) + ) + return violations diff --git a/scripts/rules/tests/__init__.py b/scripts/rules/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/scripts/rules/tests/fixtures/policyengine_api_pr.yml b/scripts/rules/tests/fixtures/policyengine_api_pr.yml new file mode 100644 index 000000000..01d1cbbf8 --- /dev/null +++ b/scripts/rules/tests/fixtures/policyengine_api_pr.yml @@ -0,0 +1,201 @@ +name: Pull request + +on: pull_request + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number }} + cancel-in-progress: true + +env: + ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true + OLD_SIMULATION_GATEWAY_URL: ${{ vars.OLD_SIMULATION_GATEWAY_URL }} + +jobs: + ensure-policyengine-bundle-supported-by-simulation-api: + name: Ensure PolicyEngine bundle is supported by simulation API + runs-on: ubuntu-latest + environment: staging + env: + SIM_ENTRYPOINT: ${{ vars.SIM_ENTRYPOINT }} + SIMULATION_ENTRYPOINT_URL: ${{ secrets.SIMULATION_ENTRYPOINT_URL }} + permissions: + contents: read + steps: + - name: Checkout repo + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Install jq + run: sudo apt-get install -y jq + - name: Set up uv for installed bundle validation + uses: astral-sh/setup-uv@v6 + - name: Check simulation API supports updated PolicyEngine bundle + run: bash .github/check-policyengine-bundle-supported.sh --if-changed-from-base "${{ github.base_ref }}" + + lint: + name: Lint + runs-on: ubuntu-latest + steps: + - name: Checkout repo + uses: actions/checkout@v4 + - name: Setup Python + uses: actions/setup-python@v5 + - name: Install ruff + run: pip install ruff>=0.9.0 + - name: Format check with ruff + run: ruff format --check . + typecheck-v2: + name: Type-check API v2 + runs-on: ubuntu-latest + steps: + - name: Checkout repo + uses: actions/checkout@v4 + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Setup uv + uses: astral-sh/setup-uv@v6 + - name: Install locked dependencies + run: uv sync --frozen --extra dev + - name: Type-check API v2 + run: uv run --frozen --extra dev mypy + quality-guards: + name: Quality guards + runs-on: ubuntu-latest + steps: + - name: Checkout repo + uses: actions/checkout@v4 + with: + # The rules registry guard compares baselines with the base branch. + fetch-depth: 0 + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Test the rules registry tool + run: python -m unittest discover -s scripts/rules/tests -t . + - name: Run quality guards + run: python scripts/run_quality_guards.py + + alembic-v1-check: + name: Alembic v1 qualification + uses: ./.github/workflows/alembic-v1-check.yml + alembic-v2-check: + name: Alembic v2 qualification + uses: ./.github/workflows/alembic-v2-check.yml + v2-integration-check: + name: Cross-database integration tests + uses: ./.github/workflows/v2-integration-check.yml + secrets: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + check-changelog: + name: Check changelog fragment + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Check for changelog fragment + run: bash .github/scripts/check_changelog_fragment.sh + test_container_builds: + name: Docker + runs-on: ubuntu-latest + needs: ensure-policyengine-bundle-supported-by-simulation-api + permissions: + contents: read + packages: write + steps: + - name: Checkout repo + uses: actions/checkout@v4 + - name: Log in to the Container registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Build container + run: docker build -t ghcr.io/policyengine/policyengine docker + test_cloud_run_container_builds: + name: Cloud Run container + runs-on: ubuntu-latest + needs: ensure-policyengine-bundle-supported-by-simulation-api + permissions: + contents: read + steps: + - name: Checkout repo + uses: actions/checkout@v4 + - name: Build Cloud Run container + run: docker build -f gcp/cloud_run/Dockerfile -t policyengine-api-cloud-run:test . + test_env_vars: + name: Test environment variables + runs-on: ubuntu-latest + needs: ensure-policyengine-bundle-supported-by-simulation-api + permissions: + contents: read + id-token: write + steps: + - name: Checkout repo + uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Auth + uses: google-github-actions/auth@v2 + with: + workload_identity_provider: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }} + service_account: ${{ secrets.GCP_DEPLOY_SERVICE_ACCOUNT }} + - name: Wait until PolicyEngine bundle version is available on PyPI + run: .github/wait-for-pypi.sh + - name: Install dependencies + run: make install + - name: Run environment variable tests + run: pytest tests/env_variables/test_environment_variables.py + env: + POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN: ${{ secrets.POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN }} + # Compatibility name required by PolicyEngine Core. The GitHub + # credential name describes its provider, data, and read-only scope. + HUGGING_FACE_TOKEN: ${{ secrets.PE_UK_PRIVATE_HF_READ_TOKEN }} + POLICYENGINE_DB_PASSWORD: ${{ secrets.POLICYENGINE_DB_PASSWORD }} + test: + name: Test + runs-on: ubuntu-latest + needs: + - ensure-policyengine-bundle-supported-by-simulation-api + - test_env_vars + permissions: + contents: read + id-token: write + steps: + - name: Checkout repo + uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Install system dependencies + run: sudo apt-get update && sudo apt-get install -y redis-server + - name: Start Redis + run: sudo systemctl start redis-server + - name: Auth + uses: google-github-actions/auth@v2 + with: + workload_identity_provider: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }} + service_account: ${{ secrets.GCP_DEPLOY_SERVICE_ACCOUNT }} + - name: Set up Cloud SDK + uses: google-github-actions/setup-gcloud@v2 + with: + project_id: policyengine-api + - name: Install dependencies + run: make install + - name: Test the API + run: make test + env: + POLICYENGINE_DB_PASSWORD: ${{ secrets.POLICYENGINE_DB_PASSWORD }} + POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN: ${{ secrets.POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + HUGGING_FACE_TOKEN: ${{ secrets.PE_UK_PRIVATE_HF_READ_TOKEN }} + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v5 + with: + token: ${{ secrets.CODECOV_TOKEN }} + slug: PolicyEngine/policyengine-api diff --git a/scripts/rules/tests/fixtures/policyengine_sim_api_pr.yml b/scripts/rules/tests/fixtures/policyengine_sim_api_pr.yml new file mode 100644 index 000000000..196fb5486 --- /dev/null +++ b/scripts/rules/tests/fixtures/policyengine_sim_api_pr.yml @@ -0,0 +1,179 @@ +name: PR checks + +on: + pull_request: + branches: [main] + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + test: + name: Test + runs-on: ubuntu-latest + strategy: + matrix: + # Each entry is a uv project tested in its own locked environment. + # For the gateway this is the parity guarantee: the same lock the + # Modal image installs with uv_sync(frozen=True). + project: + - projects/policyengine-simulation-executor + - projects/policyengine-simulation-entry + - projects/policyengine-simulation-gateway + - libs/policyengine-simulation-contract + - libs/policyengine-simulation-observability + - libs/policyengine-stage12-persistence + + steps: + - uses: actions/checkout@v6 + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: '3.13' + + - name: Install uv + uses: astral-sh/setup-uv@v8.1.0 + with: + enable-cache: true + + - name: Install dependencies + run: | + cd ${{ matrix.project }} + uv sync --extra test + + - name: Run tests + run: | + cd ${{ matrix.project }} + uv run pytest tests/ -v + + lint: + name: Lint + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v6 + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: '3.13' + + - name: Install uv + uses: astral-sh/setup-uv@v8.1.0 + + - name: Install ruff + run: uv tool install ruff + + - name: Run ruff format check + run: | + for dir in projects/*/src libs/*/src; do + if [ -d "$dir" ]; then + echo "Checking format in $dir..." + uv run ruff format --check $dir + fi + done + + quality-guards: + name: Quality guards + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v6 + with: + # The rules registry guard compares baselines with the base branch. + fetch-depth: 0 + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: '3.13' + + - name: Test the rules registry tool + run: python -m unittest discover -s scripts/rules/tests -t . + + - name: Run quality guards + run: python scripts/run_quality_guards.py + + docker-build: + name: Docker build + needs: [test, lint] # Only run if tests and linting pass + runs-on: ubuntu-latest + strategy: + matrix: + service: [simulation-entry, simulation-executor] + + steps: + - uses: actions/checkout@v6 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v4 + + - name: Build Docker image + uses: docker/build-push-action@v7 + with: + context: . + file: projects/policyengine-${{ matrix.service }}/Dockerfile + push: false + tags: policyengine-${{ matrix.service }}:test + cache-from: type=gha + cache-to: type=gha,mode=max + platforms: linux/amd64 + + test-integration: + name: Test integration + needs: docker-build # Only run if docker builds succeed + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v6 + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: '3.13' + + - name: Install uv + uses: astral-sh/setup-uv@v8.1.0 + with: + enable-cache: true + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v4 + + - name: Generate API clients + run: | + ./scripts/generate-clients.sh + + - name: Start services + run: | + docker compose -f deployment/docker-compose.yml up -d + # Wait for services to be ready + sleep 10 + # Check services are responding + for i in {1..30}; do + if curl -f http://localhost:8082/ping/alive; then + echo "Services are ready!" + break + fi + echo "Waiting for services... (attempt $i/30)" + sleep 2 + done + + - name: Run integration tests (local services only) + run: | + cd projects/policyengine-apis-integ + uv sync --extra test + # Run tests against local Docker services only (not Modal staging) + uv run pytest tests/ -v -m "not requires_gcp and not beta_only" + + - name: Show service logs on failure + if: failure() + run: | + docker compose -f deployment/docker-compose.yml logs + + - name: Stop services + if: always() + run: | + docker compose -f deployment/docker-compose.yml down diff --git a/scripts/rules/tests/support.py b/scripts/rules/tests/support.py new file mode 100644 index 000000000..d10653122 --- /dev/null +++ b/scripts/rules/tests/support.py @@ -0,0 +1,232 @@ +"""Build small throwaway repositories for exercising the rules tool.""" + +from __future__ import annotations + +import tempfile +import textwrap +from datetime import date +from pathlib import Path + +from scripts.rules import TOOL_VERSION +from scripts.rules.guard import GuardResult, run_guard +from scripts.rules.load import load_registry +from scripts.rules.render import write_outputs +from scripts.rules.resolve import clear_caches + +TODAY = date(2026, 6, 1) + +REGISTRY = f""" +schema_version = 1 +repo = "example" +tool_version = "{TOOL_VERSION}" + +[[category]] +id = "layers" +title = "Layers" + +[[category]] +id = "quality" +title = "Quality" + +[discovery] +guard_test_globs = ["tests/**/test_*boundar*.py"] + +[[rule]] +id = "ex.layers.no-sql-in-routes" +category = "layers" +statement = "Routes never issue SQL." +rationale = "docs/guide.md#routes" +scope = ["pkg/*.py"] +tier = "mechanical" +status = "enforced" +autofix = false +[rule.enforcer] +kind = "pytest" +nodes = ["tests/unit/test_boundaries.py::test_routes_have_no_sql"] +[[rule.ci]] +workflow = ".github/workflows/pr.yml" +job = "test" +invocation = "make test" + +[[rule]] +id = "ex.layers.services-no-http" +category = "layers" +statement = "Services never build HTTP responses." +rationale = "docs/guide.md" +scope = ["pkg/*.py"] +tier = "mechanical" +status = "enforced" +autofix = false +[rule.enforcer] +kind = "pytest" +nodes = ["tests/unit/test_boundaries.py::TestServices::test_no_http"] +[[rule.ci]] +workflow = ".github/workflows/pr.yml" +job = "test" +invocation = "make test" + +[[rule]] +id = "ex.process.registry" +category = "quality" +statement = "The registry is consistent." +rationale = "docs/guide.md" +scope = ["rules/registry.toml"] +tier = "mechanical" +status = "enforced" +autofix = false +[rule.enforcer] +kind = "quality-guard" +name = "rules-registry" +[[rule.ci]] +workflow = ".github/workflows/pr.yml" +job = "quality-guards" +invocation = "run_quality_guards.py" + +[[rule]] +id = "ex.quality.format" +category = "quality" +statement = "Code is formatted." +rationale = "docs/guide.md" +scope = ["pkg/*.py"] +tier = "mechanical" +status = "enforced" +autofix = true +[rule.enforcer] +kind = "ruff" +mode = "format" +[[rule.ci]] +workflow = ".github/workflows/pr.yml" +job = "lint" +invocation = "ruff format --check" + +[[rule]] +id = "ex.quality.types" +category = "quality" +statement = "The package type-checks." +rationale = "docs/guide.md" +scope = ["pkg/*.py"] +tier = "mechanical" +status = "enforced" +autofix = false +[rule.enforcer] +kind = "mypy" +paths = ["pkg"] +[[rule.ci]] +workflow = ".github/workflows/pr.yml" +job = "typecheck" +invocation = "mypy" + +[[rule]] +id = "ex.layers.connectors-never-commit" +category = "layers" +statement = "Connectors never commit." +rationale = "docs/guide.md" +scope = ["pkg/*.py"] +tier = "mechanical" +status = "doc-only" +autofix = false +[rule.enforcer] +kind = "none" +""" + +WORKFLOW = """ +name: Pull request +on: pull_request +jobs: + test: + name: Test + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + # make test is mentioned in this comment only in some cases + - name: Run tests + run: make test + lint: + runs-on: ubuntu-latest + steps: + - name: Format + run: | + pip install ruff + ruff format --check . + typecheck: + runs-on: ubuntu-latest + steps: + - run: uv run mypy + quality-guards: + runs-on: ubuntu-latest + steps: + - name: Guards + run: python scripts/run_quality_guards.py +""" + +FILES = { + "rules/registry.toml": REGISTRY, + ".github/workflows/pr.yml": WORKFLOW, + "Makefile": "test:\n\tpython -m pytest tests/unit --disable-pytest-warnings\n", + "pyproject.toml": '[tool.mypy]\nfiles = ["pkg"]\n', + "scripts/run_quality_guards.py": ('GUARDS = (("rules-registry", lambda: []),)\n'), + "tests/unit/test_boundaries.py": textwrap.dedent( + """ + import pytest + + + @pytest.mark.parametrize("path", ["a", "b"]) + def test_routes_have_no_sql(path): + pass + + + class TestServices: + def test_no_http(self): + pass + """ + ), + "docs/guide.md": "# Guide\n\n## Routes\n", + "pkg/routes.py": "def route():\n return 1\n", + "AGENTS.md": "# Agent Instructions\n\nRead the guide.\n", +} + + +class Repo: + """A temporary repository tree. Use as a context manager.""" + + def __init__(self, files: dict[str, str] | None = None, *, render: bool = True): + self._tmp = tempfile.TemporaryDirectory() + self.root = Path(self._tmp.name) + for path, text in {**FILES, **(files or {})}.items(): + self.write(path, text) + if render: + self.render() + + def __enter__(self) -> Repo: + return self + + def __exit__(self, *exc: object) -> None: + self._tmp.cleanup() + + def write(self, path: str, text: str) -> None: + target = self.root / path + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(text) + clear_caches() + + def read(self, path: str) -> str: + return (self.root / path).read_text() + + def edit(self, path: str, old: str, new: str) -> None: + text = self.read(path) + assert old in text, f"{old!r} not in {path}" + self.write(path, text.replace(old, new, 1)) + + def render(self) -> None: + clear_caches() + registry, violations = load_registry(self.root) + assert registry is not None and not violations, violations + write_outputs(registry, self.root) + + def guard(self, **kwargs) -> GuardResult: + kwargs.setdefault("today", TODAY) + kwargs.setdefault("compare_base", "base_ref" in kwargs) + return run_guard(self.root, **kwargs) + + def messages(self, **kwargs) -> list[str]: + return [str(v) for v in self.guard(**kwargs).violations] diff --git a/scripts/rules/tests/test_guard.py b/scripts/rules/tests/test_guard.py new file mode 100644 index 000000000..7d14cc540 --- /dev/null +++ b/scripts/rules/tests/test_guard.py @@ -0,0 +1,518 @@ +"""Tests for the rules registry guard, using throwaway repositories.""" + +from __future__ import annotations + +import subprocess +import unittest +from datetime import date + +from scripts.rules.baseline import check_baselines +from scripts.rules.load import load_text +from scripts.rules.render import render_agents_block, render_rules_md +from scripts.rules.tests.support import REGISTRY, Repo + +RATCHETED_RULE = """ +[[rule]] +id = "ex.layers.ratcheted" +category = "layers" +statement = "Ratcheted rule." +rationale = "docs/guide.md" +scope = ["pkg/*.py"] +tier = "mechanical" +status = "ratcheted" +autofix = false +[rule.enforcer] +kind = "pytest" +nodes = ["tests/unit/test_boundaries.py::test_routes_have_no_sql"] +[[rule.ci]] +workflow = ".github/workflows/pr.yml" +job = "test" +invocation = "make test" +[rule.baseline] +file = "rules/baselines/ratcheted.txt" +count = {count} +""" + + +def with_rule(extra: str) -> dict[str, str]: + return {"rules/registry.toml": REGISTRY + extra} + + +class ValidRepositoryTest(unittest.TestCase): + def test_valid_repository_passes(self) -> None: + with Repo() as repo: + self.assertEqual(repo.messages(), []) + + def test_render_is_deterministic(self) -> None: + with Repo() as repo: + registry, _ = load_text(repo.read("rules/registry.toml")) + assert registry is not None + self.assertEqual(render_rules_md(registry), render_rules_md(registry)) + self.assertEqual( + render_agents_block(registry), render_agents_block(registry) + ) + + def test_rules_md_lists_gaps(self) -> None: + with Repo() as repo: + rules_md = repo.read("docs/generated/rules.md") + self.assertIn( + "ex.layers.connectors-never-commit", rules_md.split("## Gaps")[1] + ) + self.assertIn("```mermaid", rules_md) + self.assertIn("## Rules", repo.read("AGENTS.md")) + + +class SchemaTest(unittest.TestCase): + def assertViolation(self, repo: Repo, fragment: str) -> None: + messages = repo.messages() + self.assertTrue(any(fragment in m for m in messages), messages) + + def test_missing_required_field(self) -> None: + with Repo(render=False) as repo: + repo.edit( + "rules/registry.toml", 'statement = "Routes never issue SQL."\n', "" + ) + self.assertViolation(repo, "statement must be a non-empty string") + + def test_unknown_enforcer_kind(self) -> None: + with Repo(render=False) as repo: + repo.edit("rules/registry.toml", 'kind = "mypy"', 'kind = "flake8"') + self.assertViolation(repo, "enforcer.kind is 'flake8'") + + def test_bad_status(self) -> None: + with Repo(render=False) as repo: + repo.edit( + "rules/registry.toml", 'status = "doc-only"', 'status = "optional"' + ) + self.assertViolation(repo, "status is 'optional'") + + def test_unknown_key(self) -> None: + with Repo(render=False) as repo: + repo.edit( + "rules/registry.toml", + "autofix = true", + 'autofix = true\nowner = "someone"', + ) + self.assertViolation(repo, "unknown key 'owner'") + + def test_duplicate_rule_id(self) -> None: + with Repo(render=False) as repo: + repo.edit( + "rules/registry.toml", + 'id = "ex.quality.types"', + 'id = "ex.quality.format"', + ) + self.assertViolation(repo, "duplicate rule id") + + def test_tool_version_mismatch(self) -> None: + with Repo(render=False) as repo: + text = repo.read("rules/registry.toml") + repo.write( + "rules/registry.toml", + text.replace('tool_version = "', 'tool_version = "9', 1), + ) + self.assertViolation(repo, "tool_version is 9") + + def test_scope_glob_must_match(self) -> None: + with Repo(render=False) as repo: + repo.edit( + "rules/registry.toml", + 'scope = ["rules/registry.toml"]', + 'scope = ["nowhere/*.py"]', + ) + self.assertViolation(repo, "scope 'nowhere/*.py' matches no files") + + def test_doc_only_rule_cannot_claim_enforcer(self) -> None: + with Repo(render=False) as repo: + repo.edit( + "rules/registry.toml", + 'status = "enforced"\nautofix = true', + 'status = "doc-only"\nautofix = true', + ) + self.assertViolation(repo, "doc-only rules have no enforcer") + + def test_enforced_rule_needs_ci(self) -> None: + extra = """ +[[rule]] +id = "ex.layers.local-only" +category = "layers" +statement = "Only checked locally." +rationale = "docs/guide.md" +scope = ["pkg/*.py"] +tier = "mechanical" +status = "enforced" +autofix = false +[rule.enforcer] +kind = "pytest" +nodes = ["tests/unit/test_boundaries.py::test_routes_have_no_sql"] +""" + with Repo(with_rule(extra), render=False) as repo: + self.assertViolation(repo, "enforced rules must run in CI") + + def test_ruff_is_never_ratcheted(self) -> None: + with Repo(render=False) as repo: + repo.write("rules/baselines/ruff.txt", "pkg/routes.py E501\n") + repo.edit( + "rules/registry.toml", + 'kind = "ruff"\nmode = "format"', + 'kind = "ruff"\nmode = "format"\n[rule.baseline]\nfile = "rules/baselines/ruff.txt"\ncount = 1', + ) + self.assertViolation(repo, "ruff is never ratcheted") + + def test_ratcheted_without_baseline(self) -> None: + with Repo(render=False) as repo: + repo.edit( + "rules/registry.toml", + 'status = "enforced"\nautofix = false\n[rule.enforcer]\nkind = "mypy"', + 'status = "ratcheted"\nautofix = false\n[rule.enforcer]\nkind = "mypy"', + ) + self.assertViolation(repo, "ratcheted rules need a baseline") + + def test_baseline_count_must_match_file(self) -> None: + with Repo(with_rule(RATCHETED_RULE.format(count=3)), render=False) as repo: + repo.write( + "rules/baselines/ratcheted.txt", + "# tolerated\npkg/routes.py\npkg/other.py\n", + ) + self.assertViolation( + repo, "baseline count is 3 but rules/baselines/ratcheted.txt lists 2" + ) + + def test_judgment_rules_need_review_enforcer(self) -> None: + with Repo(render=False) as repo: + repo.edit( + "rules/registry.toml", + 'tier = "mechanical"\nstatus = "enforced"\nautofix = true', + 'tier = "judgment"\nstatus = "enforced"\nautofix = true', + ) + self.assertViolation(repo, "judgment rules are enforced by review") + + +class ResolutionTest(unittest.TestCase): + def assertViolation(self, repo: Repo, fragment: str) -> None: + messages = repo.messages() + self.assertTrue(any(fragment in m for m in messages), messages) + + def test_unresolved_pytest_node(self) -> None: + with Repo() as repo: + repo.edit( + "tests/unit/test_boundaries.py", + "def test_routes_have_no_sql", + "def test_renamed", + ) + self.assertViolation(repo, "has no test 'test_routes_have_no_sql'") + + def test_parametrized_node_resolves(self) -> None: + with Repo() as repo: + repo.edit( + "rules/registry.toml", + 'test_boundaries.py::test_routes_have_no_sql"', + 'test_boundaries.py::test_routes_have_no_sql[a]"', + ) + repo.render() + self.assertEqual(repo.messages(), []) + + def test_test_outside_make_target_is_not_collected(self) -> None: + with Repo() as repo: + repo.edit("Makefile", "tests/unit", "tests/contract") + self.assertViolation(repo, "is not collected by 'make test'") + + def test_project_must_be_in_matrix(self) -> None: + workflow = """ +jobs: + test: + strategy: + matrix: + project: + - projects/alpha + steps: + - run: | + cd ${{ matrix.project }} + uv run pytest tests/ +""" + extra = """ +[[rule]] +id = "ex.layers.beta" +category = "layers" +statement = "Beta rule." +rationale = "docs/guide.md" +scope = ["projects/beta/src/*.py"] +tier = "mechanical" +status = "enforced" +autofix = false +[rule.enforcer] +kind = "pytest" +project = "projects/beta" +nodes = ["tests/test_beta.py::test_beta"] +[[rule.ci]] +workflow = ".github/workflows/matrix.yml" +job = "test" +invocation = "pytest tests/" +""" + files = { + ".github/workflows/matrix.yml": workflow, + "projects/beta/tests/test_beta.py": "def test_beta():\n pass\n", + "projects/beta/src/beta.py": "", + **with_rule(extra), + } + with Repo(files) as repo: + self.assertViolation( + repo, "project projects/beta is not in the matrix of job 'test'" + ) + + def test_missing_job(self) -> None: + with Repo() as repo: + repo.edit("rules/registry.toml", 'job = "typecheck"', 'job = "types"') + repo.render() + self.assertViolation(repo, "no job 'types'") + + def test_invocation_only_in_comment_does_not_count(self) -> None: + with Repo() as repo: + repo.edit(".github/workflows/pr.yml", "run: make test", "run: echo skipped") + self.assertViolation(repo, "does not run 'make test'") + + def test_unknown_quality_guard(self) -> None: + with Repo() as repo: + repo.edit( + "rules/registry.toml", 'name = "rules-registry"', 'name = "renamed"' + ) + repo.render() + self.assertViolation(repo, "no guard named 'renamed'") + + def test_mypy_path_must_be_configured(self) -> None: + with Repo() as repo: + repo.edit( + "rules/registry.toml", 'paths = ["pkg"]', 'paths = ["pkg", "other"]' + ) + repo.render() + self.assertViolation(repo, "['other'] not in [tool.mypy].files") + + def test_ruff_lint_codes_must_be_selected(self) -> None: + extra = """ +[[rule]] +id = "ex.quality.bugbear" +category = "quality" +statement = "No bugbear findings." +rationale = "docs/guide.md" +scope = ["pkg/*.py"] +tier = "mechanical" +status = "advisory" +autofix = false +[rule.enforcer] +kind = "ruff" +mode = "lint" +codes = ["B"] +""" + with Repo(with_rule(extra)) as repo: + self.assertViolation(repo, "ruff does not select ['B']") + + def test_proof_files_must_exist(self) -> None: + with Repo(render=False) as repo: + repo.edit( + "rules/registry.toml", + 'kind = "mypy"\npaths = ["pkg"]', + 'kind = "mypy"\npaths = ["pkg"]\n[rule.proof]\nbad_fixture = "rules/fixtures/bad.py"\n' + 'good_fixture = "rules/fixtures/good.py"\ntest = "tests/unit/test_boundaries.py::test_routes_have_no_sql"', + ) + self.assertViolation( + repo, "proof bad_fixture rules/fixtures/bad.py does not exist" + ) + + +class HiddenRuleTest(unittest.TestCase): + def assertViolation(self, repo: Repo, fragment: str) -> None: + messages = repo.messages() + self.assertTrue(any(fragment in m for m in messages), messages) + + def test_unregistered_quality_guard(self) -> None: + with Repo() as repo: + repo.write( + "scripts/run_quality_guards.py", + 'GUARDS = (("rules-registry", lambda: []), ("secret-guard", lambda: []))\n', + ) + self.assertViolation(repo, "quality guard 'secret-guard'") + + def test_unregistered_guard_test(self) -> None: + with Repo() as repo: + text = repo.read("tests/unit/test_boundaries.py") + repo.write( + "tests/unit/test_boundaries.py", + text + "\n\ndef test_new_boundary():\n pass\n", + ) + self.assertViolation( + repo, "test_boundaries.py::test_new_boundary is not referenced" + ) + + def test_unregistered_mypy_file(self) -> None: + with Repo() as repo: + repo.write("pyproject.toml", '[tool.mypy]\nfiles = ["pkg", "scripts"]\n') + self.assertViolation(repo, "mypy checks 'scripts'") + + def test_unregistered_ruff_ignore(self) -> None: + with Repo() as repo: + repo.write( + "pyproject.toml", + '[tool.mypy]\nfiles = ["pkg"]\n\n[tool.ruff.lint.per-file-ignores]\n"pkg/*.py" = ["E501"]\n', + ) + self.assertViolation(repo, "ruff per-file-ignores 'pkg/*.py' has no rule") + + def test_unregistered_ci_job(self) -> None: + with Repo() as repo: + text = repo.read(".github/workflows/pr.yml") + repo.write( + ".github/workflows/pr.yml", + text + " deploy:\n steps:\n - run: ./deploy.sh\n", + ) + self.assertViolation(repo, "CI job 'deploy'") + + def test_ignored_ci_job_with_reason_passes(self) -> None: + with Repo() as repo: + text = repo.read(".github/workflows/pr.yml") + repo.write( + ".github/workflows/pr.yml", + text + " deploy:\n steps:\n - run: ./deploy.sh\n", + ) + repo.edit( + "rules/registry.toml", + "[discovery]\n", + '[discovery]\nignore_jobs = [{ job = "deploy", reason = "deploys, enforces nothing" }]\n', + ) + repo.render() + self.assertEqual(repo.messages(), []) + + +class RenderFreshnessTest(unittest.TestCase): + def test_stale_rules_md(self) -> None: + with Repo() as repo: + repo.write( + "docs/generated/rules.md", + repo.read("docs/generated/rules.md") + "edited\n", + ) + self.assertIn( + "[registry] render: docs/generated/rules.md is stale", + "\n".join(repo.messages()), + ) + + def test_missing_agents_markers(self) -> None: + with Repo() as repo: + repo.write("AGENTS.md", "# Agent Instructions\n") + self.assertIn( + "AGENTS.md has no generated rules block", "\n".join(repo.messages()) + ) + + def test_stale_agents_block(self) -> None: + with Repo() as repo: + repo.edit("AGENTS.md", "Routes never issue SQL.", "Routes may issue SQL.") + self.assertIn( + "the rules block in AGENTS.md is stale", "\n".join(repo.messages()) + ) + + def test_render_keeps_surrounding_agents_text(self) -> None: + with Repo() as repo: + agents = repo.read("AGENTS.md") + self.assertTrue( + agents.startswith("# Agent Instructions\n\nRead the guide.\n") + ) + repo.render() + self.assertEqual(repo.read("AGENTS.md"), agents) + + +class ExpiryTest(unittest.TestCase): + def test_expired_review_by(self) -> None: + with Repo(render=False) as repo: + repo.edit( + "rules/registry.toml", + 'statement = "Code is formatted."', + 'statement = "Code is formatted."\nreview_by = 2026-01-01', + ) + repo.render() + messages = "\n".join(repo.messages()) + self.assertIn("review_by 2026-01-01 has passed", messages) + + def test_expired_exception(self) -> None: + with Repo(render=False) as repo: + repo.edit( + "rules/registry.toml", + 'kind = "mypy"\npaths = ["pkg"]', + 'kind = "mypy"\npaths = ["pkg"]\n[[rule.exception]]\npath = "pkg/routes.py"\n' + 'reason = "migrating"\nexpires = 2026-05-01', + ) + repo.render() + self.assertIn( + "exception for pkg/routes.py expired on 2026-05-01", + "\n".join(repo.messages()), + ) + self.assertEqual(repo.messages(today=date(2026, 4, 1)), []) + + def test_exception_is_listed_in_rules_md(self) -> None: + with Repo(render=False) as repo: + repo.edit( + "rules/registry.toml", + 'kind = "mypy"\npaths = ["pkg"]', + 'kind = "mypy"\npaths = ["pkg"]\n[[rule.exception]]\npath = "pkg/routes.py"\n' + 'reason = "migrating"\nexpires = 2026-12-01', + ) + repo.render() + self.assertIn( + "exception for `pkg/routes.py` until 2026-12-01", + repo.read("docs/generated/rules.md"), + ) + + +class BaselineTest(unittest.TestCase): + def registry(self, extra: str): + registry, violations = load_text(REGISTRY + extra) + assert registry is not None and not violations, violations + return registry + + def test_baseline_may_shrink(self) -> None: + base = self.registry(RATCHETED_RULE.format(count=3)) + current = self.registry(RATCHETED_RULE.format(count=2)) + self.assertEqual(check_baselines(current, base)[0], []) + + def test_baseline_may_not_grow(self) -> None: + base = self.registry(RATCHETED_RULE.format(count=2)) + current = self.registry(RATCHETED_RULE.format(count=3)) + violations = check_baselines(current, base)[0] + self.assertIn("baseline grew from 2 to 3", str(violations[0])) + + def test_downgrade_needs_reason(self) -> None: + base = self.registry("") + weakened = REGISTRY.replace( + 'status = "enforced"\nautofix = true\n[rule.enforcer]\nkind = "ruff"\nmode = "format"\n[[rule.ci]]\n' + 'workflow = ".github/workflows/pr.yml"\njob = "lint"\ninvocation = "ruff format --check"', + 'status = "advisory"\nautofix = true\n[rule.enforcer]\nkind = "ruff"\nmode = "format"', + ) + current, _ = load_text(weakened) + assert current is not None + violations = check_baselines(current, base)[0] + self.assertIn("status weakened from enforced to advisory", str(violations[0])) + + with_reason, _ = load_text( + weakened.replace( + 'mode = "format"', + 'mode = "format"\n[rule.downgrade]\nreason = "flaky"', + 1, + ) + ) + assert with_reason is not None + self.assertEqual(check_baselines(with_reason, base)[0], []) + + def test_compares_against_git_base_ref(self) -> None: + with Repo(with_rule(RATCHETED_RULE.format(count=1)), render=False) as repo: + repo.write("rules/baselines/ratcheted.txt", "pkg/routes.py\n") + repo.render() + git = ["git", "-c", "user.email=t@example.com", "-c", "user.name=t"] + subprocess.run(["git", "init", "-q"], cwd=repo.root, check=True) + subprocess.run([*git, "add", "-A"], cwd=repo.root, check=True) + subprocess.run([*git, "commit", "-qm", "base"], cwd=repo.root, check=True) + repo.edit("rules/registry.toml", "count = 1", "count = 2") + repo.write("rules/baselines/ratcheted.txt", "pkg/routes.py\npkg/other.py\n") + repo.render() + self.assertIn( + "baseline grew from 1 to 2", "\n".join(repo.messages(base_ref="HEAD")) + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/rules/tests/test_workflows.py b/scripts/rules/tests/test_workflows.py new file mode 100644 index 000000000..137337c52 --- /dev/null +++ b/scripts/rules/tests/test_workflows.py @@ -0,0 +1,80 @@ +"""Tests for the GitHub Actions subset reader, pinned against both repos' real PR workflows.""" + +from __future__ import annotations + +import unittest +from pathlib import Path + +from scripts.rules.workflows import parse_jobs + +FIXTURES = Path(__file__).parent / "fixtures" + + +class RealWorkflowTest(unittest.TestCase): + def test_policyengine_api_pr_workflow(self) -> None: + jobs, problems = parse_jobs((FIXTURES / "policyengine_api_pr.yml").read_text()) + self.assertEqual(problems, []) + self.assertIn("make test", jobs["test"].run_text) + self.assertIn("ruff format --check", jobs["lint"].run_text) + self.assertIn("run_quality_guards.py", jobs["quality-guards"].run_text) + self.assertIn("unittest discover", jobs["quality-guards"].run_text) + self.assertTrue(jobs["alembic-v1-check"].invokes("alembic-v1-check.yml")) + self.assertEqual(jobs["typecheck-v2"].name, "Type-check API v2") + + def test_policyengine_sim_api_pr_workflow(self) -> None: + jobs, problems = parse_jobs( + (FIXTURES / "policyengine_sim_api_pr.yml").read_text() + ) + self.assertEqual(problems, []) + self.assertIn( + "projects/policyengine-simulation-entry", jobs["test"].matrix["project"] + ) + self.assertIn( + "libs/policyengine-stage12-persistence", jobs["test"].matrix["project"] + ) + self.assertIn("uv run pytest tests/", jobs["test"].run_text) + self.assertIn("ruff format --check", jobs["lint"].run_text) + self.assertIn("run_quality_guards.py", jobs["quality-guards"].run_text) + self.assertTrue(jobs["docker-build"].invokes("docker/build-push-action")) + + +class SubsetTest(unittest.TestCase): + def test_block_scalar_stops_at_sibling_key(self) -> None: + text = """ +jobs: + build: + steps: + - name: Build + run: | + make build + env: + FLAG: make test +""" + jobs, _ = parse_jobs(text) + self.assertFalse(jobs["build"].invokes("make test")) + self.assertTrue(jobs["build"].invokes("make build")) + + def test_comments_are_ignored(self) -> None: + text = "jobs:\n a:\n steps:\n # run: make test\n - run: echo hi # make test\n" + jobs, _ = parse_jobs(text) + self.assertFalse(jobs["a"].invokes("make test")) + + def test_quoted_hash_is_not_a_comment(self) -> None: + jobs, _ = parse_jobs( + 'jobs:\n a:\n steps:\n - run: "echo #1 && make test"\n' + ) + self.assertTrue(jobs["a"].invokes("make test")) + + def test_inline_matrix_list(self) -> None: + jobs, _ = parse_jobs( + 'jobs:\n a:\n strategy:\n matrix:\n python: ["3.12", "3.13"]\n' + ) + self.assertEqual(jobs["a"].matrix["python"], ("3.12", "3.13")) + + def test_anchors_are_reported(self) -> None: + _, problems = parse_jobs("jobs:\n a: &base\n steps: []\n") + self.assertTrue(problems) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/rules/validate.py b/scripts/rules/validate.py new file mode 100644 index 000000000..81aaf479f --- /dev/null +++ b/scripts/rules/validate.py @@ -0,0 +1,319 @@ +"""Semantic validation of a parsed registry against the repository tree.""" + +from __future__ import annotations + +import re +from datetime import date +from pathlib import Path +from typing import Any + +from scripts.rules import TOOL_VERSION +from scripts.rules.model import Kind, Registry, Rule, Status, Tier, Violation +from scripts.rules.resolve import node_exists + +RULE_ID = re.compile(r"^[a-z0-9]+(\.[a-z0-9][a-z0-9-]*)+$") + +# Required and optional enforcer fields per kind, with their expected types. +KIND_FIELDS: dict[Kind, tuple[dict[str, type], dict[str, type]]] = { + Kind.PYTEST: ({"nodes": list}, {"project": str}), + Kind.QUALITY_GUARD: ({"name": str}, {"runner": str}), + Kind.RUFF: ({"mode": str}, {"codes": list, "pattern": str, "config": str}), + Kind.MYPY: ({"paths": list}, {"config": str}), + Kind.PYRIGHT: ({"config": str}, {"project": str}), + Kind.IMPORT_LINTER: ({"config": str, "contract": str}, {}), + Kind.WORKFLOW_JOB: ({}, {"step": str}), + Kind.DOCS_GUARD: ({"doc": str, "marker": str}, {}), + Kind.LLM_REVIEW: ({"prompt": str, "model": str}, {}), + Kind.NONE: ({}, {}), +} + +RUFF_MODES = ("format", "lint", "per-file-ignore") + +PROOF_FIELDS: dict[Tier, dict[str, Any]] = { + Tier.MECHANICAL: {"bad_fixture": str, "good_fixture": str, "test": str}, + Tier.METRIC: { + "below_fixture": str, + "above_fixture": str, + "threshold": (int, float), + "test": str, + }, + Tier.JUDGMENT: { + "eval_set": str, + "min_precision": float, + "measured_precision": float, + "measured_on": date, + }, +} +PROOF_FILE_FIELDS = ( + "bad_fixture", + "good_fixture", + "below_fixture", + "above_fixture", + "eval_set", +) + + +def baseline_entries(path: Path) -> int: + return sum( + 1 + for line in path.read_text().splitlines() + if line.strip() and not line.lstrip().startswith("#") + ) + + +def _check_fields( + rule: Rule, + fields: dict[str, Any], + required: dict[str, Any], + optional: dict[str, Any], + where: str, +) -> list[Violation]: + violations = [] + for key, expected in required.items(): + if key not in fields: + violations.append( + Violation( + rule.id, "schema", f"{where} is missing '{key}'", f"add {key}" + ) + ) + elif not isinstance(fields[key], expected): + violations.append( + Violation( + rule.id, + "schema", + f"{where}.{key} has the wrong type", + f"make it a {expected}", + ) + ) + for key in sorted(set(fields) - set(required) - set(optional)): + violations.append( + Violation( + rule.id, + "schema", + f"unknown key '{key}' in {where}", + f"remove it; allowed: {sorted({**required, **optional})}", + ) + ) + return violations + + +def _validate_enforcer(rule: Rule) -> list[Violation]: + required, optional = KIND_FIELDS[rule.enforcer.kind] + where = f"enforcer ({rule.enforcer.kind})" + violations = _check_fields( + rule, dict(rule.enforcer.fields), required, optional, where + ) + kind, fields = rule.enforcer.kind, rule.enforcer.fields + if kind is Kind.PYTEST and not fields.get("nodes"): + violations.append( + Violation( + rule.id, + "schema", + "pytest enforcer has no nodes", + "list at least one test node", + ) + ) + if kind is Kind.RUFF: + mode = fields.get("mode") + if mode not in RUFF_MODES: + violations.append( + Violation( + rule.id, + "schema", + f"ruff mode is {mode!r}", + f"use one of {list(RUFF_MODES)}", + ) + ) + if mode in ("lint", "per-file-ignore") and not fields.get("codes"): + violations.append( + Violation( + rule.id, "schema", f"ruff {mode} needs codes", "list the ruff codes" + ) + ) + if mode == "per-file-ignore" and not fields.get("pattern"): + violations.append( + Violation( + rule.id, + "schema", + "ruff per-file-ignore needs a pattern", + "set pattern", + ) + ) + return violations + + +def _validate_cross_fields(rule: Rule) -> list[Violation]: + violations = [] + + def v(message: str, fix: str) -> None: + violations.append(Violation(rule.id, "schema", message, fix)) + + kind, status, tier = rule.enforcer.kind, rule.status, rule.tier + if status is Status.DOC_ONLY and kind is not Kind.NONE: + v( + "doc-only rules have no enforcer", + 'set enforcer.kind = "none", or pick a stronger status', + ) + if status is Status.DOC_ONLY and rule.ci: + v( + "doc-only rules have no CI wiring", + "remove rule.ci, or pick a stronger status", + ) + if kind is Kind.NONE and status not in (Status.DOC_ONLY, Status.ADVISORY): + v( + f"status {status} needs an enforcer", + "add [rule.enforcer] or use doc-only/advisory", + ) + if status in (Status.ENFORCED, Status.RATCHETED) and not rule.ci: + v(f"{status} rules must run in CI", "add a [[rule.ci]] reference") + if status is Status.RATCHETED and rule.baseline is None: + v("ratcheted rules need a baseline", "add [rule.baseline] or use enforced") + if rule.baseline is not None and status is not Status.RATCHETED: + v( + "only ratcheted rules may have a baseline", + 'remove [rule.baseline] or set status = "ratcheted"', + ) + if kind is Kind.RUFF and rule.baseline is not None: + v( + "ruff is never ratcheted", + "remove the baseline and fix every violation in a standalone PR that also changes the ruff select list", + ) + if tier is Tier.JUDGMENT and kind not in (Kind.LLM_REVIEW, Kind.NONE): + v( + "judgment rules are enforced by review, not by a mechanical checker", + 'use enforcer.kind = "llm-review" or "none"', + ) + if kind is Kind.LLM_REVIEW and tier is not Tier.JUDGMENT: + v("llm-review only enforces judgment rules", 'set tier = "judgment"') + return violations + + +def _validate_proof(rule: Rule, root: Path) -> list[Violation]: + if rule.proof is None: + return [] + fields = dict(rule.proof.fields) + optional = {"project": str} if "test" in PROOF_FIELDS[rule.tier] else {} + violations = _check_fields( + rule, fields, PROOF_FIELDS[rule.tier], optional, f"proof ({rule.tier})" + ) + for key in PROOF_FILE_FIELDS: + if isinstance(fields.get(key), str) and not (root / fields[key]).exists(): + violations.append( + Violation( + rule.id, + "proof", + f"proof {key} {fields[key]} does not exist", + "add the fixture or fix the path", + ) + ) + test = fields.get("test") + if isinstance(test, str): + prefix = ( + f"{fields['project'].rstrip('/')}/" + if isinstance(fields.get("project"), str) + else "" + ) + problem = node_exists(root, prefix + test) + if problem: + violations.append( + Violation( + rule.id, + "proof", + problem, + "point proof.test at the test that runs the fixtures", + ) + ) + return violations + + +def validate(registry: Registry, root: Path, today: date) -> list[Violation]: + violations: list[Violation] = [] + + def v(rule_id: str | None, message: str, fix: str, check: str = "schema") -> None: + violations.append(Violation(rule_id, check, message, fix)) + + if registry.tool_version != TOOL_VERSION: + v( + None, + f"tool_version is {registry.tool_version} but scripts/rules is {TOOL_VERSION}", + "sync scripts/rules (see SYNC.md) and update tool_version", + ) + + category_ids = [category.id for category in registry.categories] + for duplicate in sorted({c for c in category_ids if category_ids.count(c) > 1}): + v(None, f"duplicate category '{duplicate}'", "give each category a unique id") + + rule_ids = [rule.id for rule in registry.rules] + for duplicate in sorted({r for r in rule_ids if rule_ids.count(r) > 1}): + v(duplicate, "duplicate rule id", "give each rule a unique id") + + for rule in registry.rules: + if rule.id and not RULE_ID.match(rule.id): + v( + rule.id, + "id must be lowercase dotted, e.g. 'api.v2.routes-no-sql'", + "rename the rule", + ) + if rule.category not in category_ids: + v( + rule.id, + f"unknown category '{rule.category}'", + f"use one of {category_ids} or add a [[category]]", + ) + + rationale_path = rule.rationale.split("#", 1)[0] + if rationale_path and not (root / rationale_path).exists(): + v( + rule.id, + f"rationale {rationale_path} does not exist", + "link an existing doc", + ) + + for pattern in rule.scope: + if not any(root.glob(pattern)): + v(rule.id, f"scope '{pattern}' matches no files", "fix the glob") + + violations.extend(_validate_enforcer(rule)) + violations.extend(_validate_cross_fields(rule)) + violations.extend(_validate_proof(rule, root)) + + if rule.baseline is not None: + path = root / rule.baseline.file + if not path.is_file(): + v( + rule.id, + f"baseline file {rule.baseline.file} does not exist", + "create it", + "baseline", + ) + elif baseline_entries(path) != rule.baseline.count: + v( + rule.id, + f"baseline count is {rule.baseline.count} but {rule.baseline.file} lists {baseline_entries(path)}", + "make the count match the file", + "baseline", + ) + + if rule.review_by is not None and rule.review_by < today: + v( + rule.id, + f"review_by {rule.review_by} has passed", + "review the rule, then remove or extend review_by", + "expiry", + ) + for exception in rule.exceptions: + if exception.expires < today: + v( + rule.id, + f"exception for {exception.path} expired on {exception.expires}", + "remove the exception (fix the code) or extend it with a reason", + "expiry", + ) + if not any(root.glob(exception.path)): + v( + rule.id, + f"exception path '{exception.path}' matches no files", + "remove the stale exception", + ) + + return violations diff --git a/scripts/rules/workflows.py b/scripts/rules/workflows.py new file mode 100644 index 000000000..2c3555f9d --- /dev/null +++ b/scripts/rules/workflows.py @@ -0,0 +1,191 @@ +"""Minimal reader for GitHub Actions workflow files. + +The standard library has no YAML parser and the guard must run without +dependencies, so this reads only the subset these repositories use: block-style +``jobs:`` mappings, ``name:``/``uses:``/``run:`` keys (including ``|`` and ``>`` +block scalars), and ``strategy.matrix`` lists. Anything outside that subset is +reported as unsupported rather than guessed at. + +It answers two questions: does job X exist, and does its run text (or reusable +workflow reference) contain a given invocation string? +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass, field + +_KEY = re.compile(r"^(\s*)(-\s+)?([A-Za-z0-9_-]+):\s*(.*)$") +_LIST_ITEM = re.compile(r"^(\s*)-\s+(.*)$") +_UNSUPPORTED = re.compile(r"(<<:|:\s*[&*][A-Za-z0-9_-]+\s*$)") + + +@dataclass(frozen=True) +class JobBlock: + id: str + name: str + run_text: str + uses: str + step_names: tuple[str, ...] + step_uses: tuple[str, ...] = () + matrix: dict[str, tuple[str, ...]] = field(default_factory=dict) + + def invokes(self, invocation: str) -> bool: + """Whether the job's commands or actions mention ``invocation``.""" + return ( + invocation in self.run_text + or invocation in self.uses + or any(invocation in action for action in self.step_uses) + ) + + +def _strip_comment(line: str) -> str: + in_single = in_double = False + for index, char in enumerate(line): + if char == "'" and not in_double: + in_single = not in_single + elif char == '"' and not in_single: + in_double = not in_double + elif char == "#" and not in_single and not in_double: + if index == 0 or line[index - 1].isspace(): + return line[:index].rstrip() + return line.rstrip() + + +def _unquote(value: str) -> str: + value = value.strip() + if len(value) >= 2 and value[0] == value[-1] and value[0] in "'\"": + return value[1:-1] + return value + + +def _indent(line: str) -> int: + return len(line) - len(line.lstrip(" ")) + + +def _inline_list(value: str) -> tuple[str, ...]: + inner = value.strip()[1:-1] + return tuple(_unquote(item) for item in inner.split(",") if item.strip()) + + +def _parse_job(job_id: str, lines: list[str], job_indent: int) -> JobBlock: + name = "" + uses = "" + step_uses: list[str] = [] + runs: list[str] = [] + steps: list[str] = [] + matrix: dict[str, tuple[str, ...]] = {} + body_indent = None + index = 0 + while index < len(lines): + line = lines[index] + match = _KEY.match(line) + if not match: + index += 1 + continue + leading, dash, key, value = match.groups() + key_column = len(leading) + (len(dash) if dash else 0) + if body_indent is None and not dash: + body_indent = key_column + top_level = not dash and key_column == body_indent + + if key == "name" and top_level: + name = _unquote(value) + elif key == "name": + steps.append(_unquote(value)) + elif key == "uses" and top_level: + uses = _unquote(value) + elif key == "uses": + step_uses.append(_unquote(value)) + elif key == "run": + if value.strip()[:1] in ("|", ">"): + block = [] + index += 1 + while index < len(lines) and ( + not lines[index].strip() or _indent(lines[index]) > key_column + ): + block.append(lines[index].strip()) + index += 1 + runs.append("\n".join(block)) + continue + runs.append(_unquote(value)) + elif key == "matrix" and not value: + index += 1 + while index < len(lines) and ( + not lines[index].strip() or _indent(lines[index]) > key_column + ): + entry = _KEY.match(lines[index]) + if entry and not entry.group(2): + entry_key, entry_value = entry.group(3), entry.group(4) + entry_column = len(entry.group(1)) + if entry_value.startswith("["): + matrix[entry_key] = _inline_list(entry_value) + index += 1 + continue + items = [] + index += 1 + while index < len(lines) and ( + not lines[index].strip() + or _indent(lines[index]) >= entry_column + ): + item = _LIST_ITEM.match(lines[index]) + if not item or _KEY.match(lines[index]): + break + items.append(_unquote(item.group(2))) + index += 1 + matrix[entry_key] = tuple(items) + continue + index += 1 + continue + index += 1 + return JobBlock( + id=job_id, + name=name or job_id, + run_text="\n".join(runs), + uses=uses, + step_uses=tuple(step_uses), + step_names=tuple(steps), + matrix=matrix, + ) + + +def parse_jobs(text: str) -> tuple[dict[str, JobBlock], list[str]]: + """Return the jobs in a workflow, plus descriptions of unsupported constructs.""" + lines = [_strip_comment(line) for line in text.splitlines()] + problems = [ + f"line {number}: YAML anchors/aliases/merge keys are not supported" + for number, line in enumerate(lines, 1) + if _UNSUPPORTED.search(line) + ] + + start = next( + (i for i, line in enumerate(lines) if re.match(r"^jobs:\s*$", line)), None + ) + if start is None: + if any(re.match(r"^jobs:\s*\S", line) for line in lines): + problems.append("flow-style 'jobs:' mapping is not supported") + return {}, problems + + body: list[str] = [] + for line in lines[start + 1 :]: + if line.strip() and _indent(line) == 0: + break + body.append(line) + + job_indent = next((_indent(line) for line in body if line.strip()), 0) + jobs: dict[str, JobBlock] = {} + current: str | None = None + current_lines: list[str] = [] + for line in body: + match = re.match(rf"^ {{{job_indent}}}([A-Za-z0-9_-]+):\s*$", line) + if match and _indent(line) == job_indent: + if current: + jobs[current] = _parse_job(current, current_lines, job_indent) + current, current_lines = match.group(1), [] + elif line.strip() and _indent(line) == job_indent: + problems.append(f"unexpected line at job level: {line.strip()!r}") + elif current: + current_lines.append(line) + if current: + jobs[current] = _parse_job(current, current_lines, job_indent) + return jobs, problems From 868f1f5a900a7841f409f3bb12fe7bcccc1ac544 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Mon, 12 Oct 2026 19:12:42 +0400 Subject: [PATCH 2/6] Run the rules registry guard in CI Adds a quality guard runner and a dependency-free quality-guards PR job that runs the tool's unittest suite and the guards, with full history so baselines can be compared with the base branch. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/pr.yml | 21 +++++++++++++++++++ scripts/run_quality_guards.py | 39 +++++++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+) create mode 100644 scripts/run_quality_guards.py diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index bf8491c31..4089aec81 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -77,6 +77,27 @@ jobs: fi done + quality-guards: + name: Quality guards + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v6 + with: + # The rules registry guard compares baselines with the base branch. + fetch-depth: 0 + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: '3.13' + + - name: Test the rules registry tool + run: python -m unittest discover -s scripts/rules/tests -t . + + - name: Run quality guards + run: python scripts/run_quality_guards.py + docker-build: name: Docker build needs: [test, lint] # Only run if tests and linting pass diff --git a/scripts/run_quality_guards.py b/scripts/run_quality_guards.py new file mode 100644 index 000000000..0338c10d1 --- /dev/null +++ b/scripts/run_quality_guards.py @@ -0,0 +1,39 @@ +"""Run repository quality guards.""" + +from __future__ import annotations + +import sys +from collections.abc import Callable +from pathlib import Path + + +REPO_ROOT = Path(__file__).resolve().parents[1] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + +from scripts.rules import guard as rules_guard # noqa: E402 + + +Guard = tuple[str, Callable[[], list[str]]] + +GUARDS: tuple[Guard, ...] = (("rules-registry", rules_guard.check),) + + +def main() -> int: + failed = False + for name, check in GUARDS: + violations = check() + if not violations: + print(f"{name}: passed") + continue + + failed = True + print(f"{name}: failed") + for violation in violations: + print(f" - {violation}") + + return 1 if failed else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From 13429bd8735b10257db57cac97a7067c894daf9c Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Mon, 12 Oct 2026 19:12:42 +0400 Subject: [PATCH 3/6] Register existing simulation rules in the rules registry Registers the Stage 12, deployment-asset and OpenAPI golden tests, ruff format and CI jobs, plus rules stated only in docs as doc-only. Adds a proof fixture and test for sim.stage12.no-schema-ddl. Co-Authored-By: Claude Opus 5.5 --- .../tests/test_stage12_schema_authority.py | 9 + .../bad_stage12_runtime.py | 7 + .../good_stage12_runtime.py | 7 + rules/registry.toml | 652 ++++++++++++++++++ 4 files changed, 675 insertions(+) create mode 100644 rules/fixtures/sim.stage12.no-schema-ddl/bad_stage12_runtime.py create mode 100644 rules/fixtures/sim.stage12.no-schema-ddl/good_stage12_runtime.py create mode 100644 rules/registry.toml diff --git a/projects/policyengine-simulation-entry/tests/test_stage12_schema_authority.py b/projects/policyengine-simulation-entry/tests/test_stage12_schema_authority.py index 8002614f7..abc4a3ea1 100644 --- a/projects/policyengine-simulation-entry/tests/test_stage12_schema_authority.py +++ b/projects/policyengine-simulation-entry/tests/test_stage12_schema_authority.py @@ -31,6 +31,15 @@ def test_simulation_runtimes_contain_no_schema_ddl() -> None: assert violations == [] +def test_rule_proof_stage12_no_schema_ddl() -> None: + """Proof for rule sim.stage12.no-schema-ddl (rules/registry.toml).""" + fixtures = REPOSITORY_ROOT / "rules/fixtures/sim.stage12.no-schema-ddl" + bad = (fixtures / "bad_stage12_runtime.py").read_text(encoding="utf-8") + good = (fixtures / "good_stage12_runtime.py").read_text(encoding="utf-8") + assert DDL_PATTERN.search(bad) is not None + assert DDL_PATTERN.search(good) is None + + def test_simulation_repository_has_no_schema_migration_chain() -> None: assert not list(REPOSITORY_ROOT.glob("alembic*.ini")) assert not list(REPOSITORY_ROOT.rglob("alembic/versions")) diff --git a/rules/fixtures/sim.stage12.no-schema-ddl/bad_stage12_runtime.py b/rules/fixtures/sim.stage12.no-schema-ddl/bad_stage12_runtime.py new file mode 100644 index 000000000..f6b4b7262 --- /dev/null +++ b/rules/fixtures/sim.stage12.no-schema-ddl/bad_stage12_runtime.py @@ -0,0 +1,7 @@ +"""Proof fixture: a Stage 12 runtime that creates its own table. The rule must reject it.""" + +from sqlalchemy import text + + +def ensure_comparison_table(connection) -> None: + connection.execute(text("CREATE TABLE stage12_evaluation_reports (id uuid)")) diff --git a/rules/fixtures/sim.stage12.no-schema-ddl/good_stage12_runtime.py b/rules/fixtures/sim.stage12.no-schema-ddl/good_stage12_runtime.py new file mode 100644 index 000000000..6dddf8588 --- /dev/null +++ b/rules/fixtures/sim.stage12.no-schema-ddl/good_stage12_runtime.py @@ -0,0 +1,7 @@ +"""Proof fixture: a Stage 12 runtime that writes rows with typed DML. The rule must accept it.""" + +from sqlalchemy import insert + + +def record_report(connection, reports_table, report_id: str) -> None: + connection.execute(insert(reports_table).values(id=report_id)) diff --git a/rules/registry.toml b/rules/registry.toml new file mode 100644 index 000000000..eeea41cee --- /dev/null +++ b/rules/registry.toml @@ -0,0 +1,652 @@ +# Rules registry for policyengine-sim-api. +# +# Every rule that governs this repository is declared here: what it says, how +# mechanically it is enforced, where it runs in CI, and whether a checker has +# been proven to reject a bad example. `python -m scripts.rules guard` keeps this +# file honest; `python -m scripts.rules render` regenerates +# docs/generated/rules.md and the Rules block in AGENTS.md. +# +# How to add or change a rule: docs/engineering/skills/rules-registry.md + +schema_version = 1 +repo = "policyengine-sim-api" +tool_version = "0.1.0" + +[[category]] +id = "boundaries" +title = "Service and package boundaries" + +[[category]] +id = "stage12" +title = "Stage 12 persistence" + +[[category]] +id = "deployment" +title = "Deployment pipeline" + +[[category]] +id = "contracts" +title = "API contracts and integration" + +[[category]] +id = "observability" +title = "Observability" + +[[category]] +id = "quality" +title = "Code quality and tests" + +[[category]] +id = "process" +title = "Process and repository hygiene" + +[discovery] +workflows = [".github/workflows/pr.yml", ".github/workflows/pr-image-smoke.yml"] +guard_test_globs = [ + "projects/*/tests/test_*boundar*.py", + "libs/*/tests/test_*boundar*.py", + "projects/*/tests/test_*schema_authority*.py", + "projects/policyengine-simulation-entry/tests/test_deployment_assets.py", + "projects/policyengine-simulation-gateway/tests/test_openapi_golden.py", +] + +# --- Service and package boundaries ---------------------------------------- + +[[rule]] +id = "sim.boundaries.gateway-executor-decoupled" +category = "boundaries" +statement = "The gateway and executor never import each other; they communicate only through the models and state helpers in policyengine-simulation-contract." +rationale = "libs/policyengine-simulation-contract/README.md" +scope = ["projects/policyengine-simulation-gateway/src/**/*.py", "projects/policyengine-simulation-executor/src/**/*.py"] +tier = "mechanical" +status = "doc-only" +autofix = false +[rule.enforcer] +kind = "none" + +[[rule]] +id = "sim.boundaries.unit-tests-need-no-services" +category = "boundaries" +statement = "Unit tests never require real network credentials, Modal, Hugging Face, GCP or deployed services; those seams are mocked." +rationale = "docs/engineering/skills/testing.md#dependency-boundaries" +scope = ["projects/*/tests/**/*.py", "libs/*/tests/**/*.py"] +tier = "mechanical" +status = "doc-only" +autofix = false +[rule.enforcer] +kind = "none" + +# --- Stage 12 persistence -------------------------------------------------- + +[[rule]] +id = "sim.stage12.no-schema-ddl" +category = "stage12" +statement = "Simulation runtimes and the Stage 12 persistence library contain no schema DDL (CREATE, ALTER, DROP or TRUNCATE of tables, types, schemas or databases, or create_all); policyengine-api owns the schema." +rationale = "libs/policyengine-stage12-persistence/README.md" +scope = ["libs/policyengine-stage12-persistence/src/**/*.py", "projects/policyengine-simulation-entry/src/**/*.py", "projects/policyengine-simulation-executor/src/**/*.py"] +tier = "mechanical" +status = "enforced" +autofix = false +[rule.enforcer] +kind = "pytest" +project = "projects/policyengine-simulation-entry" +nodes = ["tests/test_stage12_schema_authority.py::test_simulation_runtimes_contain_no_schema_ddl"] +[[rule.ci]] +workflow = ".github/workflows/pr.yml" +job = "test" +invocation = "uv run pytest tests/" +[rule.proof] +bad_fixture = "rules/fixtures/sim.stage12.no-schema-ddl/bad_stage12_runtime.py" +good_fixture = "rules/fixtures/sim.stage12.no-schema-ddl/good_stage12_runtime.py" +project = "projects/policyengine-simulation-entry" +test = "tests/test_stage12_schema_authority.py::test_rule_proof_stage12_no_schema_ddl" + +[[rule]] +id = "sim.stage12.no-migration-chain" +category = "stage12" +statement = "This repository has no schema migration chain (no Alembic); Stage 12 tables are migrated by policyengine-api." +rationale = "libs/policyengine-stage12-persistence/README.md" +scope = ["projects/*/src/**/*.py", "libs/*/src/**/*.py"] +tier = "mechanical" +status = "enforced" +autofix = false +[rule.enforcer] +kind = "pytest" +project = "projects/policyengine-simulation-entry" +nodes = ["tests/test_stage12_schema_authority.py::test_simulation_repository_has_no_schema_migration_chain"] +[[rule.ci]] +workflow = ".github/workflows/pr.yml" +job = "test" +invocation = "uv run pytest tests/" + +[[rule]] +id = "sim.stage12.no-production-user-associations" +category = "stage12" +statement = "Stage 12 runtime modules never create production user associations (user_simulations or user_reports)." +rationale = "libs/policyengine-stage12-persistence/README.md" +scope = ["projects/*/src/**/stage12_*.py", "libs/policyengine-stage12-persistence/src/**/*.py"] +tier = "mechanical" +status = "enforced" +autofix = false +[rule.enforcer] +kind = "pytest" +project = "projects/policyengine-simulation-entry" +nodes = ["tests/test_stage12_schema_authority.py::test_stage12_runtime_cannot_create_production_user_associations"] +[[rule.ci]] +workflow = ".github/workflows/pr.yml" +job = "test" +invocation = "uv run pytest tests/" + +[[rule]] +id = "sim.stage12.persistence-typed-dml-only" +category = "stage12" +statement = "The Stage 12 persistence library and runtime use typed SQLAlchemy Core DML only: no raw SQL text, driver-level SQL or schema operations." +rationale = "libs/policyengine-stage12-persistence/README.md" +scope = ["libs/policyengine-stage12-persistence/src/**/*.py"] +tier = "mechanical" +status = "enforced" +autofix = false +[rule.enforcer] +kind = "pytest" +project = "libs/policyengine-stage12-persistence" +nodes = ["tests/test_source_boundaries.py::test_persistence_package_contains_no_raw_sql_or_schema_operations"] +[[rule.ci]] +workflow = ".github/workflows/pr.yml" +job = "test" +invocation = "uv run pytest tests/" + +[[rule]] +id = "sim.stage12.deployment-uses-api-schema" +category = "stage12" +statement = "Stage 12 deployment uses the API-owned schema directly, and the Stage 12 runtime it deploys uses typed DML without raw SQL or schema changes." +rationale = "libs/policyengine-stage12-persistence/README.md" +scope = ["projects/policyengine-simulation-entry/src/**/*.py"] +tier = "mechanical" +status = "enforced" +autofix = false +[rule.enforcer] +kind = "pytest" +project = "projects/policyengine-simulation-entry" +nodes = [ + "tests/test_deployment_assets.py::test_stage12_deployment_uses_the_api_owned_schema_directly", + "tests/test_deployment_assets.py::test_stage12_runtime_uses_typed_dml_without_raw_sql_or_schema_changes", +] +[[rule.ci]] +workflow = ".github/workflows/pr.yml" +job = "test" +invocation = "uv run pytest tests/" + +[[rule]] +id = "sim.stage12.deployment-scope" +category = "stage12" +statement = "Main-branch deploys roll out Stage 12 in both environments; a Stage 12-only deploy runs candidate tests and the requested promotion but never redeploys existing Modal resources; the Stage 12 worker can request a Google OIDC token." +rationale = "docs/operations" +scope = [".github/workflows/*.yml"] +tier = "mechanical" +status = "enforced" +autofix = false +[rule.enforcer] +kind = "pytest" +project = "projects/policyengine-simulation-entry" +nodes = [ + "tests/test_deployment_assets.py::test_main_deployment_automatically_deploys_stage12_in_both_environments", + "tests/test_deployment_assets.py::test_stage12_only_deployment_runs_candidate_tests_and_requested_promotion", + "tests/test_deployment_assets.py::test_stage12_only_deployment_cannot_redeploy_existing_modal_resources", + "tests/test_deployment_assets.py::test_stage12_worker_deployment_can_request_a_google_oidc_token", +] +[[rule.ci]] +workflow = ".github/workflows/pr.yml" +job = "test" +invocation = "uv run pytest tests/" + +[[rule]] +id = "sim.stage12.production-lock-has-db-runtime" +category = "stage12" +statement = "The production lock file includes the direct Stage 12 database runtime dependencies." +rationale = "libs/policyengine-stage12-persistence/README.md" +scope = ["projects/policyengine-simulation-entry/uv.lock"] +tier = "mechanical" +status = "enforced" +autofix = false +[rule.enforcer] +kind = "pytest" +project = "projects/policyengine-simulation-entry" +nodes = ["tests/test_deployment_assets.py::test_production_lock_contains_the_direct_stage12_database_runtime"] +[[rule.ci]] +workflow = ".github/workflows/pr.yml" +job = "test" +invocation = "uv run pytest tests/" + +# --- Deployment pipeline --------------------------------------------------- + +[[rule]] +id = "sim.deploy.shell-scripts-valid" +category = "deployment" +statement = "Deployment shell scripts have valid shell syntax." +rationale = "docs/operations" +scope = [".github/scripts/*.sh"] +tier = "mechanical" +status = "enforced" +autofix = false +[rule.enforcer] +kind = "pytest" +project = "projects/policyengine-simulation-entry" +nodes = ["tests/test_deployment_assets.py::test_deployment_scripts_have_valid_shell_syntax"] +[[rule.ci]] +workflow = ".github/workflows/pr.yml" +job = "test" +invocation = "uv run pytest tests/" + +[[rule]] +id = "sim.deploy.gcloud-without-terraform" +category = "deployment" +statement = "Deployment uses the gcloud-based workflow, not Terraform." +rationale = "docs/operations" +scope = [".github/workflows/*.yml"] +tier = "mechanical" +status = "enforced" +autofix = false +[rule.enforcer] +kind = "pytest" +project = "projects/policyengine-simulation-entry" +nodes = ["tests/test_deployment_assets.py::test_deployment_uses_gcloud_workflow_without_terraform"] +[[rule.ci]] +workflow = ".github/workflows/pr.yml" +job = "test" +invocation = "uv run pytest tests/" + +[[rule]] +id = "sim.deploy.workflow-shell-safety" +category = "deployment" +statement = "Cloud Run deployment never interpolates GitHub expressions into shell, escapes environment values, and pins secret versions." +rationale = "docs/operations" +scope = [".github/workflows/*.yml"] +tier = "mechanical" +status = "enforced" +autofix = false +[rule.enforcer] +kind = "pytest" +project = "projects/policyengine-simulation-entry" +nodes = [ + "tests/test_deployment_assets.py::test_cloud_run_deployment_does_not_interpolate_github_values_in_shell", + "tests/test_deployment_assets.py::test_cloud_run_deployment_escapes_environment_and_pins_secret_versions", +] +[[rule.ci]] +workflow = ".github/workflows/pr.yml" +job = "test" +invocation = "uv run pytest tests/" + +[[rule]] +id = "sim.deploy.revision-tags-valid" +category = "deployment" +statement = "Cloud Run revision tags are valid on a workflow's first run and fit the existing service names." +rationale = "docs/operations" +scope = [".github/workflows/*.yml"] +tier = "mechanical" +status = "enforced" +autofix = false +[rule.enforcer] +kind = "pytest" +project = "projects/policyengine-simulation-entry" +nodes = [ + "tests/test_deployment_assets.py::test_cloud_run_revision_tags_are_valid_on_the_first_workflow_run", + "tests/test_deployment_assets.py::test_revision_tags_fit_existing_cloud_run_service_names", +] +[[rule.ci]] +workflow = ".github/workflows/pr.yml" +job = "test" +invocation = "uv run pytest tests/" + +[[rule]] +id = "sim.deploy.promotion-order-explicit" +category = "deployment" +statement = "Full-stack promotion order is explicit, and the reusable deployment rejects unexpectedly skipped phases." +rationale = "docs/operations" +scope = [".github/workflows/*.yml"] +tier = "mechanical" +status = "enforced" +autofix = false +[rule.enforcer] +kind = "pytest" +project = "projects/policyengine-simulation-entry" +nodes = [ + "tests/test_deployment_assets.py::test_full_stack_promotion_order_is_explicit", + "tests/test_deployment_assets.py::test_reusable_deployment_rejects_unexpected_skipped_phases", +] +[[rule.ci]] +workflow = ".github/workflows/pr.yml" +job = "test" +invocation = "uv run pytest tests/" + +[[rule]] +id = "sim.deploy.smoke-tests-strict" +category = "deployment" +statement = "Candidate and stable smoke tests reject a revision mismatch without retrying, and fail if the previous revision does not converge." +rationale = "docs/operations" +scope = [".github/scripts/*.sh"] +tier = "mechanical" +status = "enforced" +autofix = false +[rule.enforcer] +kind = "pytest" +project = "projects/policyengine-simulation-entry" +nodes = [ + "tests/test_deployment_assets.py::test_candidate_smoke_rejects_a_revision_mismatch_without_retrying", + "tests/test_deployment_assets.py::test_stable_smoke_rejects_an_unexpected_revision_without_retrying", + "tests/test_deployment_assets.py::test_stable_smoke_retries_the_previous_revision_until_target_is_served", + "tests/test_deployment_assets.py::test_stable_smoke_fails_when_the_previous_revision_does_not_converge", +] +[[rule.ci]] +workflow = ".github/workflows/pr.yml" +job = "test" +invocation = "uv run pytest tests/" + +[[rule]] +id = "sim.deploy.traffic-changes-exact" +category = "deployment" +statement = "Traffic changes target an exact, validated revision, are reversible, and refuse to run over an intervening promotion." +rationale = "docs/operations" +scope = [".github/scripts/*.sh"] +tier = "mechanical" +status = "enforced" +autofix = false +[rule.enforcer] +kind = "pytest" +project = "projects/policyengine-simulation-entry" +nodes = [ + "tests/test_deployment_assets.py::test_traffic_changes_are_exact_revision_validated_and_reversible", + "tests/test_deployment_assets.py::test_traffic_change_refuses_an_intervening_promotion", +] +[[rule.ci]] +workflow = ".github/workflows/pr.yml" +job = "test" +invocation = "uv run pytest tests/" + +[[rule]] +id = "sim.deploy.container-context-minimal" +category = "deployment" +statement = "Container builds exclude local environments and unrelated projects, and the entrypoint container builds for the requested target platform." +rationale = "docs/operations" +scope = ["projects/policyengine-simulation-entry/Dockerfile"] +tier = "mechanical" +status = "enforced" +autofix = false +[rule.enforcer] +kind = "pytest" +project = "projects/policyengine-simulation-entry" +nodes = [ + "tests/test_deployment_assets.py::test_container_context_excludes_local_environments_and_unrelated_projects", + "tests/test_deployment_assets.py::test_entrypoint_container_uses_requested_target_platform", +] +[[rule.ci]] +workflow = ".github/workflows/pr.yml" +job = "test" +invocation = "uv run pytest tests/" + +[[rule]] +id = "sim.deploy.docker-images-build" +category = "deployment" +statement = "The simulation entry and executor Docker images build." +rationale = "docs/operations" +scope = ["projects/policyengine-simulation-entry/Dockerfile", "projects/policyengine-simulation-executor/Dockerfile"] +tier = "mechanical" +status = "enforced" +autofix = false +[rule.enforcer] +kind = "workflow-job" +[[rule.ci]] +workflow = ".github/workflows/pr.yml" +job = "docker-build" +invocation = "docker/build-push-action" + +[[rule]] +id = "sim.deploy.modal-image-smoke" +category = "deployment" +statement = "App entrypoints import inside the real Modal images before merge, whenever a library baked into an image changes." +rationale = ".github/workflows/pr-image-smoke.yml" +scope = ["projects/policyengine-simulation-executor/src/modal/**/*.py"] +tier = "mechanical" +status = "enforced" +autofix = false +[rule.enforcer] +kind = "workflow-job" +[[rule.ci]] +workflow = ".github/workflows/pr-image-smoke.yml" +job = "image-smoke" +invocation = "modal-image-smoke.sh" + +[[rule]] +id = "sim.deploy.image-smoke-covers-every-image-lib" +category = "deployment" +statement = "The image smoke path filter includes every library mounted into a Modal image; libs/policyengine-stage12-persistence is currently missing, so Stage 12-only changes skip the smoke." +rationale = ".github/workflows/pr-image-smoke.yml" +scope = [".github/workflows/pr-image-smoke.yml"] +tier = "mechanical" +status = "doc-only" +autofix = false +[rule.enforcer] +kind = "none" + +# --- API contracts and integration ---------------------------------------- + +[[rule]] +id = "sim.contracts.gateway-openapi-golden" +category = "contracts" +statement = "The gateway's OpenAPI spec matches the checked-in golden file; regenerate it deliberately when the API changes." +rationale = "projects/policyengine-simulation-gateway/tests/test_openapi_golden.py" +scope = ["projects/policyengine-simulation-gateway/src/**/*.py"] +tier = "mechanical" +status = "enforced" +autofix = false +[rule.enforcer] +kind = "pytest" +project = "projects/policyengine-simulation-gateway" +nodes = ["tests/test_openapi_golden.py::test_openapi_spec_matches_golden"] +[[rule.ci]] +workflow = ".github/workflows/pr.yml" +job = "test" +invocation = "uv run pytest tests/" + +[[rule]] +id = "sim.contracts.regenerate-clients" +category = "contracts" +statement = "When public API schemas change, regenerate the clients and run the generated-client integration tests." +rationale = "docs/engineering/skills/testing.md#dependency-boundaries" +scope = ["projects/policyengine-simulation-entry/src/**/*.py"] +tier = "mechanical" +status = "doc-only" +autofix = false +[rule.enforcer] +kind = "none" + +[[rule]] +id = "sim.contracts.local-integration" +category = "contracts" +statement = "Generated-client integration tests pass against the locally built services." +rationale = "docs/engineering/skills/testing.md#common-commands" +scope = ["projects/policyengine-apis-integ/tests/**/*.py"] +tier = "mechanical" +status = "enforced" +autofix = false +[rule.enforcer] +kind = "workflow-job" +[[rule.ci]] +workflow = ".github/workflows/pr.yml" +job = "test-integration" +invocation = "uv run pytest tests/" + +[[rule]] +id = "sim.contracts.full-integration-beta-only" +category = "contracts" +statement = "The complete integration suite runs against beta deployments only, never production." +rationale = "docs/engineering/skills/testing.md" +scope = [".github/workflows/*.yml"] +tier = "mechanical" +status = "enforced" +autofix = false +[rule.enforcer] +kind = "pytest" +project = "projects/policyengine-simulation-entry" +nodes = ["tests/test_deployment_assets.py::test_complete_integration_suite_is_configured_for_beta_only"] +[[rule.ci]] +workflow = ".github/workflows/pr.yml" +job = "test" +invocation = "uv run pytest tests/" + +[[rule]] +id = "sim.contracts.clients-publish-deployed-commit" +category = "contracts" +statement = "Client publication checks out the commit that was actually deployed." +rationale = "docs/operations" +scope = [".github/workflows/*.yml"] +tier = "mechanical" +status = "enforced" +autofix = false +[rule.enforcer] +kind = "pytest" +project = "projects/policyengine-simulation-entry" +nodes = ["tests/test_deployment_assets.py::test_client_publication_checks_out_the_deployed_commit"] +[[rule.ci]] +workflow = ".github/workflows/pr.yml" +job = "test" +invocation = "uv run pytest tests/" + +# --- Observability --------------------------------------------------------- + +[[rule]] +id = "sim.observability.id-is-diagnostic-only" +category = "observability" +statement = "observability_id has diagnostic meaning only: never use it for idempotency, authorization, database identity, routing or business logic." +rationale = "docs/engineering/skills/observability.md#identifier-ownership" +scope = ["projects/*/src/**/*.py", "libs/*/src/**/*.py"] +tier = "mechanical" +status = "doc-only" +autofix = false +[rule.enforcer] +kind = "none" + +[[rule]] +id = "sim.observability.header-transport-only" +category = "observability" +statement = "observability_id travels only in the X-PolicyEngine-Observability-Id header, never in a simulation JSON body, and polling never invents one." +rationale = "docs/engineering/skills/observability.md#http-transport" +scope = ["projects/*/src/**/*.py"] +tier = "mechanical" +status = "doc-only" +autofix = false +[rule.enforcer] +kind = "none" + +[[rule]] +id = "sim.observability.registered-stages-only" +category = "observability" +statement = "Span names come from the stage plans in policyengine_simulation_observability.stages; runtime code never adds span-name string literals." +rationale = "docs/engineering/skills/observability.md#runtime-stages" +scope = ["projects/*/src/**/*.py"] +tier = "mechanical" +status = "doc-only" +autofix = false +[rule.enforcer] +kind = "none" + +[[rule]] +id = "sim.observability.failures-never-break-requests" +category = "observability" +statement = "Invalid observability configuration fails at build or deploy time, and telemetry failures at runtime never fail a request or computation." +rationale = "docs/engineering/skills/observability.md#failure-behavior" +scope = ["libs/policyengine-simulation-observability/src/**/*.py"] +tier = "mechanical" +status = "doc-only" +autofix = false +[rule.enforcer] +kind = "none" + +# --- Code quality and tests ------------------------------------------------ + +[[rule]] +id = "sim.quality.ruff-format-src" +category = "quality" +statement = "Source code under projects/*/src and libs/*/src is formatted with ruff format." +rationale = "docs/engineering/skills/testing.md#common-commands" +scope = ["projects/*/src/**/*.py", "libs/*/src/**/*.py"] +tier = "mechanical" +status = "enforced" +autofix = true +[rule.enforcer] +kind = "ruff" +mode = "format" +[[rule.ci]] +workflow = ".github/workflows/pr.yml" +job = "lint" +invocation = "ruff format --check" + +[[rule]] +id = "sim.quality.ruff-format-tests" +category = "quality" +statement = "Test code is formatted with ruff format too; CI currently checks only src directories." +rationale = "docs/engineering/skills/testing.md#common-commands" +scope = ["projects/*/tests/**/*.py", "libs/*/tests/**/*.py"] +tier = "mechanical" +status = "doc-only" +autofix = true +[rule.enforcer] +kind = "none" + +[[rule]] +id = "sim.quality.tests-pass" +category = "quality" +statement = "Each project's unit tests pass in its own locked environment." +rationale = "docs/engineering/skills/testing.md#canonical-layout" +scope = ["projects/*/tests/**/*.py", "libs/*/tests/**/*.py"] +tier = "mechanical" +status = "enforced" +autofix = false +[rule.enforcer] +kind = "workflow-job" +[[rule.ci]] +workflow = ".github/workflows/pr.yml" +job = "test" +invocation = "uv run pytest tests/" + +[[rule]] +id = "sim.quality.every-project-tested" +category = "quality" +statement = "Every project and library runs its tests in PR CI; libs/policyengine-fastapi is not yet in the test matrix." +rationale = "docs/engineering/skills/testing.md#canonical-layout" +scope = ["libs/policyengine-fastapi/**/*.py"] +tier = "mechanical" +status = "doc-only" +autofix = false +[rule.enforcer] +kind = "none" + +# --- Process and repository hygiene ---------------------------------------- + +[[rule]] +id = "sim.process.rules-registry" +category = "process" +statement = "Every rule is registered in rules/registry.toml, is enforced where it claims to be, and its rendered views are current." +rationale = "docs/engineering/skills/rules-registry.md" +scope = ["rules/registry.toml"] +tier = "mechanical" +status = "enforced" +autofix = false +[rule.enforcer] +kind = "quality-guard" +name = "rules-registry" +[[rule.ci]] +workflow = ".github/workflows/pr.yml" +job = "quality-guards" +invocation = "run_quality_guards.py" + +[[rule]] +id = "sim.process.same-repo-draft-prs" +category = "process" +statement = "PRs are opened as drafts from branches in PolicyEngine/policyengine-sim-api, never from forks, and start with 'Fixes #'." +rationale = "docs/engineering/skills/github-prs.md" +scope = [".github/pull_request_template.md"] +tier = "mechanical" +status = "doc-only" +autofix = false +[rule.enforcer] +kind = "none" From ecc6ed2cf79f18b25b0577da5300e8475e53fd4e Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Mon, 12 Oct 2026 19:12:42 +0400 Subject: [PATCH 4/6] Render rules and document the rules registry Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 75 +++++- changelog_entry.yaml | 2 + docs/engineering/skills/README.md | 2 + docs/engineering/skills/rules-registry.md | 138 +++++++++++ docs/generated/rules.md | 271 ++++++++++++++++++++++ 5 files changed, 486 insertions(+), 2 deletions(-) create mode 100644 docs/engineering/skills/rules-registry.md create mode 100644 docs/generated/rules.md diff --git a/AGENTS.md b/AGENTS.md index 0f75d5d42..3b1df1ae8 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -18,6 +18,10 @@ Before changing correlation identifiers, telemetry transport, spans, stage names, logging, or observability failure handling, read `docs/engineering/skills/observability.md`. +When adding, changing, or removing a rule, a guard test, a checker +configuration, or a CI job that enforces one, read +`docs/engineering/skills/rules-registry.md`. + ## Development - Use Python 3.13 and `uv`. @@ -59,6 +63,73 @@ same-repository draft PR. - The simulation service lives in `projects/policyengine-simulation-executor`. - API integration tests live in `projects/policyengine-apis-integ`. -- PR CI runs simulation unit tests, Ruff format checks, Docker build, and local - integration tests. +- PR CI runs simulation unit tests, Ruff format checks, quality guards + (including the rules registry), Docker build, and local integration tests. - There is currently no repository-wide changelog fragment requirement. + +## Rules + + + +Every rule for this repository lives in `rules/registry.toml` and is rendered, +with its enforcement and gaps, in `docs/generated/rules.md`. Blocking rules +fail CI. Doc-only rules are not checked automatically, so follow them yourself. +To add or change a rule, read `docs/engineering/skills/rules-registry.md`. + +### Service and package boundaries + +- `sim.boundaries.gateway-executor-decoupled`: The gateway and executor never import each other; they communicate only through the models and state helpers in policyengine-simulation-contract. (doc-only, not checked: follow it yourself) +- `sim.boundaries.unit-tests-need-no-services`: Unit tests never require real network credentials, Modal, Hugging Face, GCP or deployed services; those seams are mocked. (doc-only, not checked: follow it yourself) + +### Stage 12 persistence + +- `sim.stage12.deployment-scope`: Main-branch deploys roll out Stage 12 in both environments; a Stage 12-only deploy runs candidate tests and the requested promotion but never redeploys existing Modal resources; the Stage 12 worker can request a Google OIDC token. (blocking: pytest `projects/policyengine-simulation-entry/tests/test_deployment_assets.py`) +- `sim.stage12.deployment-uses-api-schema`: Stage 12 deployment uses the API-owned schema directly, and the Stage 12 runtime it deploys uses typed DML without raw SQL or schema changes. (blocking: pytest `projects/policyengine-simulation-entry/tests/test_deployment_assets.py`) +- `sim.stage12.no-migration-chain`: This repository has no schema migration chain (no Alembic); Stage 12 tables are migrated by policyengine-api. (blocking: pytest `projects/policyengine-simulation-entry/tests/test_stage12_schema_authority.py`) +- `sim.stage12.no-production-user-associations`: Stage 12 runtime modules never create production user associations (user_simulations or user_reports). (blocking: pytest `projects/policyengine-simulation-entry/tests/test_stage12_schema_authority.py`) +- `sim.stage12.no-schema-ddl`: Simulation runtimes and the Stage 12 persistence library contain no schema DDL (CREATE, ALTER, DROP or TRUNCATE of tables, types, schemas or databases, or create_all); policyengine-api owns the schema. (blocking: pytest `projects/policyengine-simulation-entry/tests/test_stage12_schema_authority.py`) +- `sim.stage12.persistence-typed-dml-only`: The Stage 12 persistence library and runtime use typed SQLAlchemy Core DML only: no raw SQL text, driver-level SQL or schema operations. (blocking: pytest `libs/policyengine-stage12-persistence/tests/test_source_boundaries.py`) +- `sim.stage12.production-lock-has-db-runtime`: The production lock file includes the direct Stage 12 database runtime dependencies. (blocking: pytest `projects/policyengine-simulation-entry/tests/test_deployment_assets.py`) + +### Deployment pipeline + +- `sim.deploy.container-context-minimal`: Container builds exclude local environments and unrelated projects, and the entrypoint container builds for the requested target platform. (blocking: pytest `projects/policyengine-simulation-entry/tests/test_deployment_assets.py`) +- `sim.deploy.docker-images-build`: The simulation entry and executor Docker images build. (blocking: CI job `docker-build`) +- `sim.deploy.gcloud-without-terraform`: Deployment uses the gcloud-based workflow, not Terraform. (blocking: pytest `projects/policyengine-simulation-entry/tests/test_deployment_assets.py`) +- `sim.deploy.image-smoke-covers-every-image-lib`: The image smoke path filter includes every library mounted into a Modal image; libs/policyengine-stage12-persistence is currently missing, so Stage 12-only changes skip the smoke. (doc-only, not checked: follow it yourself) +- `sim.deploy.modal-image-smoke`: App entrypoints import inside the real Modal images before merge, whenever a library baked into an image changes. (blocking: CI job `image-smoke`) +- `sim.deploy.promotion-order-explicit`: Full-stack promotion order is explicit, and the reusable deployment rejects unexpectedly skipped phases. (blocking: pytest `projects/policyengine-simulation-entry/tests/test_deployment_assets.py`) +- `sim.deploy.revision-tags-valid`: Cloud Run revision tags are valid on a workflow's first run and fit the existing service names. (blocking: pytest `projects/policyengine-simulation-entry/tests/test_deployment_assets.py`) +- `sim.deploy.shell-scripts-valid`: Deployment shell scripts have valid shell syntax. (blocking: pytest `projects/policyengine-simulation-entry/tests/test_deployment_assets.py`) +- `sim.deploy.smoke-tests-strict`: Candidate and stable smoke tests reject a revision mismatch without retrying, and fail if the previous revision does not converge. (blocking: pytest `projects/policyengine-simulation-entry/tests/test_deployment_assets.py`) +- `sim.deploy.traffic-changes-exact`: Traffic changes target an exact, validated revision, are reversible, and refuse to run over an intervening promotion. (blocking: pytest `projects/policyengine-simulation-entry/tests/test_deployment_assets.py`) +- `sim.deploy.workflow-shell-safety`: Cloud Run deployment never interpolates GitHub expressions into shell, escapes environment values, and pins secret versions. (blocking: pytest `projects/policyengine-simulation-entry/tests/test_deployment_assets.py`) + +### API contracts and integration + +- `sim.contracts.clients-publish-deployed-commit`: Client publication checks out the commit that was actually deployed. (blocking: pytest `projects/policyengine-simulation-entry/tests/test_deployment_assets.py`) +- `sim.contracts.full-integration-beta-only`: The complete integration suite runs against beta deployments only, never production. (blocking: pytest `projects/policyengine-simulation-entry/tests/test_deployment_assets.py`) +- `sim.contracts.gateway-openapi-golden`: The gateway's OpenAPI spec matches the checked-in golden file; regenerate it deliberately when the API changes. (blocking: pytest `projects/policyengine-simulation-gateway/tests/test_openapi_golden.py`) +- `sim.contracts.local-integration`: Generated-client integration tests pass against the locally built services. (blocking: CI job `test-integration`) +- `sim.contracts.regenerate-clients`: When public API schemas change, regenerate the clients and run the generated-client integration tests. (doc-only, not checked: follow it yourself) + +### Observability + +- `sim.observability.failures-never-break-requests`: Invalid observability configuration fails at build or deploy time, and telemetry failures at runtime never fail a request or computation. (doc-only, not checked: follow it yourself) +- `sim.observability.header-transport-only`: observability_id travels only in the X-PolicyEngine-Observability-Id header, never in a simulation JSON body, and polling never invents one. (doc-only, not checked: follow it yourself) +- `sim.observability.id-is-diagnostic-only`: observability_id has diagnostic meaning only: never use it for idempotency, authorization, database identity, routing or business logic. (doc-only, not checked: follow it yourself) +- `sim.observability.registered-stages-only`: Span names come from the stage plans in policyengine_simulation_observability.stages; runtime code never adds span-name string literals. (doc-only, not checked: follow it yourself) + +### Code quality and tests + +- `sim.quality.every-project-tested`: Every project and library runs its tests in PR CI; libs/policyengine-fastapi is not yet in the test matrix. (doc-only, not checked: follow it yourself) +- `sim.quality.ruff-format-src`: Source code under projects/*/src and libs/*/src is formatted with ruff format. (blocking: ruff format) +- `sim.quality.ruff-format-tests`: Test code is formatted with ruff format too; CI currently checks only src directories. (doc-only, not checked: follow it yourself) +- `sim.quality.tests-pass`: Each project's unit tests pass in its own locked environment. (blocking: CI job `test`) + +### Process and repository hygiene + +- `sim.process.rules-registry`: Every rule is registered in rules/registry.toml, is enforced where it claims to be, and its rendered views are current. (blocking: quality guard `rules-registry`) +- `sim.process.same-repo-draft-prs`: PRs are opened as drafts from branches in PolicyEngine/policyengine-sim-api, never from forks, and start with 'Fixes #'. (doc-only, not checked: follow it yourself) + + diff --git a/changelog_entry.yaml b/changelog_entry.yaml index 04d21c5ce..c0aac1ff4 100644 --- a/changelog_entry.yaml +++ b/changelog_entry.yaml @@ -1,5 +1,7 @@ - bump: patch changes: + added: + - Added a rules registry (rules/registry.toml) with a standard-library tool that validates it, checks every rule is enforced where it claims, and renders docs/generated/rules.md and the Rules block in AGENTS.md changed: - Added CODEOWNERS for the rule machinery, pinned CI's ruff to 0.15.22 with an explicit rule selection, and documented that agents never bypass branch rules - Ran the Modal image smoke for Stage 12 persistence changes and added policyengine-fastapi to the PR test matrix diff --git a/docs/engineering/skills/README.md b/docs/engineering/skills/README.md index 6b066a9a3..180b4799e 100644 --- a/docs/engineering/skills/README.md +++ b/docs/engineering/skills/README.md @@ -16,3 +16,5 @@ Current skills: - `observability.md`: identifier ownership, HTTP and Modal transport, persistence, registered runtime stages, trace boundaries, and failure isolation. +- `rules-registry.md`: how rules are registered in `rules/registry.toml`, + enforced, proven, and rendered into `docs/generated/rules.md`. diff --git a/docs/engineering/skills/rules-registry.md b/docs/engineering/skills/rules-registry.md new file mode 100644 index 000000000..ea54ccc2c --- /dev/null +++ b/docs/engineering/skills/rules-registry.md @@ -0,0 +1,138 @@ +# Rules Registry + +Read this before adding, changing, relaxing or removing a rule, a guard test, a +checker configuration (ruff, mypy, Pyright, Import Linter), or a CI job that +enforces one. + +Every rule that governs this repository lives in `rules/registry.toml`. The +registry says what each rule requires, how mechanically it is enforced, which +CI job runs it, and whether its checker has been proven to reject a bad example. +`python -m scripts.rules render` turns it into `docs/generated/rules.md` and the +Rules block in `AGENTS.md`; the `rules-registry` quality guard fails CI when the +registry, the checkers and the rendered views disagree. + +## Commands + +```bash +python -m scripts.rules validate # schema and file checks only +python -m scripts.rules render # rewrite rules.md and the AGENTS.md block +python -m scripts.rules guard --base-ref origin/ # everything CI checks +python -m unittest discover -s scripts/rules/tests -t . # the tool's own tests +``` + +All commands use only the standard library, so they run without installing the +project. + +## Adding or changing a rule + +A PR that adds, tightens or relaxes a rule contains all of these: + +1. **The check**: the checker, config or test change. +2. **A registry entry** (fields below). +3. **Proof, appropriate to the tier**: see "Proof". +4. **Zero violations, or a baseline** where the baseline policy allows one. +5. **CI wiring**: a `[[rule.ci]]` reference to the job that runs the check. +6. **Doc update**: explanatory skills docs cite the rule id instead of + restating the rule as unchecked prose. +7. **Regenerated outputs**: run `python -m scripts.rules render` and commit + `docs/generated/rules.md` and `AGENTS.md`. + +If a rule exists only in prose, register it with `status = "doc-only"` so the +gap is visible, rather than leaving it out. + +## Fields + +| Field | Meaning | +| --- | --- | +| `id` | Lowercase dotted id, e.g. `api.v2.services-no-sql`. Never reuse an id. | +| `category` | One of the `[[category]]` ids; controls grouping in the rendered views. | +| `statement` | One plain-English sentence an agent can follow. | +| `rationale` | Path (optionally `#anchor`) to the doc that explains the rule. | +| `scope` | Globs for the files the rule constrains; each must match a file. | +| `tier` | How mechanically the rule can be checked (see "Tiers"). | +| `status` | `enforced`, `ratcheted`, `advisory` or `doc-only`. | +| `autofix` | Whether the enforcer can fix violations itself (e.g. ruff format). | +| `review_by` | Optional date; the guard fails once it passes. | +| `[rule.enforcer]` | What checks the rule; `kind` plus kind-specific fields. | +| `[[rule.ci]]` | `workflow`, `job`, and an `invocation` string that must appear in that job's `run:` text (or its reusable-workflow `uses:`). | +| `[rule.proof]` | Optional; see "Proof". | +| `[rule.baseline]` | Only for `ratcheted` rules: `file` listing tolerated violations, and their `count`. | +| `[[rule.exception]]` | Temporary carve-out: `path`, `reason`, `expires`. | +| `[rule.downgrade]` | Required `reason` when a rule moves from enforced/ratcheted to advisory/doc-only. | + +`blocking` is derived, never declared: a rule blocks merges when its status is +`enforced` or `ratcheted` and its tier is not `judgment`. + +### Enforcer kinds + +| Kind | Fields | Resolved by the guard | +| --- | --- | --- | +| `pytest` | `nodes` (`path::test` or `path::Class::test`), optional `project` (sub-project directory the paths are relative to) | Each test exists, and the CI job's command collects it | +| `quality-guard` | `name`, optional `runner` | The name is in `GUARDS` in `scripts/run_quality_guards.py` | +| `ruff` | `mode` = `format`, `lint` (with `codes`) or `per-file-ignore` (with `pattern`, `codes`) | Codes are selected / the ignore entry exists | +| `mypy` | `paths` | Each path is in `[tool.mypy].files` | +| `workflow-job` | optional `step` | The CI job exists (the job itself is the check) | +| `pyright`, `import-linter` | `config` (+ `contract` for Import Linter) | The config file exists | +| `docs-guard` | `doc`, `marker` | The doc exists | +| `llm-review` | `prompt`, `model` | The prompt exists; judgment tier only | +| `none` | | Nothing; only for `doc-only` and `advisory` rules | + +## Tiers + +| Tier | Covers | Can block? | Proof | +| --- | --- | --- | --- | +| `mechanical` | A pattern is present or not | Yes | A bad fixture is rejected and a good fixture is accepted | +| `metric` | A number crosses a threshold | Yes | Fixtures just below and just above the threshold | +| `judgment` | Needs reading comprehension (LLM review plus human) | No | A labelled eval set with measured precision | + +Judgment rules graduate: when the same judgment finding recurs, turn it into a +mechanical rule. + +## Proof + +A proof shows the checker actually rejects what the rule forbids. Put fixtures +under `rules/fixtures//` and add a test named `test_rule_proof_` +that runs the rule's checker on them. + +| Tier | Proof fields | +| --- | --- | +| `mechanical` | `bad_fixture`, `good_fixture`, `test` (+ optional `project`) | +| `metric` | `below_fixture`, `above_fixture`, `threshold`, `test` | +| `judgment` | `eval_set`, `min_precision`, `measured_precision`, `measured_on` | + +Where possible, make the bad fixture a real past mistake from git history. +Blocking rules without proof are listed as "unproven" in `rules.md`; they don't +fail the guard yet. + +## Baselines + +- **Ruff is never ratcheted.** Adopting a ruff rule family is one standalone PR + that changes the select list and fixes every resulting violation. Don't use + `--add-noqa` or blanket per-file ignores. +- Other enforcers (mypy, Pyright, Import Linter, custom guards) may start + `ratcheted` with a baseline file of tolerated violations. Baselines may only + shrink: the guard compares counts against the base branch. +- Moving a rule from enforced/ratcheted to advisory/doc-only needs an explicit + `[rule.downgrade] reason`. + +## Hidden rules + +The guard fails when something enforces a rule that the registry doesn't +describe: + +- a `GUARDS` entry in `scripts/run_quality_guards.py`; +- a ruff format check in CI, an explicitly selected ruff code, or a ruff + per-file ignore; +- a `[tool.mypy].files` entry; +- a test matching `[discovery].guard_test_globs` or listed in + `extra_guard_tests`; +- a job in a `[discovery].workflows` file. + +Register it, or list it under `discovery.ignore_jobs` / `discovery.ignore_tests` +with a reason. + +## Keeping the tool in sync + +`scripts/rules/` is byte-identical in policyengine-api and policyengine-sim-api; +see `scripts/rules/SYNC.md`. Change it in both repositories together and bump +`TOOL_VERSION` and each registry's `tool_version`. diff --git a/docs/generated/rules.md b/docs/generated/rules.md new file mode 100644 index 000000000..17a88b144 --- /dev/null +++ b/docs/generated/rules.md @@ -0,0 +1,271 @@ +# Rules + +Generated by `python -m scripts.rules render` (scripts/rules v0.1.0, tool sha256 `f8d7b1ab1a5b`) from `rules/registry.toml`. Do not edit by hand. + +Every rule that governs `policyengine-sim-api` is listed here with how it is enforced. Blocking rules fail CI. The Gaps section lists what is written down but not yet enforced or proven. To add or change a rule, see `docs/engineering/skills/rules-registry.md`. + +## Summary + +| Tier | enforced | ratcheted | advisory | doc-only | Total | +| --- | ---: | ---: | ---: | ---: | ---: | +| mechanical | 24 | 0 | 0 | 11 | 35 | +| metric | 0 | 0 | 0 | 0 | 0 | +| judgment | 0 | 0 | 0 | 0 | 0 | + +**Blocking:** 24 of 35 rules. + +## Gaps + +### Not enforced (doc-only) + +- `sim.boundaries.gateway-executor-decoupled`: The gateway and executor never import each other; they communicate only through the models and state helpers in policyengine-simulation-contract. +- `sim.boundaries.unit-tests-need-no-services`: Unit tests never require real network credentials, Modal, Hugging Face, GCP or deployed services; those seams are mocked. +- `sim.contracts.regenerate-clients`: When public API schemas change, regenerate the clients and run the generated-client integration tests. +- `sim.deploy.image-smoke-covers-every-image-lib`: The image smoke path filter includes every library mounted into a Modal image; libs/policyengine-stage12-persistence is currently missing, so Stage 12-only changes skip the smoke. +- `sim.observability.failures-never-break-requests`: Invalid observability configuration fails at build or deploy time, and telemetry failures at runtime never fail a request or computation. +- `sim.observability.header-transport-only`: observability_id travels only in the X-PolicyEngine-Observability-Id header, never in a simulation JSON body, and polling never invents one. +- `sim.observability.id-is-diagnostic-only`: observability_id has diagnostic meaning only: never use it for idempotency, authorization, database identity, routing or business logic. +- `sim.observability.registered-stages-only`: Span names come from the stage plans in policyengine_simulation_observability.stages; runtime code never adds span-name string literals. +- `sim.process.same-repo-draft-prs`: PRs are opened as drafts from branches in PolicyEngine/policyengine-sim-api, never from forks, and start with 'Fixes #'. +- `sim.quality.every-project-tested`: Every project and library runs its tests in PR CI; libs/policyengine-fastapi is not yet in the test matrix. +- `sim.quality.ruff-format-tests`: Test code is formatted with ruff format too; CI currently checks only src directories. + +### Advisory (checked but not blocking) + +- None. + +### Unproven (no proof that the checker rejects a bad example) + +- `sim.contracts.clients-publish-deployed-commit` +- `sim.contracts.full-integration-beta-only` +- `sim.contracts.gateway-openapi-golden` +- `sim.deploy.container-context-minimal` +- `sim.deploy.gcloud-without-terraform` +- `sim.deploy.promotion-order-explicit` +- `sim.deploy.revision-tags-valid` +- `sim.deploy.shell-scripts-valid` +- `sim.deploy.smoke-tests-strict` +- `sim.deploy.traffic-changes-exact` +- `sim.deploy.workflow-shell-safety` +- `sim.process.rules-registry` +- `sim.stage12.deployment-scope` +- `sim.stage12.deployment-uses-api-schema` +- `sim.stage12.no-migration-chain` +- `sim.stage12.no-production-user-associations` +- `sim.stage12.persistence-typed-dml-only` +- `sim.stage12.production-lock-has-db-runtime` + +### Ratcheted (existing violations still tolerated) + +- None. + +### Exceptions and review dates + +- None. + +## Service and package boundaries + +| Rule | Statement | Tier | Status | Blocking | Enforced by | CI | Proof | +| --- | --- | --- | --- | --- | --- | --- | --- | +| `sim.boundaries.gateway-executor-decoupled` | The gateway and executor never import each other; they communicate only through the models and state helpers in policyengine-simulation-contract. | mechanical | doc-only | no | nothing | — | — | +| `sim.boundaries.unit-tests-need-no-services` | Unit tests never require real network credentials, Modal, Hugging Face, GCP or deployed services; those seams are mocked. | mechanical | doc-only | no | nothing | — | — | + +```mermaid +flowchart LR + r_sim_boundaries_gateway_executor_decoupled("sim.boundaries.gateway-executor-decoupled (doc-only)") + gap_unenforced>"not enforced"] + r_sim_boundaries_unit_tests_need_no_services("sim.boundaries.unit-tests-need-no-services (doc-only)") + r_sim_boundaries_gateway_executor_decoupled -.-> gap_unenforced + r_sim_boundaries_unit_tests_need_no_services -.-> gap_unenforced +``` + +## Stage 12 persistence + +| Rule | Statement | Tier | Status | Blocking | Enforced by | CI | Proof | +| --- | --- | --- | --- | --- | --- | --- | --- | +| `sim.stage12.deployment-scope` | Main-branch deploys roll out Stage 12 in both environments; a Stage 12-only deploy runs candidate tests and the requested promotion but never redeploys existing Modal resources; the Stage 12 worker can request a Google OIDC token. | mechanical | enforced | yes | pytest `projects/policyengine-simulation-entry/tests/test_deployment_assets.py` | `pr.yml:test` | **unproven** | +| `sim.stage12.deployment-uses-api-schema` | Stage 12 deployment uses the API-owned schema directly, and the Stage 12 runtime it deploys uses typed DML without raw SQL or schema changes. | mechanical | enforced | yes | pytest `projects/policyengine-simulation-entry/tests/test_deployment_assets.py` | `pr.yml:test` | **unproven** | +| `sim.stage12.no-migration-chain` | This repository has no schema migration chain (no Alembic); Stage 12 tables are migrated by policyengine-api. | mechanical | enforced | yes | pytest `projects/policyengine-simulation-entry/tests/test_stage12_schema_authority.py` | `pr.yml:test` | **unproven** | +| `sim.stage12.no-production-user-associations` | Stage 12 runtime modules never create production user associations (user_simulations or user_reports). | mechanical | enforced | yes | pytest `projects/policyengine-simulation-entry/tests/test_stage12_schema_authority.py` | `pr.yml:test` | **unproven** | +| `sim.stage12.no-schema-ddl` | Simulation runtimes and the Stage 12 persistence library contain no schema DDL (CREATE, ALTER, DROP or TRUNCATE of tables, types, schemas or databases, or create_all); policyengine-api owns the schema. | mechanical | enforced | yes | pytest `projects/policyengine-simulation-entry/tests/test_stage12_schema_authority.py` | `pr.yml:test` | yes | +| `sim.stage12.persistence-typed-dml-only` | The Stage 12 persistence library and runtime use typed SQLAlchemy Core DML only: no raw SQL text, driver-level SQL or schema operations. | mechanical | enforced | yes | pytest `libs/policyengine-stage12-persistence/tests/test_source_boundaries.py` | `pr.yml:test` | **unproven** | +| `sim.stage12.production-lock-has-db-runtime` | The production lock file includes the direct Stage 12 database runtime dependencies. | mechanical | enforced | yes | pytest `projects/policyengine-simulation-entry/tests/test_deployment_assets.py` | `pr.yml:test` | **unproven** | + +```mermaid +flowchart LR + r_sim_stage12_deployment_scope("sim.stage12.deployment-scope (enforced)") + e_pytest_projects_policyengine_simulation_entry_tests_test_deployment_assets_py["pytest projects/policyengine-simulation-entry/tests/test_deployment_assets.py"] + j_CI_pr_yml_test["CI pr.yml:test"] + r_sim_stage12_deployment_uses_api_schema("sim.stage12.deployment-uses-api-schema (enforced)") + r_sim_stage12_no_migration_chain("sim.stage12.no-migration-chain (enforced)") + e_pytest_projects_policyengine_simulation_entry_tests_test_stage12_schema_authority_py["pytest projects/policyengine-simulation-entry/tests/test_stage12_schema_authority.py"] + r_sim_stage12_no_production_user_associations("sim.stage12.no-production-user-associations (enforced)") + r_sim_stage12_no_schema_ddl("sim.stage12.no-schema-ddl (enforced)") + r_sim_stage12_persistence_typed_dml_only("sim.stage12.persistence-typed-dml-only (enforced)") + e_pytest_libs_policyengine_stage12_persistence_tests_test_source_boundaries_py["pytest libs/policyengine-stage12-persistence/tests/test_source_boundaries.py"] + r_sim_stage12_production_lock_has_db_runtime("sim.stage12.production-lock-has-db-runtime (enforced)") + r_sim_stage12_deployment_scope --> e_pytest_projects_policyengine_simulation_entry_tests_test_deployment_assets_py + e_pytest_projects_policyengine_simulation_entry_tests_test_deployment_assets_py --> j_CI_pr_yml_test + r_sim_stage12_deployment_uses_api_schema --> e_pytest_projects_policyengine_simulation_entry_tests_test_deployment_assets_py + r_sim_stage12_no_migration_chain --> e_pytest_projects_policyengine_simulation_entry_tests_test_stage12_schema_authority_py + e_pytest_projects_policyengine_simulation_entry_tests_test_stage12_schema_authority_py --> j_CI_pr_yml_test + r_sim_stage12_no_production_user_associations --> e_pytest_projects_policyengine_simulation_entry_tests_test_stage12_schema_authority_py + r_sim_stage12_no_schema_ddl --> e_pytest_projects_policyengine_simulation_entry_tests_test_stage12_schema_authority_py + r_sim_stage12_persistence_typed_dml_only --> e_pytest_libs_policyengine_stage12_persistence_tests_test_source_boundaries_py + e_pytest_libs_policyengine_stage12_persistence_tests_test_source_boundaries_py --> j_CI_pr_yml_test + r_sim_stage12_production_lock_has_db_runtime --> e_pytest_projects_policyengine_simulation_entry_tests_test_deployment_assets_py +``` + +## Deployment pipeline + +| Rule | Statement | Tier | Status | Blocking | Enforced by | CI | Proof | +| --- | --- | --- | --- | --- | --- | --- | --- | +| `sim.deploy.container-context-minimal` | Container builds exclude local environments and unrelated projects, and the entrypoint container builds for the requested target platform. | mechanical | enforced | yes | pytest `projects/policyengine-simulation-entry/tests/test_deployment_assets.py` | `pr.yml:test` | **unproven** | +| `sim.deploy.docker-images-build` | The simulation entry and executor Docker images build. | mechanical | enforced | yes | CI job `docker-build` | `pr.yml:docker-build` | — | +| `sim.deploy.gcloud-without-terraform` | Deployment uses the gcloud-based workflow, not Terraform. | mechanical | enforced | yes | pytest `projects/policyengine-simulation-entry/tests/test_deployment_assets.py` | `pr.yml:test` | **unproven** | +| `sim.deploy.image-smoke-covers-every-image-lib` | The image smoke path filter includes every library mounted into a Modal image; libs/policyengine-stage12-persistence is currently missing, so Stage 12-only changes skip the smoke. | mechanical | doc-only | no | nothing | — | — | +| `sim.deploy.modal-image-smoke` | App entrypoints import inside the real Modal images before merge, whenever a library baked into an image changes. | mechanical | enforced | yes | CI job `image-smoke` | `pr-image-smoke.yml:image-smoke` | — | +| `sim.deploy.promotion-order-explicit` | Full-stack promotion order is explicit, and the reusable deployment rejects unexpectedly skipped phases. | mechanical | enforced | yes | pytest `projects/policyengine-simulation-entry/tests/test_deployment_assets.py` | `pr.yml:test` | **unproven** | +| `sim.deploy.revision-tags-valid` | Cloud Run revision tags are valid on a workflow's first run and fit the existing service names. | mechanical | enforced | yes | pytest `projects/policyengine-simulation-entry/tests/test_deployment_assets.py` | `pr.yml:test` | **unproven** | +| `sim.deploy.shell-scripts-valid` | Deployment shell scripts have valid shell syntax. | mechanical | enforced | yes | pytest `projects/policyengine-simulation-entry/tests/test_deployment_assets.py` | `pr.yml:test` | **unproven** | +| `sim.deploy.smoke-tests-strict` | Candidate and stable smoke tests reject a revision mismatch without retrying, and fail if the previous revision does not converge. | mechanical | enforced | yes | pytest `projects/policyengine-simulation-entry/tests/test_deployment_assets.py` | `pr.yml:test` | **unproven** | +| `sim.deploy.traffic-changes-exact` | Traffic changes target an exact, validated revision, are reversible, and refuse to run over an intervening promotion. | mechanical | enforced | yes | pytest `projects/policyengine-simulation-entry/tests/test_deployment_assets.py` | `pr.yml:test` | **unproven** | +| `sim.deploy.workflow-shell-safety` | Cloud Run deployment never interpolates GitHub expressions into shell, escapes environment values, and pins secret versions. | mechanical | enforced | yes | pytest `projects/policyengine-simulation-entry/tests/test_deployment_assets.py` | `pr.yml:test` | **unproven** | + +```mermaid +flowchart LR + r_sim_deploy_container_context_minimal("sim.deploy.container-context-minimal (enforced)") + e_pytest_projects_policyengine_simulation_entry_tests_test_deployment_assets_py["pytest projects/policyengine-simulation-entry/tests/test_deployment_assets.py"] + j_CI_pr_yml_test["CI pr.yml:test"] + r_sim_deploy_docker_images_build("sim.deploy.docker-images-build (enforced)") + e_CI_job_docker_build["CI job docker-build"] + j_CI_pr_yml_docker_build["CI pr.yml:docker-build"] + r_sim_deploy_gcloud_without_terraform("sim.deploy.gcloud-without-terraform (enforced)") + r_sim_deploy_image_smoke_covers_every_image_lib("sim.deploy.image-smoke-covers-every-image-lib (doc-only)") + gap_unenforced>"not enforced"] + r_sim_deploy_modal_image_smoke("sim.deploy.modal-image-smoke (enforced)") + e_CI_job_image_smoke["CI job image-smoke"] + j_CI_pr_image_smoke_yml_image_smoke["CI pr-image-smoke.yml:image-smoke"] + r_sim_deploy_promotion_order_explicit("sim.deploy.promotion-order-explicit (enforced)") + r_sim_deploy_revision_tags_valid("sim.deploy.revision-tags-valid (enforced)") + r_sim_deploy_shell_scripts_valid("sim.deploy.shell-scripts-valid (enforced)") + r_sim_deploy_smoke_tests_strict("sim.deploy.smoke-tests-strict (enforced)") + r_sim_deploy_traffic_changes_exact("sim.deploy.traffic-changes-exact (enforced)") + r_sim_deploy_workflow_shell_safety("sim.deploy.workflow-shell-safety (enforced)") + r_sim_deploy_container_context_minimal --> e_pytest_projects_policyengine_simulation_entry_tests_test_deployment_assets_py + e_pytest_projects_policyengine_simulation_entry_tests_test_deployment_assets_py --> j_CI_pr_yml_test + r_sim_deploy_docker_images_build --> e_CI_job_docker_build + e_CI_job_docker_build --> j_CI_pr_yml_docker_build + r_sim_deploy_gcloud_without_terraform --> e_pytest_projects_policyengine_simulation_entry_tests_test_deployment_assets_py + r_sim_deploy_image_smoke_covers_every_image_lib -.-> gap_unenforced + r_sim_deploy_modal_image_smoke --> e_CI_job_image_smoke + e_CI_job_image_smoke --> j_CI_pr_image_smoke_yml_image_smoke + r_sim_deploy_promotion_order_explicit --> e_pytest_projects_policyengine_simulation_entry_tests_test_deployment_assets_py + r_sim_deploy_revision_tags_valid --> e_pytest_projects_policyengine_simulation_entry_tests_test_deployment_assets_py + r_sim_deploy_shell_scripts_valid --> e_pytest_projects_policyengine_simulation_entry_tests_test_deployment_assets_py + r_sim_deploy_smoke_tests_strict --> e_pytest_projects_policyengine_simulation_entry_tests_test_deployment_assets_py + r_sim_deploy_traffic_changes_exact --> e_pytest_projects_policyengine_simulation_entry_tests_test_deployment_assets_py + r_sim_deploy_workflow_shell_safety --> e_pytest_projects_policyengine_simulation_entry_tests_test_deployment_assets_py +``` + +## API contracts and integration + +| Rule | Statement | Tier | Status | Blocking | Enforced by | CI | Proof | +| --- | --- | --- | --- | --- | --- | --- | --- | +| `sim.contracts.clients-publish-deployed-commit` | Client publication checks out the commit that was actually deployed. | mechanical | enforced | yes | pytest `projects/policyengine-simulation-entry/tests/test_deployment_assets.py` | `pr.yml:test` | **unproven** | +| `sim.contracts.full-integration-beta-only` | The complete integration suite runs against beta deployments only, never production. | mechanical | enforced | yes | pytest `projects/policyengine-simulation-entry/tests/test_deployment_assets.py` | `pr.yml:test` | **unproven** | +| `sim.contracts.gateway-openapi-golden` | The gateway's OpenAPI spec matches the checked-in golden file; regenerate it deliberately when the API changes. | mechanical | enforced | yes | pytest `projects/policyengine-simulation-gateway/tests/test_openapi_golden.py` | `pr.yml:test` | **unproven** | +| `sim.contracts.local-integration` | Generated-client integration tests pass against the locally built services. | mechanical | enforced | yes | CI job `test-integration` | `pr.yml:test-integration` | — | +| `sim.contracts.regenerate-clients` | When public API schemas change, regenerate the clients and run the generated-client integration tests. | mechanical | doc-only | no | nothing | — | — | + +```mermaid +flowchart LR + r_sim_contracts_clients_publish_deployed_commit("sim.contracts.clients-publish-deployed-commit (enforced)") + e_pytest_projects_policyengine_simulation_entry_tests_test_deployment_assets_py["pytest projects/policyengine-simulation-entry/tests/test_deployment_assets.py"] + j_CI_pr_yml_test["CI pr.yml:test"] + r_sim_contracts_full_integration_beta_only("sim.contracts.full-integration-beta-only (enforced)") + r_sim_contracts_gateway_openapi_golden("sim.contracts.gateway-openapi-golden (enforced)") + e_pytest_projects_policyengine_simulation_gateway_tests_test_openapi_golden_py["pytest projects/policyengine-simulation-gateway/tests/test_openapi_golden.py"] + r_sim_contracts_local_integration("sim.contracts.local-integration (enforced)") + e_CI_job_test_integration["CI job test-integration"] + j_CI_pr_yml_test_integration["CI pr.yml:test-integration"] + r_sim_contracts_regenerate_clients("sim.contracts.regenerate-clients (doc-only)") + gap_unenforced>"not enforced"] + r_sim_contracts_clients_publish_deployed_commit --> e_pytest_projects_policyengine_simulation_entry_tests_test_deployment_assets_py + e_pytest_projects_policyengine_simulation_entry_tests_test_deployment_assets_py --> j_CI_pr_yml_test + r_sim_contracts_full_integration_beta_only --> e_pytest_projects_policyengine_simulation_entry_tests_test_deployment_assets_py + r_sim_contracts_gateway_openapi_golden --> e_pytest_projects_policyengine_simulation_gateway_tests_test_openapi_golden_py + e_pytest_projects_policyengine_simulation_gateway_tests_test_openapi_golden_py --> j_CI_pr_yml_test + r_sim_contracts_local_integration --> e_CI_job_test_integration + e_CI_job_test_integration --> j_CI_pr_yml_test_integration + r_sim_contracts_regenerate_clients -.-> gap_unenforced +``` + +## Observability + +| Rule | Statement | Tier | Status | Blocking | Enforced by | CI | Proof | +| --- | --- | --- | --- | --- | --- | --- | --- | +| `sim.observability.failures-never-break-requests` | Invalid observability configuration fails at build or deploy time, and telemetry failures at runtime never fail a request or computation. | mechanical | doc-only | no | nothing | — | — | +| `sim.observability.header-transport-only` | observability_id travels only in the X-PolicyEngine-Observability-Id header, never in a simulation JSON body, and polling never invents one. | mechanical | doc-only | no | nothing | — | — | +| `sim.observability.id-is-diagnostic-only` | observability_id has diagnostic meaning only: never use it for idempotency, authorization, database identity, routing or business logic. | mechanical | doc-only | no | nothing | — | — | +| `sim.observability.registered-stages-only` | Span names come from the stage plans in policyengine_simulation_observability.stages; runtime code never adds span-name string literals. | mechanical | doc-only | no | nothing | — | — | + +```mermaid +flowchart LR + r_sim_observability_failures_never_break_requests("sim.observability.failures-never-break-requests (doc-only)") + gap_unenforced>"not enforced"] + r_sim_observability_header_transport_only("sim.observability.header-transport-only (doc-only)") + r_sim_observability_id_is_diagnostic_only("sim.observability.id-is-diagnostic-only (doc-only)") + r_sim_observability_registered_stages_only("sim.observability.registered-stages-only (doc-only)") + r_sim_observability_failures_never_break_requests -.-> gap_unenforced + r_sim_observability_header_transport_only -.-> gap_unenforced + r_sim_observability_id_is_diagnostic_only -.-> gap_unenforced + r_sim_observability_registered_stages_only -.-> gap_unenforced +``` + +## Code quality and tests + +| Rule | Statement | Tier | Status | Blocking | Enforced by | CI | Proof | +| --- | --- | --- | --- | --- | --- | --- | --- | +| `sim.quality.every-project-tested` | Every project and library runs its tests in PR CI; libs/policyengine-fastapi is not yet in the test matrix. | mechanical | doc-only | no | nothing | — | — | +| `sim.quality.ruff-format-src` | Source code under projects/*/src and libs/*/src is formatted with ruff format. | mechanical | enforced | yes | ruff format | `pr.yml:lint` | — | +| `sim.quality.ruff-format-tests` | Test code is formatted with ruff format too; CI currently checks only src directories. | mechanical | doc-only | no | nothing | — | — | +| `sim.quality.tests-pass` | Each project's unit tests pass in its own locked environment. | mechanical | enforced | yes | CI job `test` | `pr.yml:test` | — | + +```mermaid +flowchart LR + r_sim_quality_every_project_tested("sim.quality.every-project-tested (doc-only)") + gap_unenforced>"not enforced"] + r_sim_quality_ruff_format_src("sim.quality.ruff-format-src (enforced)") + e_ruff_format["ruff format"] + j_CI_pr_yml_lint["CI pr.yml:lint"] + r_sim_quality_ruff_format_tests("sim.quality.ruff-format-tests (doc-only)") + r_sim_quality_tests_pass("sim.quality.tests-pass (enforced)") + e_CI_job_test["CI job test"] + j_CI_pr_yml_test["CI pr.yml:test"] + r_sim_quality_every_project_tested -.-> gap_unenforced + r_sim_quality_ruff_format_src --> e_ruff_format + e_ruff_format --> j_CI_pr_yml_lint + r_sim_quality_ruff_format_tests -.-> gap_unenforced + r_sim_quality_tests_pass --> e_CI_job_test + e_CI_job_test --> j_CI_pr_yml_test +``` + +## Process and repository hygiene + +| Rule | Statement | Tier | Status | Blocking | Enforced by | CI | Proof | +| --- | --- | --- | --- | --- | --- | --- | --- | +| `sim.process.rules-registry` | Every rule is registered in rules/registry.toml, is enforced where it claims to be, and its rendered views are current. | mechanical | enforced | yes | quality guard `rules-registry` | `pr.yml:quality-guards` | **unproven** | +| `sim.process.same-repo-draft-prs` | PRs are opened as drafts from branches in PolicyEngine/policyengine-sim-api, never from forks, and start with 'Fixes #'. | mechanical | doc-only | no | nothing | — | — | + +```mermaid +flowchart LR + r_sim_process_rules_registry("sim.process.rules-registry (enforced)") + e_quality_guard_rules_registry["quality guard rules-registry"] + j_CI_pr_yml_quality_guards["CI pr.yml:quality-guards"] + r_sim_process_same_repo_draft_prs("sim.process.same-repo-draft-prs (doc-only)") + gap_unenforced>"not enforced"] + r_sim_process_rules_registry --> e_quality_guard_rules_registry + e_quality_guard_rules_registry --> j_CI_pr_yml_quality_guards + r_sim_process_same_repo_draft_prs -.-> gap_unenforced +``` From f192413e5afc145ba2661c240baa08e160454bbc Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Mon, 12 Oct 2026 23:29:25 +0400 Subject: [PATCH 5/6] Check that proof tests run in CI Sync scripts/rules with policyengine-api: the guard now checks that each proof test is collected by a CI job, not only that it exists. A pytest-enforced rule's proof runs in the rule's own job; other rules name the job with proof.workflow, proof.job and proof.invocation. Co-Authored-By: Claude Opus 5.5 --- docs/engineering/skills/rules-registry.md | 9 ++ scripts/rules/guard.py | 8 +- scripts/rules/resolve.py | 169 +++++++++++++++------- scripts/rules/tests/test_guard.py | 95 ++++++++++++ scripts/rules/validate.py | 16 +- 5 files changed, 245 insertions(+), 52 deletions(-) diff --git a/docs/engineering/skills/rules-registry.md b/docs/engineering/skills/rules-registry.md index ea54ccc2c..f94f8cae0 100644 --- a/docs/engineering/skills/rules-registry.md +++ b/docs/engineering/skills/rules-registry.md @@ -100,6 +100,15 @@ that runs the rule's checker on them. | `metric` | `below_fixture`, `above_fixture`, `threshold`, `test` | | `judgment` | `eval_set`, `min_precision`, `measured_precision`, `measured_on` | +The proof test asserts that the bad fixture is rejected and the good one +accepted. The guard doesn't run it; it checks that the fixtures exist and that a +CI job collects the proof test: + +- For a `pytest`-enforced rule, the proof test must be collected by the rule's + own `[[rule.ci]]` job (for example, under the paths `make test` runs). +- For any other enforcer, name the job that runs the proof test with + `proof.workflow`, `proof.job` and `proof.invocation` (all three or none). + Where possible, make the bad fixture a real past mistake from git history. Blocking rules without proof are listed as "unproven" in `rules.md`; they don't fail the guard yet. diff --git a/scripts/rules/guard.py b/scripts/rules/guard.py index aee74163a..f6c703e2c 100644 --- a/scripts/rules/guard.py +++ b/scripts/rules/guard.py @@ -23,7 +23,12 @@ render_agents_block, render_rules_md, ) -from scripts.rules.resolve import clear_caches, resolve_ci, resolve_enforcer +from scripts.rules.resolve import ( + clear_caches, + resolve_ci, + resolve_enforcer, + resolve_proof_ci, +) from scripts.rules.validate import validate REPO_ROOT = Path(__file__).resolve().parents[2] @@ -80,6 +85,7 @@ def run_guard( for rule in registry.rules: result.violations += resolve_enforcer(rule, root) result.violations += resolve_ci(rule, root) + result.violations += resolve_proof_ci(rule, root) result.violations += discover_hidden(registry, root) result.violations += _stale_outputs(registry, root) diff --git a/scripts/rules/resolve.py b/scripts/rules/resolve.py index 596557e9b..1e47b09c7 100644 --- a/scripts/rules/resolve.py +++ b/scripts/rules/resolve.py @@ -15,11 +15,12 @@ from pathlib import Path from typing import Any -from scripts.rules.model import Kind, Rule, Violation +from scripts.rules.model import CiRef, Kind, Rule, Violation from scripts.rules.workflows import JobBlock, parse_jobs DEFAULT_RUNNER = "scripts/run_quality_guards.py" RUFF_DEFAULT_SELECT = ("E4", "E7", "E9", "F") +PROOF_CI_FIELDS = ("workflow", "job", "invocation") # --- shared readers ------------------------------------------------------- @@ -326,75 +327,143 @@ def resolve_enforcer(rule: Rule, root: Path) -> list[Violation]: # --- CI wiring ------------------------------------------------------------ +def _job_for( + rule_id: str, ref: CiRef, root: Path, check: str, field: str +) -> tuple[JobBlock | None, list[Violation]]: + """The job a CI reference names, if it exists and runs the invocation.""" + + def v(message: str, fix: str) -> Violation: + return Violation(rule_id, check, message, fix) + + if not (root / ref.workflow).is_file(): + return None, [ + v(f"workflow {ref.workflow} does not exist", f"fix {field}.workflow") + ] + jobs, problems = workflow_jobs(root / ref.workflow) + violations = [ + v( + f"{ref.workflow}: {problem}", + "keep workflows within the supported YAML subset", + ) + for problem in problems + ] + job = jobs.get(ref.job) + if job is None: + violations.append( + v(f"no job '{ref.job}' in {ref.workflow}", f"use one of {sorted(jobs)}") + ) + return None, violations + if not job.invokes(ref.invocation): + violations.append( + v( + f"job '{ref.job}' in {ref.workflow} does not run '{ref.invocation}'", + f"correct {field}.invocation, or make the job run it", + ) + ) + return None, violations + return job, violations + + def resolve_ci(rule: Rule, root: Path) -> list[Violation]: """Each CI reference names a real job that really runs the enforcer.""" violations = [] for ref in rule.ci: + job, problems = _job_for(rule.id, ref, root, "ci", "rule.ci") + violations += problems + if job is not None and rule.enforcer.kind is Kind.PYTEST: + violations += _test_coverage( + rule.id, + root_relative_nodes(rule), + rule.enforcer.get("project"), + ref.invocation, + job, + root, + "ci", + ) + return violations - def v(message: str, fix: str) -> Violation: - return Violation(rule.id, "ci", message, fix) - jobs, problems = workflow_jobs(root / ref.workflow) - if not (root / ref.workflow).is_file(): - violations.append( - v(f"workflow {ref.workflow} does not exist", "fix rule.ci.workflow") - ) - continue - for problem in problems: - violations.append( - v( - f"{ref.workflow}: {problem}", - "keep workflows within the supported YAML subset", - ) - ) - job = jobs.get(ref.job) - if job is None: - violations.append( - v(f"no job '{ref.job}' in {ref.workflow}", f"use one of {sorted(jobs)}") +def resolve_proof_ci(rule: Rule, root: Path) -> list[Violation]: + """A proof test must run in CI, or nothing shows the checker still works. + + The proof test itself asserts that the bad fixture is rejected and the good + one accepted; the guard checks that some CI job collects it. A proof runs + in the rule's own CI job when the rule is pytest-enforced; otherwise the + proof names its job with ``workflow``, ``job`` and ``invocation``. + """ + if rule.proof is None or not isinstance(rule.proof.fields.get("test"), str): + return [] + fields = rule.proof.fields + project = fields.get("project") if isinstance(fields.get("project"), str) else None + prefix = f"{project.rstrip('/')}/" if project else "" + node = prefix + fields["test"] + if all(isinstance(fields.get(key), str) for key in PROOF_CI_FIELDS): + refs = [CiRef(fields["workflow"], fields["job"], fields["invocation"])] + elif rule.enforcer.kind is Kind.PYTEST: + refs = list(rule.ci) + else: + refs = [] + if not refs: + return [ + Violation( + rule.id, + "proof", + "the proof test is not tied to a CI job", + "add proof.workflow, proof.job and proof.invocation naming the job that runs it", ) - continue - if not job.invokes(ref.invocation): - violations.append( - v( - f"job '{ref.job}' in {ref.workflow} does not run '{ref.invocation}'", - "correct rule.ci.invocation, or make the job run the enforcer", - ) + ] + violations = [] + for ref in refs: + job, problems = _job_for(rule.id, ref, root, "proof", "proof") + violations += problems + if job is not None: + violations += _test_coverage( + rule.id, [node], project, ref.invocation, job, root, "proof" ) - continue - if rule.enforcer.kind is Kind.PYTEST: - violations += _pytest_coverage(rule, root, ref.invocation, job) return violations -def _pytest_coverage( - rule: Rule, root: Path, invocation: str, job: JobBlock +def _test_coverage( + rule_id: str, + nodes: list[str], + project: str | None, + invocation: str, + job: JobBlock, + root: Path, + check: str, ) -> list[Violation]: - """The job's test command actually collects the rule's tests.""" + """The job's test command actually collects these root-relative test nodes.""" violations = [] + collected: list[str] | None = None + described = "" if invocation.startswith("make "): target = invocation.split()[1] - paths = make_target_paths(root, target) - if paths: - for node in root_relative_nodes(rule): - path = split_node(node)[0] - if not any( - path == p or path.startswith(p.rstrip("/") + "/") for p in paths - ): - violations.append( - Violation( - rule.id, - "ci", - f"{path} is not collected by 'make {target}' (collects {paths})", - f"add it to the Makefile '{target}' target or move the test", - ) + collected = make_target_paths(root, target) + described = f"'make {target}'" + elif "unittest discover" in invocation: + match = re.search(r"(?:-s|--start-directory)\s+(\S+)", invocation) + collected = [match.group(1)] if match else None + described = f"'{invocation}'" + if collected: + for node in nodes: + path = split_node(node)[0] + if not any( + path == p or path.startswith(p.rstrip("/") + "/") for p in collected + ): + violations.append( + Violation( + rule_id, + check, + f"{path} is not collected by {described} (collects {collected})", + "add it to what the job runs, or move the test", ) - project = rule.enforcer.get("project") + ) if project and job.matrix: if not any(project in values for values in job.matrix.values()): violations.append( Violation( - rule.id, - "ci", + rule_id, + check, f"project {project} is not in the matrix of job '{job.id}'", "add the project to the job matrix", ) diff --git a/scripts/rules/tests/test_guard.py b/scripts/rules/tests/test_guard.py index 7d14cc540..4b98fa303 100644 --- a/scripts/rules/tests/test_guard.py +++ b/scripts/rules/tests/test_guard.py @@ -319,6 +319,101 @@ def test_proof_files_must_exist(self) -> None: ) +class ProofCiTest(unittest.TestCase): + PROOF_FILES = { + "rules/fixtures/bad.py": "import sqlalchemy\n", + "rules/fixtures/good.py": "x = 1\n", + } + + def with_proof(self, repo: Repo, rule_marker: str, proof: str) -> None: + repo.edit( + "rules/registry.toml", rule_marker, rule_marker + "\n[rule.proof]\n" + proof + ) + repo.render() + + def test_pytest_rule_proof_runs_in_its_ci_job(self) -> None: + files = { + **self.PROOF_FILES, + "tests/unit/test_proof.py": "def test_rule_proof_routes():\n pass\n", + } + with Repo(files) as repo: + self.with_proof( + repo, + 'nodes = ["tests/unit/test_boundaries.py::test_routes_have_no_sql"]', + 'bad_fixture = "rules/fixtures/bad.py"\ngood_fixture = "rules/fixtures/good.py"\n' + 'test = "tests/unit/test_proof.py::test_rule_proof_routes"', + ) + self.assertEqual(repo.messages(), []) + + def test_proof_outside_the_ci_command_is_rejected(self) -> None: + files = { + **self.PROOF_FILES, + "tests/contract/test_proof.py": "def test_rule_proof_routes():\n pass\n", + } + with Repo(files) as repo: + self.with_proof( + repo, + 'nodes = ["tests/unit/test_boundaries.py::test_routes_have_no_sql"]', + 'bad_fixture = "rules/fixtures/bad.py"\ngood_fixture = "rules/fixtures/good.py"\n' + 'test = "tests/contract/test_proof.py::test_rule_proof_routes"', + ) + messages = "\n".join(repo.messages()) + self.assertIn( + "proof: tests/contract/test_proof.py is not collected by 'make test'", + messages, + ) + + def test_non_pytest_rule_proof_must_name_its_job(self) -> None: + files = { + **self.PROOF_FILES, + "tests/unit/test_proof.py": "def test_rule_proof_types():\n pass\n", + } + with Repo(files) as repo: + self.with_proof( + repo, + 'paths = ["pkg"]', + 'bad_fixture = "rules/fixtures/bad.py"\ngood_fixture = "rules/fixtures/good.py"\n' + 'test = "tests/unit/test_proof.py::test_rule_proof_types"', + ) + self.assertIn( + "the proof test is not tied to a CI job", "\n".join(repo.messages()) + ) + + def test_proof_with_named_unittest_job(self) -> None: + workflow_job = ( + " tool-tests:\n steps:\n" + " - run: python -m unittest discover -s tools/tests -t .\n" + ) + files = { + **self.PROOF_FILES, + "tools/tests/test_proof.py": "class TestProof:\n def test_rule_proof_types(self):\n pass\n", + } + with Repo(files, render=False) as repo: + repo.write( + ".github/workflows/pr.yml", + repo.read(".github/workflows/pr.yml") + workflow_job, + ) + repo.edit( + "rules/registry.toml", + "[discovery]\n", + '[discovery]\nignore_jobs = [{ job = "tool-tests", reason = "runs tool tests" }]\n', + ) + proof = ( + 'bad_fixture = "rules/fixtures/bad.py"\ngood_fixture = "rules/fixtures/good.py"\n' + 'test = "tools/tests/test_proof.py::TestProof::test_rule_proof_types"\n' + 'workflow = ".github/workflows/pr.yml"\njob = "tool-tests"\n' + 'invocation = "unittest discover -s tools/tests"' + ) + self.with_proof(repo, 'paths = ["pkg"]', proof) + self.assertEqual(repo.messages(), []) + + repo.edit("rules/registry.toml", 'job = "tool-tests"\n', "") + repo.render() + self.assertIn( + "proof names its CI job only partly", "\n".join(repo.messages()) + ) + + class HiddenRuleTest(unittest.TestCase): def assertViolation(self, repo: Repo, fragment: str) -> None: messages = repo.messages() diff --git a/scripts/rules/validate.py b/scripts/rules/validate.py index 81aaf479f..011a33c66 100644 --- a/scripts/rules/validate.py +++ b/scripts/rules/validate.py @@ -192,10 +192,24 @@ def _validate_proof(rule: Rule, root: Path) -> list[Violation]: if rule.proof is None: return [] fields = dict(rule.proof.fields) - optional = {"project": str} if "test" in PROOF_FIELDS[rule.tier] else {} + optional = ( + {"project": str, "workflow": str, "job": str, "invocation": str} + if "test" in PROOF_FIELDS[rule.tier] + else {} + ) violations = _check_fields( rule, fields, PROOF_FIELDS[rule.tier], optional, f"proof ({rule.tier})" ) + named = [key for key in ("workflow", "job", "invocation") if key in fields] + if named and len(named) != 3: + violations.append( + Violation( + rule.id, + "schema", + "proof names its CI job only partly", + "set all of proof.workflow, proof.job and proof.invocation, or none", + ) + ) for key in PROOF_FILE_FIELDS: if isinstance(fields.get(key), str) and not (root / fields[key]).exists(): violations.append( From 665c34f15bf31a5bfc8f79d1458972492384cd62 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Mon, 12 Oct 2026 23:29:25 +0400 Subject: [PATCH 6/6] Update the simulation rules registry for Phase 0 Register the explicit ruff rule selection in ruff.toml as an advisory lint rule, update the image-smoke and test-matrix rules now that Phase 0 fixed both gaps, and register the code-owned rule machinery and the agent no-bypass rule as doc-only until branch rules enforce them. Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 7 +++++-- docs/generated/rules.md | 32 +++++++++++++++++++++--------- rules/registry.toml | 44 ++++++++++++++++++++++++++++++++++++++--- 3 files changed, 69 insertions(+), 14 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 3b1df1ae8..8e20f7476 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -96,7 +96,7 @@ To add or change a rule, read `docs/engineering/skills/rules-registry.md`. - `sim.deploy.container-context-minimal`: Container builds exclude local environments and unrelated projects, and the entrypoint container builds for the requested target platform. (blocking: pytest `projects/policyengine-simulation-entry/tests/test_deployment_assets.py`) - `sim.deploy.docker-images-build`: The simulation entry and executor Docker images build. (blocking: CI job `docker-build`) - `sim.deploy.gcloud-without-terraform`: Deployment uses the gcloud-based workflow, not Terraform. (blocking: pytest `projects/policyengine-simulation-entry/tests/test_deployment_assets.py`) -- `sim.deploy.image-smoke-covers-every-image-lib`: The image smoke path filter includes every library mounted into a Modal image; libs/policyengine-stage12-persistence is currently missing, so Stage 12-only changes skip the smoke. (doc-only, not checked: follow it yourself) +- `sim.deploy.image-smoke-covers-every-image-lib`: The image smoke path filter includes every library mounted into a Modal image, so a change to any of them runs the smoke. (doc-only, not checked: follow it yourself) - `sim.deploy.modal-image-smoke`: App entrypoints import inside the real Modal images before merge, whenever a library baked into an image changes. (blocking: CI job `image-smoke`) - `sim.deploy.promotion-order-explicit`: Full-stack promotion order is explicit, and the reusable deployment rejects unexpectedly skipped phases. (blocking: pytest `projects/policyengine-simulation-entry/tests/test_deployment_assets.py`) - `sim.deploy.revision-tags-valid`: Cloud Run revision tags are valid on a workflow's first run and fit the existing service names. (blocking: pytest `projects/policyengine-simulation-entry/tests/test_deployment_assets.py`) @@ -122,13 +122,16 @@ To add or change a rule, read `docs/engineering/skills/rules-registry.md`. ### Code quality and tests -- `sim.quality.every-project-tested`: Every project and library runs its tests in PR CI; libs/policyengine-fastapi is not yet in the test matrix. (doc-only, not checked: follow it yourself) +- `sim.quality.every-project-tested`: Every project and library with unit tests runs them in the PR test matrix. (doc-only, not checked: follow it yourself) - `sim.quality.ruff-format-src`: Source code under projects/*/src and libs/*/src is formatted with ruff format. (blocking: ruff format) - `sim.quality.ruff-format-tests`: Test code is formatted with ruff format too; CI currently checks only src directories. (doc-only, not checked: follow it yourself) +- `sim.quality.ruff-lint`: Python code passes ruff check with the rules selected in ruff.toml (ruff's defaults, made explicit); CI does not run ruff check yet. (advisory: ruff check (E4, E7, E9, F), not blocking) - `sim.quality.tests-pass`: Each project's unit tests pass in its own locked environment. (blocking: CI job `test`) ### Process and repository hygiene +- `sim.process.agents-never-bypass`: Agents never bypass branch rules, even with admin credentials: no gh pr merge --admin and no bypass option; a blocked merge is reported to a human. (doc-only, not checked: follow it yourself) +- `sim.process.rule-machinery-code-owned`: Changes to the rule machinery (rules/, scripts/rules/, scripts/run_quality_guards.py and CODEOWNERS) need approval from a core developer other than the author. (doc-only, not checked: follow it yourself) - `sim.process.rules-registry`: Every rule is registered in rules/registry.toml, is enforced where it claims to be, and its rendered views are current. (blocking: quality guard `rules-registry`) - `sim.process.same-repo-draft-prs`: PRs are opened as drafts from branches in PolicyEngine/policyengine-sim-api, never from forks, and start with 'Fixes #'. (doc-only, not checked: follow it yourself) diff --git a/docs/generated/rules.md b/docs/generated/rules.md index 17a88b144..7ab474e57 100644 --- a/docs/generated/rules.md +++ b/docs/generated/rules.md @@ -1,6 +1,6 @@ # Rules -Generated by `python -m scripts.rules render` (scripts/rules v0.1.0, tool sha256 `f8d7b1ab1a5b`) from `rules/registry.toml`. Do not edit by hand. +Generated by `python -m scripts.rules render` (scripts/rules v0.1.0, tool sha256 `803fdc2cb511`) from `rules/registry.toml`. Do not edit by hand. Every rule that governs `policyengine-sim-api` is listed here with how it is enforced. Blocking rules fail CI. The Gaps section lists what is written down but not yet enforced or proven. To add or change a rule, see `docs/engineering/skills/rules-registry.md`. @@ -8,11 +8,11 @@ Every rule that governs `policyengine-sim-api` is listed here with how it is enf | Tier | enforced | ratcheted | advisory | doc-only | Total | | --- | ---: | ---: | ---: | ---: | ---: | -| mechanical | 24 | 0 | 0 | 11 | 35 | +| mechanical | 24 | 0 | 1 | 13 | 38 | | metric | 0 | 0 | 0 | 0 | 0 | | judgment | 0 | 0 | 0 | 0 | 0 | -**Blocking:** 24 of 35 rules. +**Blocking:** 24 of 38 rules. ## Gaps @@ -21,18 +21,20 @@ Every rule that governs `policyengine-sim-api` is listed here with how it is enf - `sim.boundaries.gateway-executor-decoupled`: The gateway and executor never import each other; they communicate only through the models and state helpers in policyengine-simulation-contract. - `sim.boundaries.unit-tests-need-no-services`: Unit tests never require real network credentials, Modal, Hugging Face, GCP or deployed services; those seams are mocked. - `sim.contracts.regenerate-clients`: When public API schemas change, regenerate the clients and run the generated-client integration tests. -- `sim.deploy.image-smoke-covers-every-image-lib`: The image smoke path filter includes every library mounted into a Modal image; libs/policyengine-stage12-persistence is currently missing, so Stage 12-only changes skip the smoke. +- `sim.deploy.image-smoke-covers-every-image-lib`: The image smoke path filter includes every library mounted into a Modal image, so a change to any of them runs the smoke. - `sim.observability.failures-never-break-requests`: Invalid observability configuration fails at build or deploy time, and telemetry failures at runtime never fail a request or computation. - `sim.observability.header-transport-only`: observability_id travels only in the X-PolicyEngine-Observability-Id header, never in a simulation JSON body, and polling never invents one. - `sim.observability.id-is-diagnostic-only`: observability_id has diagnostic meaning only: never use it for idempotency, authorization, database identity, routing or business logic. - `sim.observability.registered-stages-only`: Span names come from the stage plans in policyengine_simulation_observability.stages; runtime code never adds span-name string literals. +- `sim.process.agents-never-bypass`: Agents never bypass branch rules, even with admin credentials: no gh pr merge --admin and no bypass option; a blocked merge is reported to a human. +- `sim.process.rule-machinery-code-owned`: Changes to the rule machinery (rules/, scripts/rules/, scripts/run_quality_guards.py and CODEOWNERS) need approval from a core developer other than the author. - `sim.process.same-repo-draft-prs`: PRs are opened as drafts from branches in PolicyEngine/policyengine-sim-api, never from forks, and start with 'Fixes #'. -- `sim.quality.every-project-tested`: Every project and library runs its tests in PR CI; libs/policyengine-fastapi is not yet in the test matrix. +- `sim.quality.every-project-tested`: Every project and library with unit tests runs them in the PR test matrix. - `sim.quality.ruff-format-tests`: Test code is formatted with ruff format too; CI currently checks only src directories. ### Advisory (checked but not blocking) -- None. +- `sim.quality.ruff-lint`: Python code passes ruff check with the rules selected in ruff.toml (ruff's defaults, made explicit); CI does not run ruff check yet. ### Unproven (no proof that the checker rejects a bad example) @@ -123,7 +125,7 @@ flowchart LR | `sim.deploy.container-context-minimal` | Container builds exclude local environments and unrelated projects, and the entrypoint container builds for the requested target platform. | mechanical | enforced | yes | pytest `projects/policyengine-simulation-entry/tests/test_deployment_assets.py` | `pr.yml:test` | **unproven** | | `sim.deploy.docker-images-build` | The simulation entry and executor Docker images build. | mechanical | enforced | yes | CI job `docker-build` | `pr.yml:docker-build` | — | | `sim.deploy.gcloud-without-terraform` | Deployment uses the gcloud-based workflow, not Terraform. | mechanical | enforced | yes | pytest `projects/policyengine-simulation-entry/tests/test_deployment_assets.py` | `pr.yml:test` | **unproven** | -| `sim.deploy.image-smoke-covers-every-image-lib` | The image smoke path filter includes every library mounted into a Modal image; libs/policyengine-stage12-persistence is currently missing, so Stage 12-only changes skip the smoke. | mechanical | doc-only | no | nothing | — | — | +| `sim.deploy.image-smoke-covers-every-image-lib` | The image smoke path filter includes every library mounted into a Modal image, so a change to any of them runs the smoke. | mechanical | doc-only | no | nothing | — | — | | `sim.deploy.modal-image-smoke` | App entrypoints import inside the real Modal images before merge, whenever a library baked into an image changes. | mechanical | enforced | yes | CI job `image-smoke` | `pr-image-smoke.yml:image-smoke` | — | | `sim.deploy.promotion-order-explicit` | Full-stack promotion order is explicit, and the reusable deployment rejects unexpectedly skipped phases. | mechanical | enforced | yes | pytest `projects/policyengine-simulation-entry/tests/test_deployment_assets.py` | `pr.yml:test` | **unproven** | | `sim.deploy.revision-tags-valid` | Cloud Run revision tags are valid on a workflow's first run and fit the existing service names. | mechanical | enforced | yes | pytest `projects/policyengine-simulation-entry/tests/test_deployment_assets.py` | `pr.yml:test` | **unproven** | @@ -227,9 +229,10 @@ flowchart LR | Rule | Statement | Tier | Status | Blocking | Enforced by | CI | Proof | | --- | --- | --- | --- | --- | --- | --- | --- | -| `sim.quality.every-project-tested` | Every project and library runs its tests in PR CI; libs/policyengine-fastapi is not yet in the test matrix. | mechanical | doc-only | no | nothing | — | — | +| `sim.quality.every-project-tested` | Every project and library with unit tests runs them in the PR test matrix. | mechanical | doc-only | no | nothing | — | — | | `sim.quality.ruff-format-src` | Source code under projects/*/src and libs/*/src is formatted with ruff format. | mechanical | enforced | yes | ruff format | `pr.yml:lint` | — | | `sim.quality.ruff-format-tests` | Test code is formatted with ruff format too; CI currently checks only src directories. | mechanical | doc-only | no | nothing | — | — | +| `sim.quality.ruff-lint` | Python code passes ruff check with the rules selected in ruff.toml (ruff's defaults, made explicit); CI does not run ruff check yet. | mechanical | advisory | no | ruff check (E4, E7, E9, F) | — | — | | `sim.quality.tests-pass` | Each project's unit tests pass in its own locked environment. | mechanical | enforced | yes | CI job `test` | `pr.yml:test` | — | ```mermaid @@ -240,6 +243,9 @@ flowchart LR e_ruff_format["ruff format"] j_CI_pr_yml_lint["CI pr.yml:lint"] r_sim_quality_ruff_format_tests("sim.quality.ruff-format-tests (doc-only)") + r_sim_quality_ruff_lint("sim.quality.ruff-lint (advisory)") + e_ruff_check__E4__E7__E9__F_["ruff check (E4, E7, E9, F)"] + local_only>"not run in CI"] r_sim_quality_tests_pass("sim.quality.tests-pass (enforced)") e_CI_job_test["CI job test"] j_CI_pr_yml_test["CI pr.yml:test"] @@ -247,6 +253,8 @@ flowchart LR r_sim_quality_ruff_format_src --> e_ruff_format e_ruff_format --> j_CI_pr_yml_lint r_sim_quality_ruff_format_tests -.-> gap_unenforced + r_sim_quality_ruff_lint --> e_ruff_check__E4__E7__E9__F_ + e_ruff_check__E4__E7__E9__F_ -.-> local_only r_sim_quality_tests_pass --> e_CI_job_test e_CI_job_test --> j_CI_pr_yml_test ``` @@ -255,16 +263,22 @@ flowchart LR | Rule | Statement | Tier | Status | Blocking | Enforced by | CI | Proof | | --- | --- | --- | --- | --- | --- | --- | --- | +| `sim.process.agents-never-bypass` | Agents never bypass branch rules, even with admin credentials: no gh pr merge --admin and no bypass option; a blocked merge is reported to a human. | mechanical | doc-only | no | nothing | — | — | +| `sim.process.rule-machinery-code-owned` | Changes to the rule machinery (rules/, scripts/rules/, scripts/run_quality_guards.py and CODEOWNERS) need approval from a core developer other than the author. | mechanical | doc-only | no | nothing | — | — | | `sim.process.rules-registry` | Every rule is registered in rules/registry.toml, is enforced where it claims to be, and its rendered views are current. | mechanical | enforced | yes | quality guard `rules-registry` | `pr.yml:quality-guards` | **unproven** | | `sim.process.same-repo-draft-prs` | PRs are opened as drafts from branches in PolicyEngine/policyengine-sim-api, never from forks, and start with 'Fixes #'. | mechanical | doc-only | no | nothing | — | — | ```mermaid flowchart LR + r_sim_process_agents_never_bypass("sim.process.agents-never-bypass (doc-only)") + gap_unenforced>"not enforced"] + r_sim_process_rule_machinery_code_owned("sim.process.rule-machinery-code-owned (doc-only)") r_sim_process_rules_registry("sim.process.rules-registry (enforced)") e_quality_guard_rules_registry["quality guard rules-registry"] j_CI_pr_yml_quality_guards["CI pr.yml:quality-guards"] r_sim_process_same_repo_draft_prs("sim.process.same-repo-draft-prs (doc-only)") - gap_unenforced>"not enforced"] + r_sim_process_agents_never_bypass -.-> gap_unenforced + r_sim_process_rule_machinery_code_owned -.-> gap_unenforced r_sim_process_rules_registry --> e_quality_guard_rules_registry e_quality_guard_rules_registry --> j_CI_pr_yml_quality_guards r_sim_process_same_repo_draft_prs -.-> gap_unenforced diff --git a/rules/registry.toml b/rules/registry.toml index eeea41cee..e15c1c8b6 100644 --- a/rules/registry.toml +++ b/rules/registry.toml @@ -418,7 +418,7 @@ invocation = "modal-image-smoke.sh" [[rule]] id = "sim.deploy.image-smoke-covers-every-image-lib" category = "deployment" -statement = "The image smoke path filter includes every library mounted into a Modal image; libs/policyengine-stage12-persistence is currently missing, so Stage 12-only changes skip the smoke." +statement = "The image smoke path filter includes every library mounted into a Modal image, so a change to any of them runs the smoke." rationale = ".github/workflows/pr-image-smoke.yml" scope = [".github/workflows/pr-image-smoke.yml"] tier = "mechanical" @@ -592,6 +592,20 @@ autofix = true [rule.enforcer] kind = "none" +[[rule]] +id = "sim.quality.ruff-lint" +category = "quality" +statement = "Python code passes ruff check with the rules selected in ruff.toml (ruff's defaults, made explicit); CI does not run ruff check yet." +rationale = "docs/engineering/skills/testing.md#common-commands" +scope = ["projects/*/src/**/*.py", "libs/*/src/**/*.py"] +tier = "mechanical" +status = "advisory" +autofix = true +[rule.enforcer] +kind = "ruff" +mode = "lint" +codes = ["E4", "E7", "E9", "F"] + [[rule]] id = "sim.quality.tests-pass" category = "quality" @@ -611,9 +625,9 @@ invocation = "uv run pytest tests/" [[rule]] id = "sim.quality.every-project-tested" category = "quality" -statement = "Every project and library runs its tests in PR CI; libs/policyengine-fastapi is not yet in the test matrix." +statement = "Every project and library with unit tests runs them in the PR test matrix." rationale = "docs/engineering/skills/testing.md#canonical-layout" -scope = ["libs/policyengine-fastapi/**/*.py"] +scope = ["projects/*/tests/**/*.py", "libs/*/tests/**/*.py"] tier = "mechanical" status = "doc-only" autofix = false @@ -639,6 +653,30 @@ workflow = ".github/workflows/pr.yml" job = "quality-guards" invocation = "run_quality_guards.py" +[[rule]] +id = "sim.process.rule-machinery-code-owned" +category = "process" +statement = "Changes to the rule machinery (rules/, scripts/rules/, scripts/run_quality_guards.py and CODEOWNERS) need approval from a core developer other than the author." +rationale = ".github/CODEOWNERS" +scope = [".github/CODEOWNERS"] +tier = "mechanical" +status = "doc-only" +autofix = false +[rule.enforcer] +kind = "none" + +[[rule]] +id = "sim.process.agents-never-bypass" +category = "process" +statement = "Agents never bypass branch rules, even with admin credentials: no gh pr merge --admin and no bypass option; a blocked merge is reported to a human." +rationale = "docs/engineering/skills/github-prs.md#branch-rules" +scope = ["AGENTS.md"] +tier = "mechanical" +status = "doc-only" +autofix = false +[rule.enforcer] +kind = "none" + [[rule]] id = "sim.process.same-repo-draft-prs" category = "process"