From 472d9067879ff91bac38ecc34811a95d7be75aaa Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Mon, 12 Oct 2026 23:37:57 +0400 Subject: [PATCH 1/5] Check agent adapters and CI scripts in the rules tool Sync scripts/rules with policyengine-api. The rules-registry guard now verifies that agent adapters ([agents]) only point to AGENTS.md, and a new ci-scripts quality guard ([ci_scripts]) fails on run: blocks longer than one logical line, workflow scripts that no test runs, and shellcheck findings, each ratcheted against a baseline that may only shrink. Co-Authored-By: Claude Opus 5.5 --- scripts/rules/adapters.py | 60 +++++ scripts/rules/ci_scripts.py | 215 ++++++++++++++++++ scripts/rules/cli.py | 23 +- scripts/rules/guard.py | 3 + scripts/rules/load.py | 60 ++++- scripts/rules/model.py | 23 ++ .../tests/fixtures/adapters/bad_CLAUDE.md | 5 + .../tests/fixtures/adapters/good_CLAUDE.md | 3 + .../multiline_bad/.github/workflows/ci.yml | 10 + .../multiline_good/.github/workflows/ci.yml | 12 + .../ci_scripts/shellcheck/bad_script.sh | 7 + .../ci_scripts/shellcheck/good_script.sh | 7 + .../untested_bad/.github/scripts/deploy.sh | 5 + .../untested_bad/.github/workflows/ci.yml | 8 + .../untested_good/.github/scripts/deploy.sh | 5 + .../untested_good/.github/workflows/ci.yml | 8 + .../untested_good/tests/test_deploy.py | 3 + scripts/rules/tests/test_adapters.py | 58 +++++ scripts/rules/tests/test_ci_scripts.py | 178 +++++++++++++++ scripts/rules/validate.py | 28 +++ scripts/rules/workflows.py | 76 ++++++- scripts/run_quality_guards.py | 6 +- 22 files changed, 798 insertions(+), 5 deletions(-) create mode 100644 scripts/rules/adapters.py create mode 100644 scripts/rules/ci_scripts.py create mode 100644 scripts/rules/tests/fixtures/adapters/bad_CLAUDE.md create mode 100644 scripts/rules/tests/fixtures/adapters/good_CLAUDE.md create mode 100644 scripts/rules/tests/fixtures/ci_scripts/multiline_bad/.github/workflows/ci.yml create mode 100644 scripts/rules/tests/fixtures/ci_scripts/multiline_good/.github/workflows/ci.yml create mode 100644 scripts/rules/tests/fixtures/ci_scripts/shellcheck/bad_script.sh create mode 100644 scripts/rules/tests/fixtures/ci_scripts/shellcheck/good_script.sh create mode 100644 scripts/rules/tests/fixtures/ci_scripts/untested_bad/.github/scripts/deploy.sh create mode 100644 scripts/rules/tests/fixtures/ci_scripts/untested_bad/.github/workflows/ci.yml create mode 100644 scripts/rules/tests/fixtures/ci_scripts/untested_good/.github/scripts/deploy.sh create mode 100644 scripts/rules/tests/fixtures/ci_scripts/untested_good/.github/workflows/ci.yml create mode 100644 scripts/rules/tests/fixtures/ci_scripts/untested_good/tests/test_deploy.py create mode 100644 scripts/rules/tests/test_adapters.py create mode 100644 scripts/rules/tests/test_ci_scripts.py diff --git a/scripts/rules/adapters.py b/scripts/rules/adapters.py new file mode 100644 index 000000000..b22bd112a --- /dev/null +++ b/scripts/rules/adapters.py @@ -0,0 +1,60 @@ +"""Check that per-tool agent instruction files stay thin pointers. + +``AGENTS.md`` is the one source of agent instructions: it lists the skills and +carries the generated Rules block. Tool adapters such as ``CLAUDE.md`` and +``.github/copilot-instructions.md`` only point to it. When an adapter starts +listing skills or rules itself, the copies drift apart, so this rejects it. +""" + +from __future__ import annotations + +import re +from pathlib import Path + +from scripts.rules.model import Agents, Violation +from scripts.rules.render import BEGIN + +_DOC_REFERENCE = re.compile(r"[\w./-]+\.md\b") + + +def check_adapters(agents: Agents, root: Path) -> list[Violation]: + violations = [] + instructions = Path(agents.instructions).name + for adapter in agents.adapters: + path = root / adapter + if not path.is_file(): + continue + text = path.read_text() + + def v(message: str, fix: str) -> None: + violations.append(Violation(None, "adapters", f"{adapter} {message}", fix)) + + if instructions not in text: + v( + f"does not point to {instructions}", + f"say that the guidance lives in {instructions}", + ) + others = sorted( + { + ref + for ref in _DOC_REFERENCE.findall(text) + if Path(ref).name != instructions + } + ) + if others: + v( + f"references other docs ({', '.join(others)})", + f"list skills and rules in {instructions} only; keep {adapter} a pointer", + ) + if BEGIN in text: + v( + "contains the generated rules block", + f"the block belongs in {instructions} only", + ) + lines = sum(1 for line in text.splitlines() if line.strip()) + if lines > agents.max_adapter_lines: + v( + f"has {lines} non-empty lines (limit {agents.max_adapter_lines})", + f"move the guidance into {instructions} or a skill", + ) + return violations diff --git a/scripts/rules/ci_scripts.py b/scripts/rules/ci_scripts.py new file mode 100644 index 000000000..e21391ac6 --- /dev/null +++ b/scripts/rules/ci_scripts.py @@ -0,0 +1,215 @@ +"""The ci-scripts quality guard: workflow logic lives in tested shell scripts. + +Three checks, each ratcheted against a baseline file listed in ``[ci_scripts]``: + +1. A workflow ``run:`` block holds at most one logical line of code. +2. Every shell script a workflow calls is referenced by at least one test. +3. Every shell script in the scripts directory passes ``shellcheck``. + +Anything found that isn't in the baseline fails, and so does a baseline entry +that no longer applies, so baselines only shrink. +""" + +from __future__ import annotations + +import os +import re +import shutil +import subprocess +from dataclasses import dataclass, field +from pathlib import Path + +from scripts.rules.load import load_registry +from scripts.rules.model import CiScripts, Registry, Violation +from scripts.rules.resolve import clear_caches, workflow_jobs + +REPO_ROOT = Path(__file__).resolve().parents[2] +SEPARATOR = " :: " + + +@dataclass +class Findings: + multiline: set[str] = field(default_factory=set) + untested: set[str] = field(default_factory=set) + shellcheck: set[str] | None = None # None when shellcheck could not run + notes: list[str] = field(default_factory=list) + + +def multiline_runs(config: CiScripts, root: Path) -> set[str]: + """``workflow :: job :: step`` for every run: block over one logical line.""" + found = set() + for workflow in sorted(root.glob(config.workflows)): + relative = workflow.relative_to(root).as_posix() + for job in workflow_jobs(workflow)[0].values(): + for block in job.run_blocks: + if block.code_lines > 1: + found.add(SEPARATOR.join((relative, job.id, block.step))) + return found + + +def called_scripts(config: CiScripts, root: Path) -> set[str]: + """Shell scripts in the scripts directory that a workflow calls.""" + pattern = re.compile(re.escape(config.scripts_dir.rstrip("/")) + r"/[\w.-]+\.sh") + called = set() + for workflow in root.glob(config.workflows): + for job in workflow_jobs(workflow)[0].values(): + called.update(pattern.findall(job.run_text)) + return {path for path in called if (root / path).is_file()} + + +def untested_scripts(config: CiScripts, root: Path) -> set[str]: + """Called scripts whose file name appears in no test file.""" + tests = [ + path.read_text(errors="replace") + for pattern in config.test_globs + for path in root.glob(pattern) + if path.is_file() and ".venv" not in path.parts + ] + return { + script + for script in called_scripts(config, root) + if not any(Path(script).name in text for text in tests) + } + + +def shellcheck_failures(config: CiScripts, root: Path) -> set[str] | None: + """Scripts with any shellcheck finding, or None if shellcheck isn't installed.""" + shellcheck = shutil.which("shellcheck") + if shellcheck is None: + return None + scripts = sorted( + path.relative_to(root).as_posix() + for path in (root / config.scripts_dir).glob("*.sh") + ) + if not scripts: + return set() + result = subprocess.run( + [shellcheck, "--format=gcc", *scripts], + cwd=root, + capture_output=True, + text=True, + ) + return {line.split(":", 1)[0] for line in result.stdout.splitlines() if ":" in line} + + +def find(config: CiScripts, root: Path) -> Findings: + clear_caches() + findings = Findings( + multiline=multiline_runs(config, root), + untested=untested_scripts(config, root), + shellcheck=shellcheck_failures(config, root), + ) + if findings.shellcheck is None: + findings.notes.append( + "shellcheck is not installed; the shellcheck check was skipped" + ) + return findings + + +def _baseline(root: Path, path: str) -> set[str]: + file = root / path + if not file.is_file(): + return set() + return { + line.strip() + for line in file.read_text().splitlines() + if line.strip() and not line.lstrip().startswith("#") + } + + +def _rule_for(registry: Registry, baseline: str) -> str | None: + for rule in registry.rules: + if rule.baseline is not None and rule.baseline.file == baseline: + return rule.id + return None + + +def compare(registry: Registry, findings: Findings, root: Path) -> list[Violation]: + config = registry.ci_scripts + assert config is not None + checks = ( + ( + "multi-line run: block", + findings.multiline, + config.multiline_baseline, + "move the commands into a tested script under " + f"{config.scripts_dir} and call it from a single-line run:", + ), + ( + "untested workflow script", + findings.untested, + config.untested_baseline, + "add a test that runs the script (see docs/engineering/skills/ci-workflows.md)", + ), + ( + "shellcheck finding in", + findings.shellcheck, + config.shellcheck_baseline, + "fix what `shellcheck