diff --git a/README.md b/README.md index 927c34b..753476f 100644 --- a/README.md +++ b/README.md @@ -222,6 +222,12 @@ Every run reports what it resolved, so the answer never has to be inferred from with an error annotation saying which, and in review mode the failure comment says it too. No partial analysis is published. +When CodeBoarding's free hosted credits run out, the failure comment and job summary link to +[buy a CodeBoarding license](https://codeboarding.org/pricing) and explain how to configure +`llm: license` and `license_key`. The free weekly allowance is shared across repositories +under the same GitHub owner. A supplied license that did not grant paid access gets a +license-check hint; provider billing failures keep their own guidance. + ## Model selection All model inputs are optional and are passed directly to Core without action-side validation: diff --git a/action.yml b/action.yml index 3de0b22..c79652b 100644 --- a/action.yml +++ b/action.yml @@ -741,17 +741,21 @@ runs: # this same sticky comment with the input and secret to fix, and "see the workflow # logs" posted over the top of it would send the reader hunting for what they had # just been told. - - name: Post review failure + - name: Build review failure + id: review_failure if: failure() && steps.guard.outputs.skip != 'true' && steps.guard.outputs.mode == 'review' && steps.review_comment.outcome != 'success' && steps.llm.outputs.error == '' continue-on-error: true + shell: bash + env: + FAILURE_REASON: ${{ steps.review_baseline.outputs.failure_reason || steps.review_analyze.outputs.failure_reason }} + run: '"$GITHUB_ACTION_PATH/scripts/action/build-review-failure.sh"' + + - name: Post review failure + if: failure() && steps.review_failure.outputs.path != '' + continue-on-error: true uses: marocchino/sticky-pull-request-comment@v2 with: header: ${{ steps.guard.outputs.comment_id }} number: ${{ steps.guard.outputs.pr_number }} GITHUB_TOKEN: ${{ inputs.github_token }} - message: | - ### CodeBoarding review · failed - - ${{ steps.review_baseline.outputs.failure_reason || steps.review_analyze.outputs.failure_reason }} - - See the [workflow logs](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}). + path: ${{ steps.review_failure.outputs.path }} diff --git a/scripts/action/build-review-failure.sh b/scripts/action/build-review-failure.sh new file mode 100755 index 0000000..ff05d36 --- /dev/null +++ b/scripts/action/build-review-failure.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +# Builds the failure body shared by the PR comment and the job summary. +set -euo pipefail +BODY="${RUNNER_TEMP}/review-failure.md" +{ + printf '### CodeBoarding review · failed\n\n' + if [ -n "${FAILURE_REASON:-}" ]; then + printf '%s\n\n' "$FAILURE_REASON" + fi + printf 'See the [workflow logs](%s/%s/actions/runs/%s).\n' \ + "$GITHUB_SERVER_URL" "$GITHUB_REPOSITORY" "$GITHUB_RUN_ID" +} > "$BODY" +echo "path=$BODY" >> "$GITHUB_OUTPUT" +cat "$BODY" >> "$GITHUB_STEP_SUMMARY" diff --git a/scripts/action/with-auth.sh b/scripts/action/with-auth.sh index 57a8b00..bcd1d8f 100755 --- a/scripts/action/with-auth.sh +++ b/scripts/action/with-auth.sh @@ -30,6 +30,8 @@ for file in "$AUTH_DIR/env"/*; do done export CODEBOARDING_SOURCE=github_action +CODEBOARDING_LLM_TIER="$(cat "$AUTH_DIR/tier")" +export CODEBOARDING_LLM_TIER unset ACTIONS_ID_TOKEN_REQUEST_URL ACTIONS_ID_TOKEN_REQUEST_TOKEN if [ -n "${MODEL:-}" ]; then export AGENT_MODEL="$MODEL" diff --git a/scripts/analyze_repository.py b/scripts/analyze_repository.py index 446bdce..bf78056 100755 --- a/scripts/analyze_repository.py +++ b/scripts/analyze_repository.py @@ -17,8 +17,23 @@ ENGINE_REFUSALS = { 2: "The LLM provider rejected the API key. Check the key's secret and re-run.", 3: "The LLM provider's token or credit quota is exhausted. Add credits or raise the quota, then re-run.", + 4: ( + "Your GitHub owner's free CodeBoarding credits for this week are exhausted. " + "The allowance is shared across its repositories. " + "[Buy a CodeBoarding license](https://codeboarding.org/pricing) to continue now, " + "or wait for the weekly allowance to reset. After purchasing, add the `CODEBOARDING_LICENSE` " + "repository secret and set `llm: license` and `license_key: ${{ secrets.CODEBOARDING_LICENSE }}` " + "on each CodeBoarding Action invocation, then re-run." + ), } +LICENSE_NOT_ACCEPTED = ( + "The supplied CodeBoarding license did not grant paid access, and your GitHub owner's free credits " + "are exhausted. Check that the license is active and `license_key` uses the correct " + "`CODEBOARDING_LICENSE` secret. " + "[Manage or buy a CodeBoarding license](https://codeboarding.org/pricing), then re-run." +) + class AnalysisError(RuntimeError): pass @@ -88,10 +103,14 @@ def _run_command(args: list[str], output_dir: Path) -> str: # CLI stdout to stderr so engine progress remains visible in Actions. print(line, end="", file=sys.stderr, flush=True) + if process.stdout is not None: + process.stdout.close() return_code = process.wait() stdout = "".join(stdout_lines) if return_code != 0: reason = ENGINE_REFUSALS.get(return_code) + if return_code == 4 and os.getenv("CODEBOARDING_LLM_TIER") == "license": + reason = LICENSE_NOT_ACCEPTED if reason: # Written straight to the step's outputs: the shell never sees this script's stdout on failure. with open(os.environ.get("GITHUB_OUTPUT", os.devnull), "a", encoding="utf-8") as outputs: diff --git a/tests/test_action_inputs.py b/tests/test_action_inputs.py index 5fc96e7..c48dc1d 100644 --- a/tests/test_action_inputs.py +++ b/tests/test_action_inputs.py @@ -139,9 +139,12 @@ def test_the_generic_failure_comment_never_buries_the_actionable_one(self) -> No fix, then immediately replaces it with "see the workflow logs" -- sending the reader to hunt for what they had just been told. """ - start = ACTION.index("- name: Post review failure") - condition = ACTION[start : ACTION.index("message:", start)] + start = ACTION.index("- name: Build review failure") + condition = ACTION[start : ACTION.index("run:", start)] self.assertIn("steps.llm.outputs.error == ''", condition) + start = ACTION.index("- name: Post review failure") + condition = ACTION[start : ACTION.index("uses:", start)] + self.assertIn("steps.review_failure.outputs.path != ''", condition) def test_python_is_available_before_the_credential_check_runs(self) -> None: """The check is a Python program, so a runner without a system python3 would fail a diff --git a/tests/test_quota_comment.py b/tests/test_quota_comment.py new file mode 100644 index 0000000..e3a62f3 --- /dev/null +++ b/tests/test_quota_comment.py @@ -0,0 +1,105 @@ +"""Exercise core's subprocess refusal contract through the posted comment body.""" + +import os +import re +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "scripts")) +import analyze_repository as ar + +ROOT = Path(__file__).resolve().parent.parent +PRICING = "https://codeboarding.org/pricing" + + +class QuotaCommentTests(unittest.TestCase): + def _failure_body(self, exit_code: int = 4, tier: str = "hosted", mode: str = "review") -> tuple[str, str]: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + outputs = root / "analysis-outputs" + program = f"import sys; sys.exit({exit_code})" + with ( + patch.dict(os.environ, {"GITHUB_OUTPUT": str(outputs), "CODEBOARDING_LLM_TIER": tier}), + self.assertRaises(ar.AnalysisError), + ): + ar._run_command([sys.executable, "-c", program], root / "out") + + reason = outputs.read_text().removeprefix("failure_reason=").strip() + summary = root / "summary.md" + script = "build-review-failure.sh" if mode == "review" else "sync-summary.sh" + result = subprocess.run( + [str(ROOT / "scripts" / "action" / script)], + env={ + "PATH": os.environ["PATH"], + "RUNNER_TEMP": tmp, + "GITHUB_OUTPUT": str(root / "comment-outputs"), + "GITHUB_STEP_SUMMARY": str(summary), + "GITHUB_SERVER_URL": "https://github.com", + "GITHUB_REPOSITORY": "octocat/hello", + "GITHUB_RUN_ID": "123", + "FAILURE_REASON": reason, + }, + capture_output=True, + text=True, + check=True, + ) + self.assertEqual(result.stderr, "") + text = summary.read_text() + if mode == "review": + self.assertEqual(text, (root / "review-failure.md").read_text()) + self.assertIn(f"path={root / 'review-failure.md'}", (root / "comment-outputs").read_text()) + return text, reason + + def test_free_credits_comment_links_to_purchase_and_explains_setup(self): + body, reason = self._failure_body() + self.assertIn("### CodeBoarding review · failed", body) + self.assertIn("free CodeBoarding credits for this week are exhausted", body) + self.assertIn(f"[Buy a CodeBoarding license]({PRICING})", body) + self.assertIn("shared across its repositories", body) + self.assertIn("`CODEBOARDING_LICENSE`", body) + self.assertIn("`llm: license`", body) + self.assertIn("`license_key: ${{ secrets.CODEBOARDING_LICENSE }}`", body) + self.assertIn("https://github.com/octocat/hello/actions/runs/123", body) + self.assertNotIn("Traceback", body) + self.assertIn(reason, body) + + def test_sync_reports_the_same_purchase_instructions(self): + body, reason = self._failure_body(mode="sync") + self.assertIn(f"[Buy a CodeBoarding license]({PRICING})", body) + self.assertIn(reason, body) + + def test_supplied_license_is_checked_before_suggesting_another_purchase(self): + body, _ = self._failure_body(tier="license") + self.assertIn("Check that the license is active", body) + self.assertIn(f"]({PRICING})", body) + self.assertNotIn("After purchasing", body) + + def test_provider_billing_does_not_sell_a_codeboarding_license(self): + for tier in ("hosted", "license", "byok", "byok+license"): + with self.subTest(tier=tier): + body, _ = self._failure_body(exit_code=3, tier=tier) + self.assertNotIn(PRICING, body) + self.assertNotIn("Buy", body) + self.assertIn("LLM provider's token or credit quota is exhausted", body) + + def test_workflow_posts_the_tested_failure_file(self): + action = (ROOT / "action.yml").read_text() + steps = re.split(r"\n - name: ", action[action.index("\nruns:\n") :])[1:] + build = next(step for step in steps if step.startswith("Build review failure\n")) + post = next(step for step in steps if step.startswith("Post review failure\n")) + self.assertIn("failure()", build) + self.assertIn( + "steps.review_baseline.outputs.failure_reason || steps.review_analyze.outputs.failure_reason", build + ) + self.assertIn("build-review-failure.sh", build) + self.assertIn("marocchino/sticky-pull-request-comment@v2", post) + self.assertIn("path: ${{ steps.review_failure.outputs.path }}", post) + self.assertIn("header: ${{ steps.guard.outputs.comment_id }}", post) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_with_auth.py b/tests/test_with_auth.py index 49cfdbc..4e3745a 100644 --- a/tests/test_with_auth.py +++ b/tests/test_with_auth.py @@ -44,6 +44,17 @@ def test_a_step_that_keeps_them_leaves_them_for_the_next(self) -> None: self.assertEqual(self._run().returncode, 0, "the second analysis step found no credentials") self.assertFalse(self.auth.exists()) + def test_the_resolved_tier_reaches_the_analysis_adapter(self) -> None: + (self.auth / "tier").write_text("license", encoding="utf-8") + result = subprocess.run( + [str(WITH_AUTH), "bash", "-c", 'test "$CODEBOARDING_LLM_TIER" = license'], + env={"PATH": os.environ["PATH"], "RUNNER_TEMP": str(self.runner_temp)}, + capture_output=True, + text=True, + check=False, + ) + self.assertEqual(result.returncode, 0, result.stderr) + class ActionWiringTests(unittest.TestCase): def _steps(self) -> list[str]: