Repository navigation
feat: Run against a non-production platform with matching links (#1422) #14
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI (master evals) | |
| # Kept outside `on_master.yaml` so evals do not hold the release lock. | |
| on: | |
| push: | |
| branches: | |
| - master | |
| tags-ignore: | |
| - "**" # Ignore all tags, matching on_master.yaml. | |
| # Same set as on_pull_request_evals.yaml: a merge is re-gated on exactly what | |
| # opened the PR gate. Docs, tests and release commits skip both tiers. | |
| paths: | |
| - 'src/**' | |
| - 'evals/**' | |
| - 'package.json' | |
| - 'pnpm-lock.yaml' | |
| - 'pnpm-workspace.yaml' | |
| - 'tsconfig.json' | |
| - '.nvmrc' | |
| - '.github/workflows/_evaluations.yaml' | |
| - '.github/workflows/on_master_evals.yaml' | |
| jobs: | |
| evaluations_pr: | |
| name: MCP agent evaluations (pr) | |
| concurrency: | |
| group: eval-pr-master | |
| cancel-in-progress: false | |
| uses: ./.github/workflows/_evaluations.yaml | |
| with: | |
| tier: pr | |
| secrets: | |
| ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} | |
| LANGFUSE_PUBLIC_KEY: ${{ secrets.LANGFUSE_PUBLIC_KEY }} | |
| LANGFUSE_SECRET_KEY: ${{ secrets.LANGFUSE_SECRET_KEY }} | |
| LANGFUSE_BASE_URL: ${{ secrets.LANGFUSE_BASE_URL }} | |
| APIFY_TOKEN: ${{ secrets.APIFY_TEST_USER_API_TOKEN }} | |
| evaluations_merge: | |
| name: MCP agent evaluations (merge) | |
| # Serialized: the merge tier's `eval-*` task fixtures are shared per account. | |
| concurrency: | |
| group: eval-merge-master | |
| cancel-in-progress: false | |
| uses: ./.github/workflows/_evaluations.yaml | |
| with: | |
| tier: merge | |
| secrets: | |
| ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} | |
| LANGFUSE_PUBLIC_KEY: ${{ secrets.LANGFUSE_PUBLIC_KEY }} | |
| LANGFUSE_SECRET_KEY: ${{ secrets.LANGFUSE_SECRET_KEY }} | |
| LANGFUSE_BASE_URL: ${{ secrets.LANGFUSE_BASE_URL }} | |
| APIFY_TOKEN: ${{ secrets.APIFY_TEST_USER_API_TOKEN }} | |
| OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} |