Repository navigation
fix(evals): Make schedule eval runs safe to run in parallel #1544
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI (PR label) | |
| on: | |
| # Also trigger in case the PR labels change so that we can re-check the execution condition | |
| pull_request: | |
| types: [labeled, synchronize, reopened] | |
| jobs: | |
| wait_for_checks: | |
| name: Wait for code checks to pass | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Wait for existing checks or skip if none | |
| uses: lewagon/wait-on-check-action@v1.3.4 | |
| with: | |
| ref: ${{ github.event.pull_request.head.sha }} | |
| repo-token: ${{ secrets.GITHUB_TOKEN }} | |
| check-regexp: (Code checks) | |
| wait-interval: 10 | |
| running-workflow-name: 'Wait for code checks to pass' | |
| allowed-conclusions: success,neutral,skipped | |
| continue-on-error: false | |
| push_pkg_pr_new: | |
| needs: [ wait_for_checks ] | |
| # Run ONLY when PR has the "beta" label | |
| if: contains(github.event.pull_request.labels.*.name, 'beta') | |
| name: Push to pkg.pr.new | |
| # Only publishing shares the release lock with `on_master.yaml`. | |
| concurrency: | |
| group: release | |
| cancel-in-progress: false | |
| uses: ./.github/workflows/_publish_to_pkg_pr_new.yaml | |
| with: | |
| ref: ${{ github.event.pull_request.head.ref }} | |
| evaluations: | |
| needs: [ wait_for_checks ] | |
| name: MCP agent evaluations (pr) | |
| # Reruns a skipped same-repo PR. Fork PRs do not receive secrets. | |
| if: | | |
| github.event.action == 'labeled' && github.event.label.name == 'validated' && | |
| github.event.pull_request.head.repo.full_name == github.repository | |
| concurrency: | |
| group: eval-pr-${{ github.event.pull_request.number }} | |
| cancel-in-progress: true | |
| uses: ./.github/workflows/_evaluations.yaml | |
| with: | |
| tier: pr | |
| secrets: | |
| ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} | |
| LANGFUSE_PUBLIC_KEY: ${{ secrets.LANGFUSE_PUBLIC_KEY }} | |
| LANGFUSE_SECRET_KEY: ${{ secrets.LANGFUSE_SECRET_KEY }} | |
| LANGFUSE_BASE_URL: ${{ secrets.LANGFUSE_BASE_URL }} | |
| APIFY_TOKEN: ${{ secrets.APIFY_TEST_USER_API_TOKEN }} |