apify-mcp-server/.github/workflows/on_master_evals.yaml at master · apify/apify-mcp-server · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
name: CI (master evals)

# Kept outside `on_master.yaml` so evals do not hold the release lock.
on:
    push:
        branches:
            - master
        tags-ignore:
            - "**" # Ignore all tags, matching on_master.yaml.
        # Same set as on_pull_request_evals.yaml: a merge is re-gated on exactly what
        # opened the PR gate. Docs, tests and release commits skip both tiers.
        paths:
            - 'src/**'
            - 'evals/**'
            - 'package.json'
            - 'pnpm-lock.yaml'
            - 'pnpm-workspace.yaml'
            - 'tsconfig.json'
            - '.nvmrc'
            - '.github/workflows/_evaluations.yaml'
            - '.github/workflows/on_master_evals.yaml'

jobs:
    evaluations_pr:
        name: MCP agent evaluations (pr)
        concurrency:
            group: eval-pr-master
            cancel-in-progress: false
        uses: ./.github/workflows/_evaluations.yaml
        with:
            tier: pr
        secrets:
            ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
            LANGFUSE_PUBLIC_KEY: ${{ secrets.LANGFUSE_PUBLIC_KEY }}
            LANGFUSE_SECRET_KEY: ${{ secrets.LANGFUSE_SECRET_KEY }}
            LANGFUSE_BASE_URL: ${{ secrets.LANGFUSE_BASE_URL }}
            APIFY_TOKEN: ${{ secrets.APIFY_TEST_USER_API_TOKEN }}

    evaluations_merge:
        name: MCP agent evaluations (merge)
        # Serialized: the merge tier's `eval-*` task fixtures are shared per account.
        concurrency:
            group: eval-merge-master
            cancel-in-progress: false
        uses: ./.github/workflows/_evaluations.yaml
        with:
            tier: merge
        secrets:
            ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
            LANGFUSE_PUBLIC_KEY: ${{ secrets.LANGFUSE_PUBLIC_KEY }}
            LANGFUSE_SECRET_KEY: ${{ secrets.LANGFUSE_SECRET_KEY }}
            LANGFUSE_BASE_URL: ${{ secrets.LANGFUSE_BASE_URL }}
            APIFY_TOKEN: ${{ secrets.APIFY_TEST_USER_API_TOKEN }}
            OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}