name: Prompt regression

on:
  pull_request:
    paths:
      - "prompts/**"
      - "evals/**"
      - "scripts/prompt_regress.py"
  workflow_dispatch:

permissions:
  contents: read

concurrency:
  group: prompt-regression-${{ github.ref }}
  cancel-in-progress: true

jobs:
  regress:
    runs-on: ubuntu-latest
    timeout-minutes: 15
    steps:
      - uses: actions/checkout@v4

      - uses: actions/setup-python@v5
        with:
          python-version: "3.12"

      # Secrets are not passed to workflows triggered from forks, so skip instead of failing.
      - name: Check for API key
        id: key
        env:
          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
        run: |
          if [ -z "$ANTHROPIC_API_KEY" ]; then
            echo "present=false" >> "$GITHUB_OUTPUT"
            echo "::notice::ANTHROPIC_API_KEY is not available; skipping prompt regression."
          else
            echo "present=true" >> "$GITHUB_OUTPUT"
          fi

      - name: Install SDK
        if: steps.key.outputs.present == 'true'
        run: python -m pip install --disable-pip-version-check anthropic

      - name: Run regression cases
        if: steps.key.outputs.present == 'true'
        env:
          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
          ANTHROPIC_MODEL: ${{ vars.ANTHROPIC_MODEL }}
        run: python scripts/prompt_regress.py evals/cases.jsonl --out eval-results.jsonl

      - name: Upload results
        if: always() && steps.key.outputs.present == 'true'
        uses: actions/upload-artifact@v4
        with:
          name: prompt-regression-results
          path: eval-results.jsonl
          if-no-files-found: ignore