ruff/.github/workflows/typing_conformance.yaml

name: Run typing conformance

permissions: {}

on:
  pull_request:
    paths:
      - "crates/ty*/**"
      - "crates/ruff_db"
      - "crates/ruff_python_ast"
      - "crates/ruff_python_parser"
      - ".github/workflows/typing_conformance.yaml"
      - ".github/workflows/typing_conformance_comment.yaml"
      - "Cargo.lock"
      - "!**.md"

concurrency:
  group: ${{ github.workflow }}-${{ github.ref_name }}-${{ github.event.pull_request.number || github.sha }}
  cancel-in-progress: true

env:
  CARGO_INCREMENTAL: 0
  CARGO_NET_RETRY: 10
  CARGO_TERM_COLOR: always
  RUSTUP_MAX_RETRIES: 10
  RUST_BACKTRACE: 1
  CONFORMANCE_SUITE_COMMIT: 9f6d8ced7cd1c8d92687a4e9c96d7716452e471e

jobs:
  typing_conformance:
    name: Compute diagnostic diff
    runs-on: ${{ github.repository == 'astral-sh/ruff' && 'depot-ubuntu-22.04-32' || 'ubuntu-latest' }}
    timeout-minutes: 10
    steps:
      - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
        with:
          path: ruff
          fetch-depth: 0
          persist-credentials: false

      - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
        with:
          repository: python/typing
          ref: ${{ env.CONFORMANCE_SUITE_COMMIT }}
          path: typing
          persist-credentials: false

      - uses: Swatinem/rust-cache@f13886b937689c021905a6b90929199931d60db1 # v2.8.1
        with:
          workspaces: "ruff"

      - name: Install Rust toolchain
        run: rustup show

      - name: Compute diagnostic diff
        shell: bash
        env:
          # TODO: Remove this once we fixed the remaining panics in the conformance suite.
          TY_MAX_PARALLELISM: 1
        run: |
          RUFF_DIR="$GITHUB_WORKSPACE/ruff"

          # Build the executable for the old and new commit
          (
            cd ruff

            echo "new commit"
            git rev-list --format=%s --max-count=1 "$GITHUB_SHA"
            cargo build --bin ty
            mv target/debug/ty ty-new

            MERGE_BASE="$(git merge-base "$GITHUB_SHA" "origin/$GITHUB_BASE_REF")"
            git checkout -b old_commit "$MERGE_BASE"
            echo "old commit (merge base)"
            git rev-list --format=%s --max-count=1 old_commit
            cargo build --bin ty
            mv target/debug/ty ty-old
          )

          (
            cd typing/conformance/tests

            echo "Running ty on old commit (merge base)"
            "$RUFF_DIR/ty-old" check --color=never --output-format=concise . > "$GITHUB_WORKSPACE/old-output.txt" 2>&1 || true

            echo "Running ty on new commit"
            "$RUFF_DIR/ty-new" check --color=never --output-format=concise . > "$GITHUB_WORKSPACE/new-output.txt" 2>&1 || true
          )

          if ! diff -u old-output.txt new-output.txt > typing_conformance_diagnostics.diff; then
            echo "Differences found between base and PR"
          else
            echo "No differences found"
            touch typing_conformance_diagnostics.diff
          fi

          echo ${{ github.event.number }} > pr-number
          echo "${CONFORMANCE_SUITE_COMMIT}" > conformance-suite-commit

      - name: Upload diff
        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
        with:
          name: typing_conformance_diagnostics_diff
          path: typing_conformance_diagnostics.diff

      - name: Upload pr-number
        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
        with:
          name: pr-number
          path: pr-number

      - name: Upload conformance suite commit
        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
        with:
          name: conformance-suite-commit
          path: conformance-suite-commit