attempt to shard mypy-primer

2025-04-18 17:21:17 -07:00
parent b78112aae4
commit f218b228e3
3 changed files with 95 additions and 86 deletions
--- a/.github/workflows/mypy_primer.yaml
+++ b/.github/workflows/mypy_primer.yaml
@@ -26,6 +26,10 @@ jobs:
  mypy_primer:
    name: Run mypy_primer
    runs-on: depot-ubuntu-22.04-16
+    strategy:
+      matrix:
+        shard-index: [0, 1, 2]
+      fail-fast: false
    timeout-minutes: 20
    steps:
      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
@@ -64,6 +68,8 @@ jobs:

          cd ..

+          DIFF_FILE="mypy_primer_${{ matrix.shard-index }}.diff"
+
          echo "Project selector: $PRIMER_SELECTOR"
          # Allow the exit code to be 0 or 1, only fail for actual mypy_primer crashes/bugs
          uvx mypy_primer \
@@ -71,26 +77,31 @@ jobs:
            --type-checker knot \
            --old base_commit \
            --new "$GITHUB_SHA" \
+            --num-shards 3 --shard-index ${{ matrix.shard-index }} \
            --project-selector "/($PRIMER_SELECTOR)\$" \
            --output concise \
-            --debug > mypy_primer.diff || [ $? -eq 1 ]
+            --debug > $DIFF_FILE || [ $? -eq 1 ]

          # Output diff with ANSI color codes
-          cat mypy_primer.diff
+          cat $DIFF_FILE

          # Remove ANSI color codes before uploading
-          sed -ie 's/\x1b\[[0-9;]*m//g' mypy_primer.diff
+          sed -ie 's/\x1b\[[0-9;]*m//g' $DIFF_FILE

-          echo ${{ github.event.number }} > pr-number
+      - if: ${{ matrix.shard-index == 0 }}
+        name: Save PR number
+        run: |
+          echo ${{ github.event.pull_request.number }} | tee pr-number

      - name: Upload diff
        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
        with:
-          name: mypy_primer_diff
-          path: mypy_primer.diff
+          name: mypy_primer_diffs-${{ matrix.shard-index }}
+          path: mypy_primer_${{ matrix.shard-index }}.diff

      - name: Upload pr-number
        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        if: ${{ matrix.shard-index == 0 }}
        with:
          name: pr-number
          path: pr-number
--- a/.github/workflows/mypy_primer_comment.yaml
+++ b/.github/workflows/mypy_primer_comment.yaml
@@ -1,97 +1,96 @@
-name: PR comment (mypy_primer)
-
 on: # zizmor: ignore[dangerous-triggers]
  workflow_run:
-    workflows: [Run mypy_primer]
-    types: [completed]
-  workflow_dispatch:
-    inputs:
-      workflow_run_id:
-        description: The mypy_primer workflow that triggers the workflow run
-        required: true
+    workflows:
+      - Run mypy_primer
+    types:
+      - completed
+
+permissions: {}

 jobs:
  comment:
-    runs-on: ubuntu-24.04
+    name: Comment PR from mypy_primer
+    runs-on: ubuntu-latest
    permissions:
+      contents: read
      pull-requests: write
    steps:
-      - uses: dawidd6/action-download-artifact@20319c5641d495c8a52e688b7dc5fada6c3a9fbc # v8
-        name: Download PR number
+      - name: Download diffs
+        uses: actions/github-script@v7
        with:
-          name: pr-number
-          run_id: ${{ github.event.workflow_run.id ||  github.event.inputs.workflow_run_id }}
-          if_no_artifact_found: ignore
-          allow_forks: true
+          script: |
+            const fs = require('fs');
+            const artifacts = await github.rest.actions.listWorkflowRunArtifacts({
+               owner: context.repo.owner,
+               repo: context.repo.repo,
+               run_id: ${{ github.event.workflow_run.id }},
+            });
+            const [matchArtifact] = artifacts.data.artifacts.filter((artifact) =>
+              artifact.name == "mypy_primer_diffs");

-      - name: Parse pull request number
-        id: pr-number
-        run: |
-          if [[ -f pr-number ]]
-          then
-            echo "pr-number=$(<pr-number)" >> "$GITHUB_OUTPUT"
-          fi
+            const download = await github.rest.actions.downloadArtifact({
+               owner: context.repo.owner,
+               repo: context.repo.repo,
+               artifact_id: matchArtifact.id,
+               archive_format: "zip",
+            });
+            fs.writeFileSync("diff.zip", Buffer.from(download.data));

-      - uses: dawidd6/action-download-artifact@20319c5641d495c8a52e688b7dc5fada6c3a9fbc # v8
-        name: "Download mypy_primer results"
-        id: download-mypy_primer_diff
-        if: steps.pr-number.outputs.pr-number
+      - run: unzip diff.zip
+      - run: |
+          cat mypy_primer_*.diff | tee fulldiff.txt
+
+      - name: Post comment
+        id: post-comment
+        uses: actions/github-script@v7
        with:
-          name: mypy_primer_diff
-          workflow: mypy_primer.yaml
-          pr: ${{ steps.pr-number.outputs.pr-number }}
-          path: pr/mypy_primer_diff
-          workflow_conclusion: completed
-          if_no_artifact_found: ignore
-          allow_forks: true
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          script: |
+            const MAX_CHARACTERS = 50000
+            const MAX_CHARACTERS_PER_PROJECT = MAX_CHARACTERS / 3

-      - name: Generate comment content
-        id: generate-comment
-        if: steps.download-mypy_primer_diff.outputs.found_artifact == 'true'
-        run: |
-          # Guard against malicious mypy_primer results that symlink to a secret
-          # file on this runner
-          if [[ -L pr/mypy_primer_diff/mypy_primer.diff ]]
-          then
-              echo "Error: mypy_primer.diff cannot be a symlink"
-              exit 1
-          fi
+            const fs = require('fs')
+            let data = fs.readFileSync('fulldiff.txt', { encoding: 'utf8' })

-          # Note this identifier is used to find the comment to update on
-          # subsequent runs
-          echo '<!-- generated-comment mypy_primer -->' >> comment.txt
+            function truncateIfNeeded(original, maxLength) {
+              if (original.length <= maxLength) {
+                return original
+              }
+              let truncated = original.substring(0, maxLength)
+              // further, remove last line that might be truncated
+              truncated = truncated.substring(0, truncated.lastIndexOf('\n'))
+              let lines_truncated = original.split('\n').length - truncated.split('\n').length
+              return `${truncated}\n\n... (truncated ${lines_truncated} lines) ...`
+            }

-          echo '## `mypy_primer` results' >> comment.txt
-          if [ -s "pr/mypy_primer_diff/mypy_primer.diff" ]; then
-            echo '<details>' >> comment.txt
-            echo '<summary>Changes were detected when running on open source projects</summary>' >> comment.txt
-            echo '' >> comment.txt
-            echo '```diff' >> comment.txt
-            cat pr/mypy_primer_diff/mypy_primer.diff >> comment.txt
-            echo '```' >> comment.txt
-            echo '</details>' >> comment.txt
-          else
-            echo 'No ecosystem changes detected ✅' >> comment.txt
-          fi
+            const projects = data.split('\n\n')
+            // don't let one project dominate
+            data = projects.map(project => truncateIfNeeded(project, MAX_CHARACTERS_PER_PROJECT)).join('\n\n')
+            // posting comment fails if too long, so truncate
+            data = truncateIfNeeded(data, MAX_CHARACTERS)

-          echo 'comment<<EOF' >> "$GITHUB_OUTPUT"
-          cat comment.txt >> "$GITHUB_OUTPUT"
-          echo 'EOF' >> "$GITHUB_OUTPUT"
+            console.log("Diff from mypy_primer:")
+            console.log(data)

-      - name: Find existing comment
-        uses: peter-evans/find-comment@3eae4d37986fb5a8592848f6a574fdf654e61f9e # v3
-        if: steps.generate-comment.outcome == 'success'
-        id: find-comment
+            let body
+            if (data.trim()) {
+              body = 'Diff from [mypy_primer](https://github.com/hauntsaninja/mypy_primer), showing the effect of this PR on open source code:\n```diff\n' + data + '```'
+            } else {
+              body = "According to [mypy_primer](https://github.com/hauntsaninja/mypy_primer), this change doesn't affect type check results on a corpus of open source code. ✅"
+            }
+            const prNumber = parseInt(fs.readFileSync("pr-number", { encoding: "utf8" }))
+            await github.rest.issues.createComment({
+              issue_number: prNumber,
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              body
+            })
+            return prNumber
+
+      - name: Hide old comments
+        # v0.4.0
+        uses: kanga333/comment-hider@c12bb20b48aeb8fc098e35967de8d4f8018fffdf
        with:
-          issue-number: ${{ steps.pr-number.outputs.pr-number }}
-          comment-author: "github-actions[bot]"
-          body-includes: "<!-- generated-comment mypy_primer -->"
-
-      - name: Create or update comment
-        if: steps.find-comment.outcome == 'success'
-        uses: peter-evans/create-or-update-comment@71345be0265236311c031f5c7866368bd1eff043 # v4
-        with:
-          comment-id: ${{ steps.find-comment.outputs.comment-id }}
-          issue-number: ${{ steps.pr-number.outputs.pr-number }}
-          body-path: comment.txt
-          edit-mode: replace
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          leave_visible: 1
+          issue_number: ${{ steps.post-comment.outputs.result }}
--- a/crates/red_knot_python_semantic/resources/primer/bad.txt
+++ b/crates/red_knot_python_semantic/resources/primer/bad.txt
@@ -7,7 +7,6 @@ artigraph  # cycle panics (value_type_)
 cpython  # missing expression ID, access to field whilst being initialized, too many cycle iterations
 colour  # cycle panics (try_metaclass_)
 core  # cycle panics (value_type_)
-discord.py  # some kind of hang, only when multi-threaded?
 dragonchain  # too many cycle iterations (member_lookup_with_policy)
 manticore  # stack overflow
 materialize  # stack overflow