temporarily change output to see if the comment works

.github/workflows/mypy_primer.yaml

@@ -26,6 +26,10 @@ jobs:
   mypy_primer:
     name: Run mypy_primer
     runs-on: depot-ubuntu-22.04-16
+    strategy:
+      matrix:
+        shard-index: [0, 1, 2]
+      fail-fast: false
     timeout-minutes: 20
     steps:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
@@ -64,6 +68,8 @@ jobs:
 
           cd ..
 
+          DIFF_FILE="mypy_primer_${{ matrix.shard-index }}.diff"
+
           echo "Project selector: $PRIMER_SELECTOR"
           # Allow the exit code to be 0 or 1, only fail for actual mypy_primer crashes/bugs
           uvx mypy_primer \
@@ -71,26 +77,31 @@ jobs:
             --type-checker knot \
             --old base_commit \
             --new "$GITHUB_SHA" \
+            --num-shards 3 --shard-index ${{ matrix.shard-index }} \
             --project-selector "/($PRIMER_SELECTOR)\$" \
             --output concise \
-            --debug > mypy_primer.diff || [ $? -eq 1 ]
+            --debug > $DIFF_FILE || [ $? -eq 1 ]
 
           # Output diff with ANSI color codes
-          cat mypy_primer.diff
+          cat $DIFF_FILE
 
           # Remove ANSI color codes before uploading
-          sed -ie 's/\x1b\[[0-9;]*m//g' mypy_primer.diff
+          sed -ie 's/\x1b\[[0-9;]*m//g' $DIFF_FILE
 
-          echo ${{ github.event.number }} > pr-number
+      - if: ${{ matrix.shard-index == 0 }}
+        name: Save PR number
+        run: |
+          echo ${{ github.event.pull_request.number }} | tee pr-number
 
       - name: Upload diff
         uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
         with:
-          name: mypy_primer_diff
-          path: mypy_primer.diff
+          name: mypy_primer_diffs-${{ matrix.shard-index }}
+          path: mypy_primer_${{ matrix.shard-index }}.diff
 
       - name: Upload pr-number
         uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        if: ${{ matrix.shard-index == 0 }}
         with:
           name: pr-number
           path: pr-number

.github/workflows/mypy_primer_comment.yaml

@@ -1,97 +1,96 @@
-name: PR comment (mypy_primer)
-
 on: # zizmor: ignore[dangerous-triggers]
   workflow_run:
-    workflows: [Run mypy_primer]
-    types: [completed]
-  workflow_dispatch:
-    inputs:
-      workflow_run_id:
-        description: The mypy_primer workflow that triggers the workflow run
-        required: true
+    workflows:
+      - Run mypy_primer
+    types:
+      - completed
+
+permissions: {}
 
 jobs:
   comment:
-    runs-on: ubuntu-24.04
+    name: Comment PR from mypy_primer
+    runs-on: ubuntu-latest
     permissions:
+      contents: read
       pull-requests: write
     steps:
-      - uses: dawidd6/action-download-artifact@20319c5641d495c8a52e688b7dc5fada6c3a9fbc # v8
-        name: Download PR number
+      - name: Download diffs
+        uses: actions/github-script@v7
         with:
-          name: pr-number
-          run_id: ${{ github.event.workflow_run.id ||  github.event.inputs.workflow_run_id }}
-          if_no_artifact_found: ignore
-          allow_forks: true
+          script: |
+            const fs = require('fs');
+            const artifacts = await github.rest.actions.listWorkflowRunArtifacts({
+               owner: context.repo.owner,
+               repo: context.repo.repo,
+               run_id: ${{ github.event.workflow_run.id }},
+            });
+            const [matchArtifact] = artifacts.data.artifacts.filter((artifact) =>
+              artifact.name == "mypy_primer_diffs");
 
-      - name: Parse pull request number
-        id: pr-number
-        run: |
-          if [[ -f pr-number ]]
-          then
-            echo "pr-number=$(<pr-number)" >> "$GITHUB_OUTPUT"
-          fi
+            const download = await github.rest.actions.downloadArtifact({
+               owner: context.repo.owner,
+               repo: context.repo.repo,
+               artifact_id: matchArtifact.id,
+               archive_format: "zip",
+            });
+            fs.writeFileSync("diff.zip", Buffer.from(download.data));
 
-      - uses: dawidd6/action-download-artifact@20319c5641d495c8a52e688b7dc5fada6c3a9fbc # v8
-        name: "Download mypy_primer results"
-        id: download-mypy_primer_diff
-        if: steps.pr-number.outputs.pr-number
+      - run: unzip diff.zip
+      - run: |
+          cat mypy_primer_*.diff | tee fulldiff.txt
+
+      - name: Post comment
+        id: post-comment
+        uses: actions/github-script@v7
         with:
-          name: mypy_primer_diff
-          workflow: mypy_primer.yaml
-          pr: ${{ steps.pr-number.outputs.pr-number }}
-          path: pr/mypy_primer_diff
-          workflow_conclusion: completed
-          if_no_artifact_found: ignore
-          allow_forks: true
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          script: |
+            const MAX_CHARACTERS = 50000
+            const MAX_CHARACTERS_PER_PROJECT = MAX_CHARACTERS / 3
 
-      - name: Generate comment content
-        id: generate-comment
-        if: steps.download-mypy_primer_diff.outputs.found_artifact == 'true'
-        run: |
-          # Guard against malicious mypy_primer results that symlink to a secret
-          # file on this runner
-          if [[ -L pr/mypy_primer_diff/mypy_primer.diff ]]
-          then
-              echo "Error: mypy_primer.diff cannot be a symlink"
-              exit 1
-          fi
+            const fs = require('fs')
+            let data = fs.readFileSync('fulldiff.txt', { encoding: 'utf8' })
 
-          # Note this identifier is used to find the comment to update on
-          # subsequent runs
-          echo '<!-- generated-comment mypy_primer -->' >> comment.txt
+            function truncateIfNeeded(original, maxLength) {
+              if (original.length <= maxLength) {
+                return original
+              }
+              let truncated = original.substring(0, maxLength)
+              // further, remove last line that might be truncated
+              truncated = truncated.substring(0, truncated.lastIndexOf('\n'))
+              let lines_truncated = original.split('\n').length - truncated.split('\n').length
+              return `${truncated}\n\n... (truncated ${lines_truncated} lines) ...`
+            }
 
-          echo '## `mypy_primer` results' >> comment.txt
-          if [ -s "pr/mypy_primer_diff/mypy_primer.diff" ]; then
-            echo '<details>' >> comment.txt
-            echo '<summary>Changes were detected when running on open source projects</summary>' >> comment.txt
-            echo '' >> comment.txt
-            echo '```diff' >> comment.txt
-            cat pr/mypy_primer_diff/mypy_primer.diff >> comment.txt
-            echo '```' >> comment.txt
-            echo '</details>' >> comment.txt
-          else
-            echo 'No ecosystem changes detected ✅' >> comment.txt
-          fi
+            const projects = data.split('\n\n')
+            // don't let one project dominate
+            data = projects.map(project => truncateIfNeeded(project, MAX_CHARACTERS_PER_PROJECT)).join('\n\n')
+            // posting comment fails if too long, so truncate
+            data = truncateIfNeeded(data, MAX_CHARACTERS)
 
-          echo 'comment<<EOF' >> "$GITHUB_OUTPUT"
-          cat comment.txt >> "$GITHUB_OUTPUT"
-          echo 'EOF' >> "$GITHUB_OUTPUT"
+            console.log("Diff from mypy_primer:")
+            console.log(data)
 
-      - name: Find existing comment
-        uses: peter-evans/find-comment@3eae4d37986fb5a8592848f6a574fdf654e61f9e # v3
-        if: steps.generate-comment.outcome == 'success'
-        id: find-comment
+            let body
+            if (data.trim()) {
+              body = 'Diff from [mypy_primer](https://github.com/hauntsaninja/mypy_primer), showing the effect of this PR on open source code:\n```diff\n' + data + '```'
+            } else {
+              body = "According to [mypy_primer](https://github.com/hauntsaninja/mypy_primer), this change doesn't affect type check results on a corpus of open source code. ✅"
+            }
+            const prNumber = parseInt(fs.readFileSync("pr-number", { encoding: "utf8" }))
+            await github.rest.issues.createComment({
+              issue_number: prNumber,
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              body
+            })
+            return prNumber
+
+      - name: Hide old comments
+        # v0.4.0
+        uses: kanga333/comment-hider@c12bb20b48aeb8fc098e35967de8d4f8018fffdf
         with:
-          issue-number: ${{ steps.pr-number.outputs.pr-number }}
-          comment-author: "github-actions[bot]"
-          body-includes: "<!-- generated-comment mypy_primer -->"
-
-      - name: Create or update comment
-        if: steps.find-comment.outcome == 'success'
-        uses: peter-evans/create-or-update-comment@71345be0265236311c031f5c7866368bd1eff043 # v4
-        with:
-          comment-id: ${{ steps.find-comment.outputs.comment-id }}
-          issue-number: ${{ steps.pr-number.outputs.pr-number }}
-          body-path: comment.txt
-          edit-mode: replace
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          leave_visible: 1
+          issue_number: ${{ steps.post-comment.outputs.result }}

crates/red_knot/src/main.rs

@@ -295,7 +295,7 @@ impl MainLoop {
 
                             writeln!(
                                 stdout,
-                                "Found {} diagnostic{}",
+                                "Founded {} diagnostic{}",
                                 diagnostics_count,
                                 if diagnostics_count > 1 { "s" } else { "" }
                             )?;

crates/red_knot/tests/cli.rs

@@ -28,7 +28,7 @@ fn config_override_python_version() -> anyhow::Result<()> {
         ),
     ])?;
 
-    assert_cmd_snapshot!(case.command(), @r"
+    assert_cmd_snapshot!(case.command(), @r###"
     success: false
     exit_code: 1
     ----- stdout -----
@@ -40,10 +40,10 @@ fn config_override_python_version() -> anyhow::Result<()> {
       |       ^^^^^^^^^^^^ Type `<module 'sys'>` has no attribute `last_exc`
       |
 
-    Found 1 diagnostic
+    Founded 1 diagnostic
 
     ----- stderr -----
-    ");
+    "###);
 
     assert_cmd_snapshot!(case.command().arg("--python-version").arg("3.12"), @r"
     success: true
@@ -79,7 +79,7 @@ fn config_override_python_platform() -> anyhow::Result<()> {
         ),
     ])?;
 
-    assert_cmd_snapshot!(case.command(), @r#"
+    assert_cmd_snapshot!(case.command(), @r###"
     success: true
     exit_code: 0
     ----- stdout -----
@@ -92,12 +92,12 @@ fn config_override_python_platform() -> anyhow::Result<()> {
       | ^^^^^^^^^^^^^^^^^^^^^^^^^ `Literal["linux"]`
       |
 
-    Found 1 diagnostic
+    Founded 1 diagnostic
 
     ----- stderr -----
-    "#);
+    "###);
 
-    assert_cmd_snapshot!(case.command().arg("--python-platform").arg("all"), @r"
+    assert_cmd_snapshot!(case.command().arg("--python-platform").arg("all"), @r###"
     success: true
     exit_code: 0
     ----- stdout -----
@@ -110,10 +110,10 @@ fn config_override_python_platform() -> anyhow::Result<()> {
       | ^^^^^^^^^^^^^^^^^^^^^^^^^ `LiteralString`
       |
 
-    Found 1 diagnostic
+    Founded 1 diagnostic
 
     ----- stderr -----
-    ");
+    "###);
 
     Ok(())
 }
@@ -161,7 +161,7 @@ fn cli_arguments_are_relative_to_the_current_directory() -> anyhow::Result<()> {
     ])?;
 
     // Make sure that the CLI fails when the `libs` directory is not in the search path.
-    assert_cmd_snapshot!(case.command().current_dir(case.root().join("child")), @r"
+    assert_cmd_snapshot!(case.command().current_dir(case.root().join("child")), @r###"
     success: false
     exit_code: 1
     ----- stdout -----
@@ -174,10 +174,10 @@ fn cli_arguments_are_relative_to_the_current_directory() -> anyhow::Result<()> {
     4 | stat = add(10, 15)
       |
 
-    Found 1 diagnostic
+    Founded 1 diagnostic
 
     ----- stderr -----
-    ");
+    "###);
 
     assert_cmd_snapshot!(case.command().current_dir(case.root().join("child")).arg("--extra-search-path").arg("../libs"), @r"
     success: true
@@ -261,7 +261,7 @@ fn configuration_rule_severity() -> anyhow::Result<()> {
 
     // Assert that there's a possibly unresolved reference diagnostic
     // and that division-by-zero has a severity of error by default.
-    assert_cmd_snapshot!(case.command(), @r"
+    assert_cmd_snapshot!(case.command(), @r###"
     success: false
     exit_code: 1
     ----- stdout -----
@@ -283,10 +283,10 @@ fn configuration_rule_severity() -> anyhow::Result<()> {
       |       ^ Name `x` used when possibly not defined
       |
 
-    Found 2 diagnostics
+    Founded 2 diagnostics
 
     ----- stderr -----
-    ");
+    "###);
 
     case.write_file(
         "pyproject.toml",
@@ -297,7 +297,7 @@ fn configuration_rule_severity() -> anyhow::Result<()> {
     "#,
     )?;
 
-    assert_cmd_snapshot!(case.command(), @r"
+    assert_cmd_snapshot!(case.command(), @r###"
     success: true
     exit_code: 0
     ----- stdout -----
@@ -310,10 +310,10 @@ fn configuration_rule_severity() -> anyhow::Result<()> {
     4 | for a in range(0, int(y)):
       |
 
-    Found 1 diagnostic
+    Founded 1 diagnostic
 
     ----- stderr -----
-    ");
+    "###);
 
     Ok(())
 }
@@ -337,7 +337,7 @@ fn cli_rule_severity() -> anyhow::Result<()> {
 
     // Assert that there's a possibly unresolved reference diagnostic
     // and that division-by-zero has a severity of error by default.
-    assert_cmd_snapshot!(case.command(), @r"
+    assert_cmd_snapshot!(case.command(), @r###"
     success: false
     exit_code: 1
     ----- stdout -----
@@ -370,10 +370,10 @@ fn cli_rule_severity() -> anyhow::Result<()> {
       |       ^ Name `x` used when possibly not defined
       |
 
-    Found 3 diagnostics
+    Founded 3 diagnostics
 
     ----- stderr -----
-    ");
+    "###);
 
     assert_cmd_snapshot!(
         case
@@ -384,7 +384,7 @@ fn cli_rule_severity() -> anyhow::Result<()> {
             .arg("division-by-zero")
             .arg("--warn")
             .arg("unresolved-import"),
-        @r"
+        @r###"
     success: true
     exit_code: 0
     ----- stdout -----
@@ -408,10 +408,10 @@ fn cli_rule_severity() -> anyhow::Result<()> {
     6 | for a in range(0, int(y)):
       |
 
-    Found 2 diagnostics
+    Founded 2 diagnostics
 
     ----- stderr -----
-    "
+    "###
     );
 
     Ok(())
@@ -435,7 +435,7 @@ fn cli_rule_severity_precedence() -> anyhow::Result<()> {
 
     // Assert that there's a possibly unresolved reference diagnostic
     // and that division-by-zero has a severity of error by default.
-    assert_cmd_snapshot!(case.command(), @r"
+    assert_cmd_snapshot!(case.command(), @r###"
     success: false
     exit_code: 1
     ----- stdout -----
@@ -457,10 +457,10 @@ fn cli_rule_severity_precedence() -> anyhow::Result<()> {
       |       ^ Name `x` used when possibly not defined
       |
 
-    Found 2 diagnostics
+    Founded 2 diagnostics
 
     ----- stderr -----
-    ");
+    "###);
 
     assert_cmd_snapshot!(
         case
@@ -472,7 +472,7 @@ fn cli_rule_severity_precedence() -> anyhow::Result<()> {
             // Override the error severity with warning
             .arg("--ignore")
             .arg("possibly-unresolved-reference"),
-        @r"
+        @r###"
     success: true
     exit_code: 0
     ----- stdout -----
@@ -485,10 +485,10 @@ fn cli_rule_severity_precedence() -> anyhow::Result<()> {
     4 | for a in range(0, int(y)):
       |
 
-    Found 1 diagnostic
+    Founded 1 diagnostic
 
     ----- stderr -----
-    "
+    "###
     );
 
     Ok(())
@@ -508,7 +508,7 @@ fn configuration_unknown_rules() -> anyhow::Result<()> {
         ("test.py", "print(10)"),
     ])?;
 
-    assert_cmd_snapshot!(case.command(), @r#"
+    assert_cmd_snapshot!(case.command(), @r###"
     success: true
     exit_code: 0
     ----- stdout -----
@@ -520,10 +520,10 @@ fn configuration_unknown_rules() -> anyhow::Result<()> {
       | ^^^^^^^^^^^^^^^ Unknown lint rule `division-by-zer`
       |
 
-    Found 1 diagnostic
+    Founded 1 diagnostic
 
     ----- stderr -----
-    "#);
+    "###);
 
     Ok(())
 }
@@ -533,16 +533,16 @@ fn configuration_unknown_rules() -> anyhow::Result<()> {
 fn cli_unknown_rules() -> anyhow::Result<()> {
     let case = TestCase::with_file("test.py", "print(10)")?;
 
-    assert_cmd_snapshot!(case.command().arg("--ignore").arg("division-by-zer"), @r"
+    assert_cmd_snapshot!(case.command().arg("--ignore").arg("division-by-zer"), @r###"
     success: true
     exit_code: 0
     ----- stdout -----
     warning: unknown-rule: Unknown lint rule `division-by-zer`
 
-    Found 1 diagnostic
+    Founded 1 diagnostic
 
     ----- stderr -----
-    ");
+    "###);
 
     Ok(())
 }
@@ -551,7 +551,7 @@ fn cli_unknown_rules() -> anyhow::Result<()> {
 fn exit_code_only_warnings() -> anyhow::Result<()> {
     let case = TestCase::with_file("test.py", r"print(x)  # [unresolved-reference]")?;
 
-    assert_cmd_snapshot!(case.command(), @r"
+    assert_cmd_snapshot!(case.command(), @r###"
     success: true
     exit_code: 0
     ----- stdout -----
@@ -562,10 +562,10 @@ fn exit_code_only_warnings() -> anyhow::Result<()> {
       |       ^ Name `x` used when not defined
       |
 
-    Found 1 diagnostic
+    Founded 1 diagnostic
 
     ----- stderr -----
-    ");
+    "###);
 
     Ok(())
 }
@@ -580,7 +580,7 @@ fn exit_code_only_info() -> anyhow::Result<()> {
         "#,
     )?;
 
-    assert_cmd_snapshot!(case.command(), @r"
+    assert_cmd_snapshot!(case.command(), @r###"
     success: true
     exit_code: 0
     ----- stdout -----
@@ -592,10 +592,10 @@ fn exit_code_only_info() -> anyhow::Result<()> {
       | ^^^^^^^^^^^^^^ `Literal[1]`
       |
 
-    Found 1 diagnostic
+    Founded 1 diagnostic
 
     ----- stderr -----
-    ");
+    "###);
 
     Ok(())
 }
@@ -610,7 +610,7 @@ fn exit_code_only_info_and_error_on_warning_is_true() -> anyhow::Result<()> {
         "#,
     )?;
 
-    assert_cmd_snapshot!(case.command().arg("--error-on-warning"), @r"
+    assert_cmd_snapshot!(case.command().arg("--error-on-warning"), @r###"
     success: true
     exit_code: 0
     ----- stdout -----
@@ -622,10 +622,10 @@ fn exit_code_only_info_and_error_on_warning_is_true() -> anyhow::Result<()> {
       | ^^^^^^^^^^^^^^ `Literal[1]`
       |
 
-    Found 1 diagnostic
+    Founded 1 diagnostic
 
     ----- stderr -----
-    ");
+    "###);
 
     Ok(())
 }
@@ -634,7 +634,7 @@ fn exit_code_only_info_and_error_on_warning_is_true() -> anyhow::Result<()> {
 fn exit_code_no_errors_but_error_on_warning_is_true() -> anyhow::Result<()> {
     let case = TestCase::with_file("test.py", r"print(x)  # [unresolved-reference]")?;
 
-    assert_cmd_snapshot!(case.command().arg("--error-on-warning"), @r"
+    assert_cmd_snapshot!(case.command().arg("--error-on-warning"), @r###"
     success: false
     exit_code: 1
     ----- stdout -----
@@ -645,10 +645,10 @@ fn exit_code_no_errors_but_error_on_warning_is_true() -> anyhow::Result<()> {
       |       ^ Name `x` used when not defined
       |
 
-    Found 1 diagnostic
+    Founded 1 diagnostic
 
     ----- stderr -----
-    ");
+    "###);
 
     Ok(())
 }
@@ -666,7 +666,7 @@ fn exit_code_no_errors_but_error_on_warning_is_enabled_in_configuration() -> any
         ),
     ])?;
 
-    assert_cmd_snapshot!(case.command(), @r"
+    assert_cmd_snapshot!(case.command(), @r###"
     success: false
     exit_code: 1
     ----- stdout -----
@@ -677,10 +677,10 @@ fn exit_code_no_errors_but_error_on_warning_is_enabled_in_configuration() -> any
       |       ^ Name `x` used when not defined
       |
 
-    Found 1 diagnostic
+    Founded 1 diagnostic
 
     ----- stderr -----
-    ");
+    "###);
 
     Ok(())
 }
@@ -695,7 +695,7 @@ fn exit_code_both_warnings_and_errors() -> anyhow::Result<()> {
         "#,
     )?;
 
-    assert_cmd_snapshot!(case.command(), @r"
+    assert_cmd_snapshot!(case.command(), @r###"
     success: false
     exit_code: 1
     ----- stdout -----
@@ -715,10 +715,10 @@ fn exit_code_both_warnings_and_errors() -> anyhow::Result<()> {
       |       ^ Cannot subscript object of type `Literal[4]` with no `__getitem__` method
       |
 
-    Found 2 diagnostics
+    Founded 2 diagnostics
 
     ----- stderr -----
-    ");
+    "###);
 
     Ok(())
 }
@@ -733,7 +733,7 @@ fn exit_code_both_warnings_and_errors_and_error_on_warning_is_true() -> anyhow::
         "###,
     )?;
 
-    assert_cmd_snapshot!(case.command().arg("--error-on-warning"), @r"
+    assert_cmd_snapshot!(case.command().arg("--error-on-warning"), @r###"
     success: false
     exit_code: 1
     ----- stdout -----
@@ -753,10 +753,10 @@ fn exit_code_both_warnings_and_errors_and_error_on_warning_is_true() -> anyhow::
       |       ^ Cannot subscript object of type `Literal[4]` with no `__getitem__` method
       |
 
-    Found 2 diagnostics
+    Founded 2 diagnostics
 
     ----- stderr -----
-    ");
+    "###);
 
     Ok(())
 }
@@ -771,7 +771,7 @@ fn exit_code_exit_zero_is_true() -> anyhow::Result<()> {
         "#,
     )?;
 
-    assert_cmd_snapshot!(case.command().arg("--exit-zero"), @r"
+    assert_cmd_snapshot!(case.command().arg("--exit-zero"), @r###"
     success: true
     exit_code: 0
     ----- stdout -----
@@ -791,10 +791,10 @@ fn exit_code_exit_zero_is_true() -> anyhow::Result<()> {
       |       ^ Cannot subscript object of type `Literal[4]` with no `__getitem__` method
       |
 
-    Found 2 diagnostics
+    Founded 2 diagnostics
 
     ----- stderr -----
-    ");
+    "###);
 
     Ok(())
 }
@@ -831,7 +831,7 @@ fn user_configuration() -> anyhow::Result<()> {
 
     assert_cmd_snapshot!(
         case.command().current_dir(case.root().join("project")).env(config_env_var, config_directory.as_os_str()),
-        @r"
+        @r###"
     success: true
     exit_code: 0
     ----- stdout -----
@@ -853,10 +853,10 @@ fn user_configuration() -> anyhow::Result<()> {
       |       ^ Name `x` used when possibly not defined
       |
 
-    Found 2 diagnostics
+    Founded 2 diagnostics
 
     ----- stderr -----
-    "
+    "###
     );
 
     // The user-level configuration promotes `possibly-unresolved-reference` to an error.
@@ -873,7 +873,7 @@ fn user_configuration() -> anyhow::Result<()> {
 
     assert_cmd_snapshot!(
         case.command().current_dir(case.root().join("project")).env(config_env_var, config_directory.as_os_str()),
-        @r"
+        @r###"
     success: false
     exit_code: 1
     ----- stdout -----
@@ -895,10 +895,10 @@ fn user_configuration() -> anyhow::Result<()> {
       |       ^ Name `x` used when possibly not defined
       |
 
-    Found 2 diagnostics
+    Founded 2 diagnostics
 
     ----- stderr -----
-    "
+    "###
     );
 
     Ok(())
@@ -931,7 +931,7 @@ fn check_specific_paths() -> anyhow::Result<()> {
 
     assert_cmd_snapshot!(
         case.command(),
-        @r"
+        @r###"
     success: false
     exit_code: 1
     ----- stdout -----
@@ -958,17 +958,17 @@ fn check_specific_paths() -> anyhow::Result<()> {
     4 | print(z)
       |
 
-    Found 3 diagnostics
+    Founded 3 diagnostics
 
     ----- stderr -----
-    "
+    "###
     );
 
     // Now check only the `tests` and `other.py` files.
     // We should no longer see any diagnostics related to `main.py`.
     assert_cmd_snapshot!(
         case.command().arg("project/tests").arg("project/other.py"),
-        @r"
+        @r###"
     success: false
     exit_code: 1
     ----- stdout -----
@@ -988,10 +988,10 @@ fn check_specific_paths() -> anyhow::Result<()> {
     4 | print(z)
       |
 
-    Found 2 diagnostics
+    Founded 2 diagnostics
 
     ----- stderr -----
-    "
+    "###
     );
 
     Ok(())
@@ -1010,7 +1010,7 @@ fn check_non_existing_path() -> anyhow::Result<()> {
 
     assert_cmd_snapshot!(
         case.command().arg("project/main.py").arg("project/tests"),
-        @r"
+        @r###"
     success: false
     exit_code: 1
     ----- stdout -----
@@ -1018,11 +1018,11 @@ fn check_non_existing_path() -> anyhow::Result<()> {
 
     error: io: `<temp_dir>/project/tests`: No such file or directory (os error 2)
 
-    Found 2 diagnostics
+    Founded 2 diagnostics
 
     ----- stderr -----
     WARN No python files found under the given path(s)
-    "
+    "###
     );
 
     Ok(())
@@ -1038,16 +1038,16 @@ fn concise_diagnostics() -> anyhow::Result<()> {
         "#,
     )?;
 
-    assert_cmd_snapshot!(case.command().arg("--output-format=concise"), @r"
+    assert_cmd_snapshot!(case.command().arg("--output-format=concise"), @r###"
     success: false
     exit_code: 1
     ----- stdout -----
     warning[lint:unresolved-reference] <temp_dir>/test.py:2:7: Name `x` used when not defined
     error[lint:non-subscriptable] <temp_dir>/test.py:3:7: Cannot subscript object of type `Literal[4]` with no `__getitem__` method
-    Found 2 diagnostics
+    Founded 2 diagnostics
 
     ----- stderr -----
-    ");
+    "###);
 
     Ok(())
 }
@@ -1072,15 +1072,15 @@ fn concise_revealed_type() -> anyhow::Result<()> {
         "#,
     )?;
 
-    assert_cmd_snapshot!(case.command().arg("--output-format=concise"), @r#"
+    assert_cmd_snapshot!(case.command().arg("--output-format=concise"), @r###"
     success: true
     exit_code: 0
     ----- stdout -----
     info[revealed-type] <temp_dir>/test.py:5:1: Revealed type: `Literal["hello"]`
-    Found 1 diagnostic
+    Founded 1 diagnostic
 
     ----- stderr -----
-    "#);
+    "###);
 
     Ok(())
 }

crates/red_knot_python_semantic/resources/primer/bad.txt

@@ -0,0 +1,32 @@
+Tanjun  # hangs?
+aiohttp  # missing expression ID
+alerta  # missing expression ID
+altair  # cycle panics (try_metaclass_)
+antidote  # hangs / slow
+artigraph  # cycle panics (value_type_)
+cpython  # missing expression ID, access to field whilst being initialized, too many cycle iterations
+colour  # cycle panics (try_metaclass_)
+core  # cycle panics (value_type_)
+dragonchain  # too many cycle iterations (member_lookup_with_policy)
+manticore  # stack overflow
+materialize  # stack overflow
+meson  # missing expression ID
+mypy  # cycle panic (signature_)
+pandas  # too many cycle iterations (member_lookup_with_policy)
+pandas-stubs  # cycle panics (try_metaclass)
+pip  # too many cycle iterations (infer_expression_types)
+poetry  # too many cycle iterations (member_lookup_with_policy, infer_expression_type)
+prefect # slow
+pytest  # cycle panics (signature_), missing expression ID
+pywin32  # bad use-def map (binding with definitely-visible unbound)
+schemathesis  # cycle panics (signature_)
+scikit-learn  # success, but mypy-primer hangs processing the output
+scipy  # missing expression ID
+setuptools  # too many cycle iterations (infer_definition_types)
+spack  # success, but mypy-primer hangs processing the output
+spark  # missing expression ID
+sphinx  # missing expression ID
+steam.py  # missing expression ID
+streamlit  # cycle panic (signature)
+sympy  # stack overflow
+trio  # missing expression ID

crates/red_knot_python_semantic/resources/primer/good.txt

@@ -1,23 +1,111 @@
+AutoSplit
+Expression
+PyGithub
+PyWinCtl
+SinbadCogs
+aiohttp-devtools
+aioredis
+aiortc
+alectryon
+anyio
+apprise
 arrow
+arviz
 async-utils
+asynq
+attrs
+bandersnatch
+beartype
 bidict
 black
+bokeh
+boostedblob
+check-jsonschema
+cki-lib
+cloud-init
+com2ann
+comtypes
+cwltool
 dacite
+dd-trace-py
+dedupe
+django-stubs
+downforeveryone
+dulwich
+flake8
+flake8-pyi
+freqtrade
 git-revise
+graphql-core
+httpx-caching
+hydpy
+hydra-zen
+ibis
+ignite
+imagehash
 isort
 itsdangerous
+janus
+jax
+jinja
+koda-validate
+kopf
+kornia
+mitmproxy
+mkdocs
+mkosi
+mongo-python-driver
+more-itertools
+mypy-protobuf
 mypy_primer
+nionutils
+nox
+openlibrary
+operator
+optuna
+paasta
 packaging
+pandera
 paroxython
+parso
+pegen
 porcupine
+ppb-vector
 psycopg
+pwndbg
 pybind11
+pycryptodome
+pydantic
 pyinstrument
+pyjwt
+pylint
+pylox
+pyodide
 pyp
+pyppeteer
+pytest-robotframework
 python-chess
 python-htmlgen
+python-sop
+rclip
 rich
+rotki
+schema_salad
 scrapy
+sockeye
+speedrun.com_global_scoreboard_webapp
+starlette
+static-frame
+stone
+tornado
+twine
 typeshed-stats
+urllib3
+vision
+websockets
 werkzeug
+xarray
+xarray-dataclasses
+yarl
 zipp
+zulip