Use FoldHash

Co-authored-by: Dhruv Manilawala <dhruvmanila@gmail.com>
Co-authored-by: Micha Reiser <micha@reiser.io>

.github/renovate.json5

@@ -14,12 +14,26 @@
     rangeStrategy: "update-lockfile",
   },
   pep621: {
+    // The default for this package manager is to only search for `pyproject.toml` files
+    // found at the repository root: https://docs.renovatebot.com/modules/manager/pep621/#file-matching
     fileMatch: ["^(python|scripts)/.*pyproject\\.toml$"],
   },
   pip_requirements: {
-    fileMatch: ["^docs/requirements.*\\.txt$"],
+    // The default for this package manager is to run on all requirements.txt files:
+    // https://docs.renovatebot.com/modules/manager/pip_requirements/#file-matching
+    // `fileMatch` doesn't work for excluding files; to exclude `requirements.txt` files
+    // outside the `doc/` directory, we instead have to use `ignorePaths`. Unlike `fileMatch`,
+    // which takes a regex string, `ignorePaths` takes a glob string, so we have to use
+    // a "negative glob pattern".
+    // See:
+    // - https://docs.renovatebot.com/modules/manager/#ignoring-files-that-match-the-default-filematch
+    // - https://docs.renovatebot.com/configuration-options/#ignorepaths
+    // - https://docs.renovatebot.com/string-pattern-matching/#negative-matching
+    ignorePaths: ["!docs/requirements*.txt"]
   },
   npm: {
+    // The default for this package manager is to only search for `package.json` files
+    // found at the repository root: https://docs.renovatebot.com/modules/manager/npm/#file-matching
     fileMatch: ["^playground/.*package\\.json$"],
   },
   "pre-commit": {

.github/workflows/ci.yaml

@@ -142,6 +142,13 @@ jobs:
 
       # Check for broken links in the documentation.
       - run: cargo doc --all --no-deps
+        env:
+          RUSTDOCFLAGS: "-D warnings"
+      # Use --document-private-items so that all our doc comments are kept in
+      # sync, not just public items. Eventually we should do this for all
+      # crates; for now add crates here as they are warning-clean to prevent
+      # regression.
+      - run: cargo doc --no-deps -p red_knot_python_semantic -p red_knot -p ruff_db --document-private-items
         env:
           # Setting RUSTDOCFLAGS because `cargo doc --check` isn't yet implemented (https://github.com/rust-lang/cargo/issues/10025).
           RUSTDOCFLAGS: "-D warnings"

.markdownlint.yaml

@@ -14,6 +14,9 @@ MD041: false
 # MD013/line-length
 MD013: false
 
+# MD014/commands-show-output
+MD014: false
+
 # MD024/no-duplicate-heading
 MD024:
   # Allow when nested under different parents e.g. CHANGELOG.md

.pre-commit-config.yaml

@@ -57,7 +57,7 @@ repos:
         pass_filenames: false # This makes it a lot faster
 
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.5.6
+    rev: v0.5.7
     hooks:
       - id: ruff-format
       - id: ruff

BREAKING_CHANGES.md

@@ -1,5 +1,43 @@
 # Breaking Changes
 
+## 0.6.0
+
+- Detect imports in `src` layouts by default for `isort` rules ([#12848](https://github.com/astral-sh/ruff/pull/12848))
+
+- The pytest rules `PT001` and `PT023` now default to omitting the decorator parentheses when there are no arguments ([#12838](https://github.com/astral-sh/ruff/pull/12838)).
+
+- Lint and format Jupyter Notebook by default ([#12878](https://github.com/astral-sh/ruff/pull/12878)).
+
+    You can disable specific rules for notebooks using [`per-file-ignores`](https://docs.astral.sh/ruff/settings/#lint_per-file-ignores):
+
+    ```toml
+    [tool.ruff.lint.per-file-ignores]
+    "*.ipynb" = ["E501"] # disable line-too-long in notebooks
+    ```
+
+    If you'd prefer to either only lint or only format Jupyter Notebook files, you can use the
+    section-specific `exclude` option to do so. For example, the following would only lint Jupyter
+    Notebook files and not format them:
+
+    ```toml
+    [tool.ruff.format]
+    exclude = ["*.ipynb"]
+    ```
+
+    And, conversely, the following would only format Jupyter Notebook files and not lint them:
+
+    ```toml
+    [tool.ruff.lint]
+    exclude = ["*.ipynb"]
+    ```
+
+    You can completely disable Jupyter Notebook support by updating the [`extend-exclude`](https://docs.astral.sh/ruff/settings/#extend-exclude) setting:
+
+    ```toml
+    [tool.ruff]
+    extend-exclude = ["*.ipynb"]
+    ```
+
 ## 0.5.0
 
 - Follow the XDG specification to discover user-level configurations on macOS (same as on other Unix platforms)

CHANGELOG.md

@@ -1,5 +1,120 @@
 # Changelog
 
+## 0.6.1
+
+This is a hotfix release to address an issue with `ruff-pre-commit`. In v0.6,
+Ruff changed its behavior to lint and format Jupyter notebooks by default;
+however, due to an oversight, these files were still excluded by default if
+Ruff was run via pre-commit, leading to inconsistent behavior.
+This has [now been fixed](https://github.com/astral-sh/ruff-pre-commit/pull/96).
+
+### Preview features
+
+- \[`fastapi`\] Implement `fast-api-unused-path-parameter` (`FAST003`) ([#12638](https://github.com/astral-sh/ruff/pull/12638))
+
+### Rule changes
+
+- \[`pylint`\] Rename `too-many-positional` to `too-many-positional-arguments` (`R0917`) ([#12905](https://github.com/astral-sh/ruff/pull/12905))
+
+### Server
+
+- Fix crash when applying "fix-all" code-action to notebook cells ([#12929](https://github.com/astral-sh/ruff/pull/12929))
+
+### Other changes
+
+- \[`flake8-naming`\]: Respect import conventions (`N817`) ([#12922](https://github.com/astral-sh/ruff/pull/12922))
+
+## 0.6.0
+
+Check out the [blog post](https://astral.sh/blog/ruff-v0.6.0) for a migration guide and overview of the changes!
+
+### Breaking changes
+
+See also, the "Remapped rules" section which may result in disabled rules.
+
+- Lint and format Jupyter Notebook by default ([#12878](https://github.com/astral-sh/ruff/pull/12878)).
+- Detect imports in `src` layouts by default for `isort` rules ([#12848](https://github.com/astral-sh/ruff/pull/12848))
+- The pytest rules `PT001` and `PT023` now default to omitting the decorator parentheses when there are no arguments ([#12838](https://github.com/astral-sh/ruff/pull/12838)).
+
+### Deprecations
+
+The following rules are now deprecated:
+
+- [`pytest-missing-fixture-name-underscore`](https://docs.astral.sh/ruff/rules/pytest-missing-fixture-name-underscore/) (`PT004`)
+- [`pytest-incorrect-fixture-name-underscore`](https://docs.astral.sh/ruff/rules/pytest-incorrect-fixture-name-underscore/) (`PT005`)
+- [`unpacked-list-comprehension`](https://docs.astral.sh/ruff/rules/unpacked-list-comprehension/) (`UP027`)
+
+### Remapped rules
+
+The following rules have been remapped to new rule codes:
+
+- [`unnecessary-dict-comprehension-for-iterable`](https://docs.astral.sh/ruff/rules/unnecessary-dict-comprehension-for-iterable/): `RUF025` to `C420`
+
+### Stabilization
+
+The following rules have been stabilized and are no longer in preview:
+
+- [`singledispatch-method`](https://docs.astral.sh/ruff/rules/singledispatch-method/) (`PLE1519`)
+- [`singledispatchmethod-function`](https://docs.astral.sh/ruff/rules/singledispatchmethod-function/) (`PLE1520`)
+- [`bad-staticmethod-argument`](https://docs.astral.sh/ruff/rules/bad-staticmethod-argument/) (`PLW0211`)
+- [`if-stmt-min-max`](https://docs.astral.sh/ruff/rules/if-stmt-min-max/) (`PLR1730`)
+- [`invalid-bytes-return-type`](https://docs.astral.sh/ruff/rules/invalid-bytes-return-type/) (`PLE0308`)
+- [`invalid-hash-return-type`](https://docs.astral.sh/ruff/rules/invalid-hash-return-type/) (`PLE0309`)
+- [`invalid-index-return-type`](https://docs.astral.sh/ruff/rules/invalid-index-return-type/) (`PLE0305`)
+- [`invalid-length-return-type`](https://docs.astral.sh/ruff/rules/invalid-length-return-type/) (`E303`)
+- [`self-or-cls-assignment`](https://docs.astral.sh/ruff/rules/self-or-cls-assignment/) (`PLW0642`)
+- [`byte-string-usage`](https://docs.astral.sh/ruff/rules/byte-string-usage/) (`PYI057`)
+- [`duplicate-literal-member`](https://docs.astral.sh/ruff/rules/duplicate-literal-member/) (`PYI062`)
+- [`redirected-noqa`](https://docs.astral.sh/ruff/rules/redirected-noqa/) (`RUF101`)
+
+The following behaviors have been stabilized:
+
+- [`cancel-scope-no-checkpoint`](https://docs.astral.sh/ruff/rules/cancel-scope-no-checkpoint/) (`ASYNC100`): Support `asyncio` and `anyio` context mangers.
+- [`async-function-with-timeout`](https://docs.astral.sh/ruff/rules/async-function-with-timeout/) (`ASYNC109`): Support `asyncio` and `anyio` context mangers.
+- [`async-busy-wait`](https://docs.astral.sh/ruff/rules/async-busy-wait/) (`ASYNC110`): Support `asyncio` and `anyio` context mangers.
+- [`async-zero-sleep`](https://docs.astral.sh/ruff/rules/async-zero-sleep/) (`ASYNC115`): Support `anyio` context mangers.
+- [`long-sleep-not-forever`](https://docs.astral.sh/ruff/rules/long-sleep-not-forever/) (`ASYNC116`): Support `anyio` context mangers.
+
+The following fixes have been stabilized:
+
+- [`superfluous-else-return`](https://docs.astral.sh/ruff/rules/superfluous-else-return/) (`RET505`)
+- [`superfluous-else-raise`](https://docs.astral.sh/ruff/rules/superfluous-else-raise/) (`RET506`)
+- [`superfluous-else-continue`](https://docs.astral.sh/ruff/rules/superfluous-else-continue/) (`RET507`)
+- [`superfluous-else-break`](https://docs.astral.sh/ruff/rules/superfluous-else-break/) (`RET508`)
+
+### Preview features
+
+- \[`flake8-simplify`\] Further simplify to binary in preview for (`SIM108`) ([#12796](https://github.com/astral-sh/ruff/pull/12796))
+- \[`pyupgrade`\] Show violations without auto-fix (`UP031`) ([#11229](https://github.com/astral-sh/ruff/pull/11229))
+
+### Rule changes
+
+- \[`flake8-import-conventions`\] Add `xml.etree.ElementTree` to default conventions ([#12455](https://github.com/astral-sh/ruff/pull/12455))
+- \[`flake8-pytest-style`\] Add a space after comma in CSV output (`PT006`) ([#12853](https://github.com/astral-sh/ruff/pull/12853))
+
+### Server
+
+- Show a message for incorrect settings ([#12781](https://github.com/astral-sh/ruff/pull/12781))
+
+### Bug fixes
+
+- \[`flake8-async`\] Do not lint yield in context manager (`ASYNC100`) ([#12896](https://github.com/astral-sh/ruff/pull/12896))
+- \[`flake8-comprehensions`\] Do not lint `async for` comprehensions (`C419`) ([#12895](https://github.com/astral-sh/ruff/pull/12895))
+- \[`flake8-return`\] Only add return `None` at end of a function (`RET503`) ([#11074](https://github.com/astral-sh/ruff/pull/11074))
+- \[`flake8-type-checking`\] Avoid treating `dataclasses.KW_ONLY` as typing-only (`TCH003`) ([#12863](https://github.com/astral-sh/ruff/pull/12863))
+- \[`pep8-naming`\] Treat `type(Protocol)` et al as metaclass base (`N805`) ([#12770](https://github.com/astral-sh/ruff/pull/12770))
+- \[`pydoclint`\] Don't enforce returns and yields in abstract methods (`DOC201`, `DOC202`) ([#12771](https://github.com/astral-sh/ruff/pull/12771))
+- \[`ruff`\] Skip tuples with slice expressions in (`RUF031`) ([#12768](https://github.com/astral-sh/ruff/pull/12768))
+- \[`ruff`\] Ignore unparenthesized tuples in subscripts when the subscript is a type annotation or type alias (`RUF031`) ([#12762](https://github.com/astral-sh/ruff/pull/12762))
+- \[`ruff`\] Ignore template strings passed to logging and `builtins._()` calls (`RUF027`) ([#12889](https://github.com/astral-sh/ruff/pull/12889))
+- \[`ruff`\] Do not remove parens for tuples with starred expressions in Python \<=3.10 (`RUF031`) ([#12784](https://github.com/astral-sh/ruff/pull/12784))
+- Evaluate default parameter values for a function in that function's enclosing scope ([#12852](https://github.com/astral-sh/ruff/pull/12852))
+
+### Other changes
+
+- Respect VS Code cell metadata when detecting the language of Jupyter Notebook cells ([#12864](https://github.com/astral-sh/ruff/pull/12864))
+- Respect `kernelspec` notebook metadata when detecting the preferred language for a Jupyter Notebook ([#12875](https://github.com/astral-sh/ruff/pull/12875))
+
 ## 0.5.7
 
 ### Preview features

CONTRIBUTING.md

@@ -361,7 +361,7 @@ even patch releases may contain [non-backwards-compatible changes](https://semve
         downstream jobs manually if needed.
 1. Verify the GitHub release:
     1. The Changelog should match the content of `CHANGELOG.md`
-    1. Append the contributors from the `bump.sh` script
+    1. Append the contributors from the `scripts/release.sh` script
 1. If needed, [update the schemastore](https://github.com/astral-sh/ruff/blob/main/scripts/update_schemastore.py).
     1. One can determine if an update is needed when
         `git diff old-version-tag new-version-tag -- ruff.schema.json` returns a non-empty diff.
@@ -911,9 +911,5 @@ There are three ways in which an import can be categorized as "first-party":
     the `src` setting and, for each directory, check for the existence of a subdirectory `foo` or a
     file `foo.py`.
 
-By default, `src` is set to the project root. In the above example, we'd want to set
-`src = ["./src"]` to ensure that we locate `./my_project/src/foo` and thus categorize `import foo`
-as first-party in `baz.py`. In practice, for this limited example, setting `src = ["./src"]` is
-unnecessary, as all imports within `./my_project/src/foo` would be categorized as first-party via
-the same-package heuristic; but if your project contains multiple packages, you'll want to set `src`
-explicitly.
+By default, `src` is set to the project root, along with `"src"` subdirectory in the project root.
+This ensures that Ruff supports both flat and "src" layouts out of the box.

Cargo.lock

@@ -95,9 +95,9 @@ dependencies = [
 
 [[package]]
 name = "anstyle"
-version = "1.0.6"
+version = "1.0.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8901269c6307e8d93993578286ac0edf7f195079ffff5ebdeea6a59ffb7e36bc"
+checksum = "1bec1de6f59aedf83baf9ff929c98f2ad654b97c9510f4e70cf6f661d49fd5b1"
 
 [[package]]
 name = "anstyle-parse"
@@ -288,7 +288,7 @@ dependencies = [
  "android-tzdata",
  "iana-time-zone",
  "num-traits",
- "windows-targets 0.52.5",
+ "windows-targets 0.52.6",
 ]
 
 [[package]]
@@ -320,9 +320,9 @@ dependencies = [
 
 [[package]]
 name = "clap"
-version = "4.5.13"
+version = "4.5.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0fbb260a053428790f3de475e304ff84cdbc4face759ea7a3e64c1edd938a7fc"
+checksum = "11d8838454fda655dafd3accb2b6e2bea645b9e4078abe84a22ceb947235c5cc"
 dependencies = [
  "clap_builder",
  "clap_derive",
@@ -330,9 +330,9 @@ dependencies = [
 
 [[package]]
 name = "clap_builder"
-version = "4.5.13"
+version = "4.5.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "64b17d7ea74e9f833c7dbf2cbe4fb12ff26783eda4782a8975b72f895c9b4d99"
+checksum = "216aec2b177652e3846684cbfe25c9964d18ec45234f0f5da5157b207ed1aab6"
 dependencies = [
  "anstream",
  "anstyle",
@@ -820,14 +820,14 @@ dependencies = [
 
 [[package]]
 name = "filetime"
-version = "0.2.23"
+version = "0.2.24"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1ee447700ac8aa0b2f2bd7bc4462ad686ba06baa6727ac149a2d6277f0d240fd"
+checksum = "bf401df4a4e3872c4fe8151134cf483738e74b67fc934d6532c882b3d24a4550"
 dependencies = [
  "cfg-if",
  "libc",
- "redox_syscall",
- "windows-sys 0.52.0",
+ "libredox",
+ "windows-sys 0.59.0",
 ]
 
 [[package]]
@@ -846,6 +846,12 @@ version = "1.0.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
 
+[[package]]
+name = "foldhash"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4deb59dd6330afa472c000b86c0c9ada26274836eb59563506c3e34e4bb9a819"
+
 [[package]]
 name = "form_urlencoded"
 version = "1.2.1"
@@ -1143,9 +1149,9 @@ dependencies = [
 
 [[package]]
 name = "is-macro"
-version = "0.3.5"
+version = "0.3.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "59a85abdc13717906baccb5a1e435556ce0df215f242892f721dff62bf25288f"
+checksum = "2069faacbe981460232f880d26bf3c7634e322d49053aa48c27e3ae642f728f1"
 dependencies = [
  "Inflector",
  "proc-macro2",
@@ -1297,6 +1303,7 @@ checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d"
 dependencies = [
  "bitflags 2.6.0",
  "libc",
+ "redox_syscall 0.5.3",
 ]
 
 [[package]]
@@ -1564,7 +1571,7 @@ checksum = "4c42a9226546d68acdd9c0a280d17ce19bfe27a46bf68784e4066115788d008e"
 dependencies = [
  "cfg-if",
  "libc",
- "redox_syscall",
+ "redox_syscall 0.4.1",
  "smallvec",
  "windows-targets 0.48.5",
 ]
@@ -1899,6 +1906,7 @@ dependencies = [
  "ruff_python_ast",
  "ruff_python_parser",
  "ruff_python_stdlib",
+ "ruff_source_file",
  "ruff_text_size",
  "rustc-hash 2.0.0",
  "salsa",
@@ -1914,10 +1922,12 @@ version = "0.0.0"
 dependencies = [
  "anyhow",
  "crossbeam",
+ "foldhash",
  "jod-thread",
  "libc",
  "lsp-server",
  "lsp-types",
+ "red_knot_python_semantic",
  "red_knot_workspace",
  "ruff_db",
  "ruff_linter",
@@ -1925,7 +1935,6 @@ dependencies = [
  "ruff_python_ast",
  "ruff_source_file",
  "ruff_text_size",
- "rustc-hash 2.0.0",
  "serde",
  "serde_json",
  "shellexpand",
@@ -1941,6 +1950,7 @@ dependencies = [
  "console_log",
  "js-sys",
  "log",
+ "red_knot_python_semantic",
  "red_knot_workspace",
  "ruff_db",
  "ruff_notebook",
@@ -1954,12 +1964,13 @@ version = "0.0.0"
 dependencies = [
  "anyhow",
  "crossbeam",
+ "foldhash",
  "notify",
  "red_knot_python_semantic",
  "ruff_cache",
  "ruff_db",
  "ruff_python_ast",
- "rustc-hash 2.0.0",
+ "ruff_text_size",
  "salsa",
  "thiserror",
  "tracing",
@@ -1974,6 +1985,15 @@ dependencies = [
  "bitflags 1.3.2",
 ]
 
+[[package]]
+name = "redox_syscall"
+version = "0.5.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2a908a6e00f1fdd0dfd9c0eb08ce85126f6d8bbda50017e74bc4a4b7d4a926a4"
+dependencies = [
+ "bitflags 2.6.0",
+]
+
 [[package]]
 name = "redox_users"
 version = "0.4.5"
@@ -2046,7 +2066,7 @@ dependencies = [
 
 [[package]]
 name = "ruff"
-version = "0.5.7"
+version = "0.6.1"
 dependencies = [
  "anyhow",
  "argfile",
@@ -2059,6 +2079,7 @@ dependencies = [
  "clearscreen",
  "colored",
  "filetime",
+ "foldhash",
  "ignore",
  "insta",
  "insta-cmd",
@@ -2081,7 +2102,6 @@ dependencies = [
  "ruff_source_file",
  "ruff_text_size",
  "ruff_workspace",
- "rustc-hash 2.0.0",
  "serde",
  "serde_json",
  "shellexpand",
@@ -2104,6 +2124,7 @@ dependencies = [
  "criterion",
  "mimalloc",
  "once_cell",
+ "red_knot_python_semantic",
  "red_knot_workspace",
  "ruff_db",
  "ruff_linter",
@@ -2139,6 +2160,7 @@ dependencies = [
  "countme",
  "dashmap 6.0.1",
  "filetime",
+ "foldhash",
  "ignore",
  "insta",
  "matchit",
@@ -2150,10 +2172,12 @@ dependencies = [
  "ruff_python_trivia",
  "ruff_source_file",
  "ruff_text_size",
- "rustc-hash 2.0.0",
  "salsa",
  "tempfile",
+ "thiserror",
  "tracing",
+ "tracing-subscriber",
+ "tracing-tree",
  "web-time",
  "zip",
 ]
@@ -2213,10 +2237,10 @@ name = "ruff_formatter"
 version = "0.0.0"
 dependencies = [
  "drop_bomb",
+ "foldhash",
  "ruff_cache",
  "ruff_macros",
  "ruff_text_size",
- "rustc-hash 2.0.0",
  "schemars",
  "serde",
  "static_assertions",
@@ -2234,7 +2258,7 @@ dependencies = [
 
 [[package]]
 name = "ruff_linter"
-version = "0.5.7"
+version = "0.6.1"
 dependencies = [
  "aho-corasick",
  "annotate-snippets 0.9.2",
@@ -2244,6 +2268,7 @@ dependencies = [
  "clap",
  "colored",
  "fern",
+ "foldhash",
  "glob",
  "globset",
  "imperative",
@@ -2276,7 +2301,6 @@ dependencies = [
  "ruff_python_trivia",
  "ruff_source_file",
  "ruff_text_size",
- "rustc-hash 2.0.0",
  "schemars",
  "serde",
  "serde_json",
@@ -2331,6 +2355,7 @@ dependencies = [
  "aho-corasick",
  "bitflags 2.6.0",
  "compact_str",
+ "foldhash",
  "is-macro",
  "itertools 0.13.0",
  "once_cell",
@@ -2339,7 +2364,6 @@ dependencies = [
  "ruff_python_trivia",
  "ruff_source_file",
  "ruff_text_size",
- "rustc-hash 2.0.0",
  "schemars",
  "serde",
 ]
@@ -2374,6 +2398,7 @@ dependencies = [
  "anyhow",
  "clap",
  "countme",
+ "foldhash",
  "insta",
  "itertools 0.13.0",
  "memchr",
@@ -2387,7 +2412,6 @@ dependencies = [
  "ruff_python_trivia",
  "ruff_source_file",
  "ruff_text_size",
- "rustc-hash 2.0.0",
  "schemars",
  "serde",
  "serde_json",
@@ -2428,13 +2452,13 @@ dependencies = [
  "bitflags 2.6.0",
  "bstr",
  "compact_str",
+ "foldhash",
  "insta",
  "memchr",
  "ruff_python_ast",
  "ruff_python_trivia",
  "ruff_source_file",
  "ruff_text_size",
- "rustc-hash 2.0.0",
  "static_assertions",
  "unicode-ident",
  "unicode-normalization",
@@ -2457,6 +2481,7 @@ name = "ruff_python_semantic"
 version = "0.0.0"
 dependencies = [
  "bitflags 2.6.0",
+ "foldhash",
  "is-macro",
  "ruff_cache",
  "ruff_index",
@@ -2466,7 +2491,6 @@ dependencies = [
  "ruff_python_stdlib",
  "ruff_source_file",
  "ruff_text_size",
- "rustc-hash 2.0.0",
  "schemars",
  "serde",
 ]
@@ -2505,6 +2529,7 @@ version = "0.2.2"
 dependencies = [
  "anyhow",
  "crossbeam",
+ "foldhash",
  "ignore",
  "insta",
  "jod-thread",
@@ -2524,7 +2549,6 @@ dependencies = [
  "ruff_source_file",
  "ruff_text_size",
  "ruff_workspace",
- "rustc-hash 2.0.0",
  "serde",
  "serde_json",
  "shellexpand",
@@ -2554,7 +2578,7 @@ dependencies = [
 
 [[package]]
 name = "ruff_wasm"
-version = "0.5.7"
+version = "0.6.1"
 dependencies = [
  "console_error_panic_hook",
  "console_log",
@@ -2584,6 +2608,7 @@ dependencies = [
  "anyhow",
  "colored",
  "etcetera",
+ "foldhash",
  "glob",
  "globset",
  "ignore",
@@ -2603,7 +2628,6 @@ dependencies = [
  "ruff_python_formatter",
  "ruff_python_semantic",
  "ruff_source_file",
- "rustc-hash 2.0.0",
  "schemars",
  "serde",
  "shellexpand",
@@ -2781,9 +2805,9 @@ checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b"
 
 [[package]]
 name = "serde"
-version = "1.0.204"
+version = "1.0.206"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bc76f558e0cbb2a839d37354c575f1dc3fdc6546b5be373ba43d95f231bf7c12"
+checksum = "5b3e4cd94123dd520a128bcd11e34d9e9e423e7e3e50425cb1b4b1e3549d0284"
 dependencies = [
  "serde_derive",
 ]
@@ -2801,9 +2825,9 @@ dependencies = [
 
 [[package]]
 name = "serde_derive"
-version = "1.0.204"
+version = "1.0.206"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e0cd7e117be63d3c3678776753929474f3b04a43a080c744d6b0ae2a8c28e222"
+checksum = "fabfb6138d2383ea8208cf98ccf69cdfb1aff4088460681d84189aa259762f97"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -2823,9 +2847,9 @@ dependencies = [
 
 [[package]]
 name = "serde_json"
-version = "1.0.122"
+version = "1.0.124"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "784b6203951c57ff748476b126ccb5e8e2959a5c19e5c617ab1956be3dbc68da"
+checksum = "66ad62847a56b3dba58cc891acd13884b9c61138d330c0d7b6181713d4fce38d"
 dependencies = [
  "itoa",
  "memchr",
@@ -2855,9 +2879,9 @@ dependencies = [
 
 [[package]]
 name = "serde_test"
-version = "1.0.176"
+version = "1.0.177"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5a2f49ace1498612d14f7e0b8245519584db8299541dfe31a06374a828d620ab"
+checksum = "7f901ee573cab6b3060453d2d5f0bae4e6d628c23c0a962ff9b5f1d7c8d4f1ed"
 dependencies = [
  "serde",
 ]
@@ -2984,9 +3008,9 @@ checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc"
 
 [[package]]
 name = "syn"
-version = "2.0.72"
+version = "2.0.74"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dc4b9b9bf2add8093d3f2c0204471e951b2285580335de42f9d2534f3ae7a8af"
+checksum = "1fceb41e3d546d0bd83421d3409b1460cc7444cd389341a4c880fe7a042cb3d7"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -3006,15 +3030,15 @@ dependencies = [
 
 [[package]]
 name = "tempfile"
-version = "3.11.0"
+version = "3.12.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b8fcd239983515c23a32fb82099f97d0b11b8c72f654ed659363a95c3dad7a53"
+checksum = "04cbcdd0c794ebb0d4cf35e88edd2f7d2c4c3e9a5a6dab322839b321c6a87a64"
 dependencies = [
  "cfg-if",
  "fastrand",
  "once_cell",
  "rustix",
- "windows-sys 0.52.0",
+ "windows-sys 0.59.0",
 ]
 
 [[package]]
@@ -3390,9 +3414,9 @@ checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1"
 
 [[package]]
 name = "ureq"
-version = "2.10.0"
+version = "2.10.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "72139d247e5f97a3eff96229a7ae85ead5328a39efe76f8bf5a06313d505b6ea"
+checksum = "b74fc6b57825be3373f7054754755f03ac3a8f5d70015ccad699ba2029956f4a"
 dependencies = [
  "base64",
  "flate2",
@@ -3684,7 +3708,7 @@ version = "0.52.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9"
 dependencies = [
- "windows-targets 0.52.5",
+ "windows-targets 0.52.6",
 ]
 
 [[package]]
@@ -3702,7 +3726,16 @@ version = "0.52.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d"
 dependencies = [
- "windows-targets 0.52.5",
+ "windows-targets 0.52.6",
+]
+
+[[package]]
+name = "windows-sys"
+version = "0.59.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b"
+dependencies = [
+ "windows-targets 0.52.6",
 ]
 
 [[package]]
@@ -3722,18 +3755,18 @@ dependencies = [
 
 [[package]]
 name = "windows-targets"
-version = "0.52.5"
+version = "0.52.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6f0713a46559409d202e70e28227288446bf7841d3211583a4b53e3f6d96e7eb"
+checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973"
 dependencies = [
- "windows_aarch64_gnullvm 0.52.5",
- "windows_aarch64_msvc 0.52.5",
- "windows_i686_gnu 0.52.5",
+ "windows_aarch64_gnullvm 0.52.6",
+ "windows_aarch64_msvc 0.52.6",
+ "windows_i686_gnu 0.52.6",
  "windows_i686_gnullvm",
- "windows_i686_msvc 0.52.5",
- "windows_x86_64_gnu 0.52.5",
- "windows_x86_64_gnullvm 0.52.5",
- "windows_x86_64_msvc 0.52.5",
+ "windows_i686_msvc 0.52.6",
+ "windows_x86_64_gnu 0.52.6",
+ "windows_x86_64_gnullvm 0.52.6",
+ "windows_x86_64_msvc 0.52.6",
 ]
 
 [[package]]
@@ -3744,9 +3777,9 @@ checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8"
 
 [[package]]
 name = "windows_aarch64_gnullvm"
-version = "0.52.5"
+version = "0.52.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7088eed71e8b8dda258ecc8bac5fb1153c5cffaf2578fc8ff5d61e23578d3263"
+checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3"
 
 [[package]]
 name = "windows_aarch64_msvc"
@@ -3756,9 +3789,9 @@ checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc"
 
 [[package]]
 name = "windows_aarch64_msvc"
-version = "0.52.5"
+version = "0.52.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9985fd1504e250c615ca5f281c3f7a6da76213ebd5ccc9561496568a2752afb6"
+checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469"
 
 [[package]]
 name = "windows_i686_gnu"
@@ -3768,15 +3801,15 @@ checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e"
 
 [[package]]
 name = "windows_i686_gnu"
-version = "0.52.5"
+version = "0.52.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "88ba073cf16d5372720ec942a8ccbf61626074c6d4dd2e745299726ce8b89670"
+checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b"
 
 [[package]]
 name = "windows_i686_gnullvm"
-version = "0.52.5"
+version = "0.52.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "87f4261229030a858f36b459e748ae97545d6f1ec60e5e0d6a3d32e0dc232ee9"
+checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66"
 
 [[package]]
 name = "windows_i686_msvc"
@@ -3786,9 +3819,9 @@ checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406"
 
 [[package]]
 name = "windows_i686_msvc"
-version = "0.52.5"
+version = "0.52.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "db3c2bf3d13d5b658be73463284eaf12830ac9a26a90c717b7f771dfe97487bf"
+checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66"
 
 [[package]]
 name = "windows_x86_64_gnu"
@@ -3798,9 +3831,9 @@ checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e"
 
 [[package]]
 name = "windows_x86_64_gnu"
-version = "0.52.5"
+version = "0.52.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4e4246f76bdeff09eb48875a0fd3e2af6aada79d409d33011886d3e1581517d9"
+checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78"
 
 [[package]]
 name = "windows_x86_64_gnullvm"
@@ -3810,9 +3843,9 @@ checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc"
 
 [[package]]
 name = "windows_x86_64_gnullvm"
-version = "0.52.5"
+version = "0.52.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "852298e482cd67c356ddd9570386e2862b5673c85bd5f88df9ab6802b334c596"
+checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d"
 
 [[package]]
 name = "windows_x86_64_msvc"
@@ -3822,9 +3855,9 @@ checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538"
 
 [[package]]
 name = "windows_x86_64_msvc"
-version = "0.52.5"
+version = "0.52.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bec47e5bfd1bff0eeaf6d8b485cc1074891a197ab4225d504cb7a1ab88b02bf0"
+checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
 
 [[package]]
 name = "winnow"

Cargo.toml

@@ -105,9 +105,10 @@ pyproject-toml = { version = "0.9.0" }
 quick-junit = { version = "0.4.0" }
 quote = { version = "1.0.23" }
 rand = { version = "0.8.5" }
+rustc-hash = { version = "2.0.0" }
 rayon = { version = "1.10.0" }
 regex = { version = "1.10.2" }
-rustc-hash = { version = "2.0.0" }
+foldhash = { version = "0.1.0" }
 salsa = { git = "https://github.com/MichaReiser/salsa.git", tag = "red-knot-0.0.1" }
 schemars = { version = "0.8.16" }
 seahash = { version = "4.1.0" }

README.md

@@ -136,8 +136,8 @@ curl -LsSf https://astral.sh/ruff/install.sh | sh
 powershell -c "irm https://astral.sh/ruff/install.ps1 | iex"
 
 # For a specific version.
-curl -LsSf https://astral.sh/ruff/0.5.7/install.sh | sh
-powershell -c "irm https://astral.sh/ruff/0.5.7/install.ps1 | iex"
+curl -LsSf https://astral.sh/ruff/0.6.1/install.sh | sh
+powershell -c "irm https://astral.sh/ruff/0.6.1/install.ps1 | iex"
 ```
 
 You can also install Ruff via [Homebrew](https://formulae.brew.sh/formula/ruff), [Conda](https://anaconda.org/conda-forge/ruff),
@@ -170,7 +170,7 @@ Ruff can also be used as a [pre-commit](https://pre-commit.com/) hook via [`ruff
 ```yaml
 - repo: https://github.com/astral-sh/ruff-pre-commit
   # Ruff version.
-  rev: v0.5.7
+  rev: v0.6.1
   hooks:
     # Run the linter.
     - id: ruff

crates/red_knot/docs/tracing.md

@@ -72,6 +72,26 @@ runs or when restoring from a persistent cache. This can be confusing for users
 don't understand why a specific lint violation isn't raised. Instead, change your
 query to return the failure as part of the query's result or use a Salsa accumulator.
 
+## Tracing in tests
+
+You can use `ruff_db::testing::setup_logging` or `ruff_db::testing::setup_logging_with_filter` to set up logging in tests.
+
+```rust
+use ruff_db::testing::setup_logging;
+
+#[test]
+fn test() {
+  let _logging = setup_logging();
+
+  tracing::info!("This message will be printed to stderr");
+}
+```
+
+Note: Most test runners capture stderr and only show its output when a test fails.
+
+Note also that `setup_logging` only sets up logging for the current thread because [`set_global_default`](https://docs.rs/tracing/latest/tracing/subscriber/fn.set_global_default.html) can only be
+called **once**.
+
 ## Release builds
 
 `trace!` events are removed in release builds.

crates/red_knot/src/main.rs

@@ -7,13 +7,13 @@ use colored::Colorize;
 use crossbeam::channel as crossbeam_channel;
 use salsa::plumbing::ZalsaDatabase;
 
+use red_knot_python_semantic::{ProgramSettings, SearchPathSettings};
 use red_knot_server::run_server;
 use red_knot_workspace::db::RootDatabase;
-use red_knot_workspace::site_packages::site_packages_dirs_of_venv;
+use red_knot_workspace::site_packages::VirtualEnvironment;
 use red_knot_workspace::watch;
 use red_knot_workspace::watch::WorkspaceWatcher;
 use red_knot_workspace::workspace::WorkspaceMetadata;
-use ruff_db::program::{ProgramSettings, SearchPathSettings};
 use ruff_db::system::{OsSystem, System, SystemPath, SystemPathBuf};
 use target_version::TargetVersion;
 
@@ -164,16 +164,9 @@ fn run() -> anyhow::Result<ExitStatus> {
 
     // TODO: Verify the remaining search path settings eagerly.
     let site_packages = venv_path
-        .map(|venv_path| {
-            let venv_path = SystemPath::absolute(venv_path, &cli_base_path);
-
-            if system.is_directory(&venv_path) {
-                Ok(site_packages_dirs_of_venv(&venv_path, &system)?)
-            } else {
-                Err(anyhow!(
-                    "Provided venv-path {venv_path} is not a directory!"
-                ))
-            }
+        .map(|path| {
+            VirtualEnvironment::new(path, &OsSystem::new(cli_base_path))
+                .and_then(|venv| venv.site_packages_directories(&system))
         })
         .transpose()?
         .unwrap_or_default();
@@ -191,7 +184,7 @@ fn run() -> anyhow::Result<ExitStatus> {
 
     // TODO: Use the `program_settings` to compute the key for the database's persistent
     //   cache and load the cache if it exists.
-    let mut db = RootDatabase::new(workspace_metadata, program_settings, system);
+    let mut db = RootDatabase::new(workspace_metadata, program_settings, system)?;
 
     let (main_loop, main_loop_cancellation_token) = MainLoop::new();
 

crates/red_knot/src/target_version.rs

@@ -13,22 +13,36 @@ pub enum TargetVersion {
     Py313,
 }
 
-impl std::fmt::Display for TargetVersion {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        ruff_db::program::TargetVersion::from(*self).fmt(f)
-    }
-}
-
-impl From<TargetVersion> for ruff_db::program::TargetVersion {
-    fn from(value: TargetVersion) -> Self {
-        match value {
-            TargetVersion::Py37 => Self::Py37,
-            TargetVersion::Py38 => Self::Py38,
-            TargetVersion::Py39 => Self::Py39,
-            TargetVersion::Py310 => Self::Py310,
-            TargetVersion::Py311 => Self::Py311,
-            TargetVersion::Py312 => Self::Py312,
-            TargetVersion::Py313 => Self::Py313,
+impl TargetVersion {
+    const fn as_str(self) -> &'static str {
+        match self {
+            Self::Py37 => "py37",
+            Self::Py38 => "py38",
+            Self::Py39 => "py39",
+            Self::Py310 => "py310",
+            Self::Py311 => "py311",
+            Self::Py312 => "py312",
+            Self::Py313 => "py313",
+        }
+    }
+}
+
+impl std::fmt::Display for TargetVersion {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.write_str(self.as_str())
+    }
+}
+
+impl From<TargetVersion> for red_knot_python_semantic::PythonVersion {
+    fn from(value: TargetVersion) -> Self {
+        match value {
+            TargetVersion::Py37 => Self::PY37,
+            TargetVersion::Py38 => Self::PY38,
+            TargetVersion::Py39 => Self::PY39,
+            TargetVersion::Py310 => Self::PY310,
+            TargetVersion::Py311 => Self::PY311,
+            TargetVersion::Py312 => Self::PY312,
+            TargetVersion::Py313 => Self::PY313,
         }
     }
 }

crates/red_knot/tests/file_watching.rs

@@ -4,15 +4,15 @@ use std::io::Write;
 use std::time::Duration;
 
 use anyhow::{anyhow, Context};
-use salsa::Setter;
 
-use red_knot_python_semantic::{resolve_module, ModuleName};
+use red_knot_python_semantic::{
+    resolve_module, ModuleName, Program, ProgramSettings, PythonVersion, SearchPathSettings,
+};
 use red_knot_workspace::db::RootDatabase;
 use red_knot_workspace::watch;
 use red_knot_workspace::watch::{directory_watcher, WorkspaceWatcher};
 use red_knot_workspace::workspace::WorkspaceMetadata;
 use ruff_db::files::{system_path_to_file, File, FileError};
-use ruff_db::program::{Program, ProgramSettings, SearchPathSettings, TargetVersion};
 use ruff_db::source::source_text;
 use ruff_db::system::{OsSystem, SystemPath, SystemPathBuf};
 use ruff_db::Upcast;
@@ -25,6 +25,7 @@ struct TestCase {
     /// We need to hold on to it in the test case or the temp files get deleted.
     _temp_dir: tempfile::TempDir,
     root_dir: SystemPathBuf,
+    search_path_settings: SearchPathSettings,
 }
 
 impl TestCase {
@@ -107,18 +108,20 @@ impl TestCase {
     fn update_search_path_settings(
         &mut self,
         f: impl FnOnce(&SearchPathSettings) -> SearchPathSettings,
-    ) {
+    ) -> anyhow::Result<()> {
         let program = Program::get(self.db());
-        let search_path_settings = program.search_paths(self.db());
 
-        let new_settings = f(search_path_settings);
+        let new_settings = f(&self.search_path_settings);
 
-        program.set_search_paths(&mut self.db).to(new_settings);
+        program.update_search_paths(&mut self.db, new_settings.clone())?;
+        self.search_path_settings = new_settings;
 
         if let Some(watcher) = &mut self.watcher {
             watcher.update(&self.db);
             assert!(!watcher.has_errored_paths());
         }
+
+        Ok(())
     }
 
     fn collect_package_files(&self, path: &SystemPath) -> Vec<File> {
@@ -220,24 +223,24 @@ where
     let system = OsSystem::new(&workspace_path);
 
     let workspace = WorkspaceMetadata::from_path(&workspace_path, &system)?;
-    let search_paths = create_search_paths(&root_path, workspace.root());
+    let search_path_settings = create_search_paths(&root_path, workspace.root());
 
-    for path in search_paths
+    for path in search_path_settings
         .extra_paths
         .iter()
-        .chain(search_paths.site_packages.iter())
-        .chain(search_paths.custom_typeshed.iter())
+        .chain(search_path_settings.site_packages.iter())
+        .chain(search_path_settings.custom_typeshed.iter())
     {
         std::fs::create_dir_all(path.as_std_path())
             .with_context(|| format!("Failed to create search path '{path}'"))?;
     }
 
     let settings = ProgramSettings {
-        target_version: TargetVersion::default(),
-        search_paths,
+        target_version: PythonVersion::default(),
+        search_paths: search_path_settings.clone(),
     };
 
-    let db = RootDatabase::new(workspace, settings, system);
+    let db = RootDatabase::new(workspace, settings, system)?;
 
     let (sender, receiver) = crossbeam::channel::unbounded();
     let watcher = directory_watcher(move |events| sender.send(events).unwrap())
@@ -252,6 +255,7 @@ where
         watcher: Some(watcher),
         _temp_dir: temp_dir,
         root_dir: root_path,
+        search_path_settings,
     };
 
     // Sometimes the file watcher reports changes for events that happened before the watcher was started.
@@ -736,7 +740,8 @@ fn add_search_path() -> anyhow::Result<()> {
     case.update_search_path_settings(|settings| SearchPathSettings {
         site_packages: vec![site_packages.clone()],
         ..settings.clone()
-    });
+    })
+    .expect("Search path settings to be valid");
 
     std::fs::write(site_packages.join("a.py").as_std_path(), "class A: ...")?;
 
@@ -766,7 +771,8 @@ fn remove_search_path() -> anyhow::Result<()> {
     case.update_search_path_settings(|settings| SearchPathSettings {
         site_packages: vec![],
         ..settings.clone()
-    });
+    })
+    .expect("Search path settings to be valid");
 
     std::fs::write(site_packages.join("a.py").as_std_path(), "class A: ...")?;
 

crates/red_knot_python_semantic/Cargo.toml

@@ -15,8 +15,10 @@ ruff_db = { workspace = true }
 ruff_index = { workspace = true }
 ruff_python_ast = { workspace = true }
 ruff_python_stdlib = { workspace = true }
+ruff_source_file = { workspace = true }
 ruff_text_size = { workspace = true }
 
+anyhow = { workspace = true }
 bitflags = { workspace = true }
 camino = { workspace = true }
 compact_str = { workspace = true }
@@ -34,12 +36,14 @@ walkdir = { workspace = true }
 zip = { workspace = true, features = ["zstd", "deflate"] }
 
 [dev-dependencies]
+ruff_db = { workspace = true, features = ["os", "testing"] }
+ruff_python_parser = { workspace = true }
+
 anyhow = { workspace = true }
 insta = { workspace = true }
 tempfile = { workspace = true }
 walkdir = { workspace = true }
 zip = { workspace = true }
-ruff_python_parser = { workspace = true }
 
 [lints]
 workspace = true

crates/red_knot_python_semantic/src/lib.rs

@@ -5,6 +5,8 @@ use rustc_hash::FxHasher;
 pub use db::Db;
 pub use module_name::ModuleName;
 pub use module_resolver::{resolve_module, system_module_search_paths, vendored_typeshed_stubs};
+pub use program::{Program, ProgramSettings, SearchPathSettings};
+pub use python_version::PythonVersion;
 pub use semantic_model::{HasTy, SemanticModel};
 
 pub mod ast_node_ref;
@@ -13,6 +15,8 @@ mod db;
 mod module_name;
 mod module_resolver;
 mod node_key;
+mod program;
+mod python_version;
 pub mod semantic_index;
 mod semantic_model;
 pub mod types;

crates/red_knot_python_semantic/src/module_name.rs

@@ -168,6 +168,24 @@ impl ModuleName {
         };
         Some(Self(name))
     }
+
+    /// Extend `self` with the components of `other`
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// use red_knot_python_semantic::ModuleName;
+    ///
+    /// let mut module_name = ModuleName::new_static("foo").unwrap();
+    /// module_name.extend(&ModuleName::new_static("bar").unwrap());
+    /// assert_eq!(&module_name, "foo.bar");
+    /// module_name.extend(&ModuleName::new_static("baz.eggs.ham").unwrap());
+    /// assert_eq!(&module_name, "foo.bar.baz.eggs.ham");
+    /// ```
+    pub fn extend(&mut self, other: &ModuleName) {
+        self.0.push('.');
+        self.0.push_str(other);
+    }
 }
 
 impl Deref for ModuleName {

crates/red_knot_python_semantic/src/module_resolver/mod.rs

@@ -2,11 +2,13 @@ use std::iter::FusedIterator;
 
 pub(crate) use module::Module;
 pub use resolver::resolve_module;
+pub(crate) use resolver::{file_to_module, SearchPaths};
 use ruff_db::system::SystemPath;
 pub use typeshed::vendored_typeshed_stubs;
 
+use crate::module_resolver::resolver::search_paths;
 use crate::Db;
-use resolver::{module_resolution_settings, SearchPathIterator};
+use resolver::SearchPathIterator;
 
 mod module;
 mod path;
@@ -20,7 +22,7 @@ mod testing;
 /// Returns an iterator over all search paths pointing to a system path
 pub fn system_module_search_paths(db: &dyn Db) -> SystemModuleSearchPathsIter {
     SystemModuleSearchPathsIter {
-        inner: module_resolution_settings(db).search_paths(db),
+        inner: search_paths(db),
     }
 }
 

crates/red_knot_python_semantic/src/module_resolver/module.rs

@@ -77,3 +77,9 @@ pub enum ModuleKind {
     /// A python package (`foo/__init__.py` or `foo/__init__.pyi`)
     Package,
 }
+
+impl ModuleKind {
+    pub const fn is_package(self) -> bool {
+        matches!(self, ModuleKind::Package)
+    }
+}

crates/red_knot_python_semantic/src/module_resolver/path.rs

@@ -620,14 +620,13 @@ impl PartialEq<SearchPath> for VendoredPathBuf {
 
 #[cfg(test)]
 mod tests {
-    use ruff_db::program::TargetVersion;
     use ruff_db::Db;
 
     use crate::db::tests::TestDb;
 
-    use crate::module_resolver::testing::{FileSpec, MockedTypeshed, TestCase, TestCaseBuilder};
-
     use super::*;
+    use crate::module_resolver::testing::{FileSpec, MockedTypeshed, TestCase, TestCaseBuilder};
+    use crate::python_version::PythonVersion;
 
     impl ModulePath {
         #[must_use]
@@ -867,7 +866,7 @@ mod tests {
 
     fn typeshed_test_case(
         typeshed: MockedTypeshed,
-        target_version: TargetVersion,
+        target_version: PythonVersion,
     ) -> (TestDb, SearchPath) {
         let TestCase { db, stdlib, .. } = TestCaseBuilder::new()
             .with_custom_typeshed(typeshed)
@@ -879,11 +878,11 @@ mod tests {
     }
 
     fn py38_typeshed_test_case(typeshed: MockedTypeshed) -> (TestDb, SearchPath) {
-        typeshed_test_case(typeshed, TargetVersion::Py38)
+        typeshed_test_case(typeshed, PythonVersion::PY38)
     }
 
     fn py39_typeshed_test_case(typeshed: MockedTypeshed) -> (TestDb, SearchPath) {
-        typeshed_test_case(typeshed, TargetVersion::Py39)
+        typeshed_test_case(typeshed, PythonVersion::PY39)
     }
 
     #[test]
@@ -899,7 +898,7 @@ mod tests {
         };
 
         let (db, stdlib_path) = py38_typeshed_test_case(TYPESHED);
-        let resolver = ResolverState::new(&db, TargetVersion::Py38);
+        let resolver = ResolverState::new(&db, PythonVersion::PY38);
 
         let asyncio_regular_package = stdlib_path.join("asyncio");
         assert!(asyncio_regular_package.is_directory(&resolver));
@@ -927,7 +926,7 @@ mod tests {
         };
 
         let (db, stdlib_path) = py38_typeshed_test_case(TYPESHED);
-        let resolver = ResolverState::new(&db, TargetVersion::Py38);
+        let resolver = ResolverState::new(&db, PythonVersion::PY38);
 
         let xml_namespace_package = stdlib_path.join("xml");
         assert!(xml_namespace_package.is_directory(&resolver));
@@ -949,7 +948,7 @@ mod tests {
         };
 
         let (db, stdlib_path) = py38_typeshed_test_case(TYPESHED);
-        let resolver = ResolverState::new(&db, TargetVersion::Py38);
+        let resolver = ResolverState::new(&db, PythonVersion::PY38);
 
         let functools_module = stdlib_path.join("functools.pyi");
         assert!(functools_module.to_file(&resolver).is_some());
@@ -965,7 +964,7 @@ mod tests {
         };
 
         let (db, stdlib_path) = py38_typeshed_test_case(TYPESHED);
-        let resolver = ResolverState::new(&db, TargetVersion::Py38);
+        let resolver = ResolverState::new(&db, PythonVersion::PY38);
 
         let collections_regular_package = stdlib_path.join("collections");
         assert_eq!(collections_regular_package.to_file(&resolver), None);
@@ -981,7 +980,7 @@ mod tests {
         };
 
         let (db, stdlib_path) = py38_typeshed_test_case(TYPESHED);
-        let resolver = ResolverState::new(&db, TargetVersion::Py38);
+        let resolver = ResolverState::new(&db, PythonVersion::PY38);
 
         let importlib_namespace_package = stdlib_path.join("importlib");
         assert_eq!(importlib_namespace_package.to_file(&resolver), None);
@@ -1002,7 +1001,7 @@ mod tests {
         };
 
         let (db, stdlib_path) = py38_typeshed_test_case(TYPESHED);
-        let resolver = ResolverState::new(&db, TargetVersion::Py38);
+        let resolver = ResolverState::new(&db, PythonVersion::PY38);
 
         let non_existent = stdlib_path.join("doesnt_even_exist");
         assert_eq!(non_existent.to_file(&resolver), None);
@@ -1030,7 +1029,7 @@ mod tests {
         };
 
         let (db, stdlib_path) = py39_typeshed_test_case(TYPESHED);
-        let resolver = ResolverState::new(&db, TargetVersion::Py39);
+        let resolver = ResolverState::new(&db, PythonVersion::PY39);
 
         // Since we've set the target version to Py39,
         // `collections` should now exist as a directory, according to VERSIONS...
@@ -1059,7 +1058,7 @@ mod tests {
         };
 
         let (db, stdlib_path) = py39_typeshed_test_case(TYPESHED);
-        let resolver = ResolverState::new(&db, TargetVersion::Py39);
+        let resolver = ResolverState::new(&db, PythonVersion::PY39);
 
         // The `importlib` directory now also exists
         let importlib_namespace_package = stdlib_path.join("importlib");
@@ -1083,7 +1082,7 @@ mod tests {
         };
 
         let (db, stdlib_path) = py39_typeshed_test_case(TYPESHED);
-        let resolver = ResolverState::new(&db, TargetVersion::Py39);
+        let resolver = ResolverState::new(&db, PythonVersion::PY39);
 
         // The `xml` package no longer exists on py39:
         let xml_namespace_package = stdlib_path.join("xml");

crates/red_knot_python_semantic/src/module_resolver/resolver.rs

@@ -1,14 +1,15 @@
 use std::borrow::Cow;
 use std::iter::FusedIterator;
 
-use ruff_db::files::{File, FilePath, FileRootKind};
-use ruff_db::program::{Program, SearchPathSettings, TargetVersion};
-use ruff_db::system::{DirectoryEntry, System, SystemPath, SystemPathBuf};
-use ruff_db::vendored::VendoredPath;
 use rustc_hash::{FxBuildHasher, FxHashSet};
 
+use ruff_db::files::{File, FilePath, FileRootKind};
+use ruff_db::system::{DirectoryEntry, SystemPath, SystemPathBuf};
+use ruff_db::vendored::VendoredPath;
+
 use crate::db::Db;
 use crate::module_name::ModuleName;
+use crate::{Program, SearchPathSettings};
 
 use super::module::{Module, ModuleKind};
 use super::path::{ModulePath, SearchPath, SearchPathValidationError};
@@ -84,9 +85,7 @@ pub(crate) fn file_to_module(db: &dyn Db, file: File) -> Option<Module> {
         FilePath::SystemVirtual(_) => return None,
     };
 
-    let settings = module_resolution_settings(db);
-
-    let mut search_paths = settings.search_paths(db);
+    let mut search_paths = search_paths(db);
 
     let module_name = loop {
         let candidate = search_paths.next()?;
@@ -119,92 +118,122 @@ pub(crate) fn file_to_module(db: &dyn Db, file: File) -> Option<Module> {
     }
 }
 
-/// Validate and normalize the raw settings given by the user
-/// into settings we can use for module resolution
-///
-/// This method also implements the typing spec's [module resolution order].
-///
-/// [module resolution order]: https://typing.readthedocs.io/en/latest/spec/distributing.html#import-resolution-ordering
-fn try_resolve_module_resolution_settings(
-    db: &dyn Db,
-) -> Result<ModuleResolutionSettings, SearchPathValidationError> {
-    let program = Program::get(db.upcast());
-
-    let SearchPathSettings {
-        extra_paths,
-        src_root,
-        custom_typeshed,
-        site_packages,
-    } = program.search_paths(db.upcast());
-
-    if !extra_paths.is_empty() {
-        tracing::info!("Extra search paths: {extra_paths:?}");
-    }
-
-    if let Some(custom_typeshed) = custom_typeshed {
-        tracing::info!("Custom typeshed directory: {custom_typeshed}");
-    }
-
-    if !site_packages.is_empty() {
-        tracing::info!("Site-packages directories: {site_packages:?}");
-    }
-
-    let system = db.system();
-    let files = db.files();
-
-    let mut static_search_paths = vec![];
-
-    for path in extra_paths {
-        files.try_add_root(db.upcast(), path, FileRootKind::LibrarySearchPath);
-        static_search_paths.push(SearchPath::extra(system, path.clone())?);
-    }
-
-    static_search_paths.push(SearchPath::first_party(system, src_root.clone())?);
-
-    static_search_paths.push(if let Some(custom_typeshed) = custom_typeshed.as_ref() {
-        files.try_add_root(
-            db.upcast(),
-            custom_typeshed,
-            FileRootKind::LibrarySearchPath,
-        );
-        SearchPath::custom_stdlib(db, custom_typeshed.clone())?
-    } else {
-        SearchPath::vendored_stdlib()
-    });
-
-    // TODO vendor typeshed's third-party stubs as well as the stdlib and fallback to them as a final step
-
-    let target_version = program.target_version(db.upcast());
-    tracing::info!("Target version: {target_version}");
-
-    // Filter out module resolution paths that point to the same directory on disk (the same invariant maintained by [`sys.path` at runtime]).
-    // (Paths may, however, *overlap* -- e.g. you could have both `src/` and `src/foo`
-    // as module resolution paths simultaneously.)
-    //
-    // [`sys.path` at runtime]: https://docs.python.org/3/library/site.html#module-site
-    // This code doesn't use an `IndexSet` because the key is the system path and not the search root.
-    let mut seen_paths =
-        FxHashSet::with_capacity_and_hasher(static_search_paths.len(), FxBuildHasher);
-
-    static_search_paths.retain(|path| {
-        if let Some(path) = path.as_system_path() {
-            seen_paths.insert(path.to_path_buf())
-        } else {
-            true
-        }
-    });
-
-    Ok(ModuleResolutionSettings {
-        target_version,
-        static_search_paths,
-        site_packages_paths: site_packages.to_owned(),
-    })
+pub(crate) fn search_paths(db: &dyn Db) -> SearchPathIterator {
+    Program::get(db).search_paths(db).iter(db)
 }
 
-#[salsa::tracked(return_ref)]
-pub(crate) fn module_resolution_settings(db: &dyn Db) -> ModuleResolutionSettings {
-    // TODO proper error handling if this returns an error:
-    try_resolve_module_resolution_settings(db).unwrap()
+#[derive(Debug, PartialEq, Eq, Default)]
+pub(crate) struct SearchPaths {
+    /// Search paths that have been statically determined purely from reading Ruff's configuration settings.
+    /// These shouldn't ever change unless the config settings themselves change.
+    static_paths: Vec<SearchPath>,
+
+    /// site-packages paths are not included in the above field:
+    /// if there are multiple site-packages paths, editable installations can appear
+    /// *between* the site-packages paths on `sys.path` at runtime.
+    /// That means we can't know where a second or third `site-packages` path should sit
+    /// in terms of module-resolution priority until we've discovered the editable installs
+    /// for the first `site-packages` path
+    site_packages: Vec<SearchPath>,
+}
+
+impl SearchPaths {
+    /// Validate and normalize the raw settings given by the user
+    /// into settings we can use for module resolution
+    ///
+    /// This method also implements the typing spec's [module resolution order].
+    ///
+    /// [module resolution order]: https://typing.readthedocs.io/en/latest/spec/distributing.html#import-resolution-ordering
+    pub(crate) fn from_settings(
+        db: &dyn Db,
+        settings: SearchPathSettings,
+    ) -> Result<Self, SearchPathValidationError> {
+        let SearchPathSettings {
+            extra_paths,
+            src_root,
+            custom_typeshed,
+            site_packages: site_packages_paths,
+        } = settings;
+
+        let system = db.system();
+        let files = db.files();
+
+        let mut static_paths = vec![];
+
+        for path in extra_paths {
+            tracing::debug!("Adding static extra search-path '{path}'");
+
+            let search_path = SearchPath::extra(system, path)?;
+            files.try_add_root(
+                db.upcast(),
+                search_path.as_system_path().unwrap(),
+                FileRootKind::LibrarySearchPath,
+            );
+            static_paths.push(search_path);
+        }
+
+        tracing::debug!("Adding static search path '{src_root}'");
+        static_paths.push(SearchPath::first_party(system, src_root)?);
+
+        static_paths.push(if let Some(custom_typeshed) = custom_typeshed {
+            tracing::debug!("Adding static custom-sdtlib search-path '{custom_typeshed}'");
+
+            let search_path = SearchPath::custom_stdlib(db, custom_typeshed)?;
+            files.try_add_root(
+                db.upcast(),
+                search_path.as_system_path().unwrap(),
+                FileRootKind::LibrarySearchPath,
+            );
+            search_path
+        } else {
+            SearchPath::vendored_stdlib()
+        });
+
+        let mut site_packages: Vec<_> = Vec::with_capacity(site_packages_paths.len());
+
+        for path in site_packages_paths {
+            tracing::debug!("Adding site-package path '{path}'");
+            let search_path = SearchPath::site_packages(system, path)?;
+            files.try_add_root(
+                db.upcast(),
+                search_path.as_system_path().unwrap(),
+                FileRootKind::LibrarySearchPath,
+            );
+            site_packages.push(search_path);
+        }
+
+        // TODO vendor typeshed's third-party stubs as well as the stdlib and fallback to them as a final step
+
+        // Filter out module resolution paths that point to the same directory on disk (the same invariant maintained by [`sys.path` at runtime]).
+        // (Paths may, however, *overlap* -- e.g. you could have both `src/` and `src/foo`
+        // as module resolution paths simultaneously.)
+        //
+        // This code doesn't use an `IndexSet` because the key is the system path and not the search root.
+        //
+        // [`sys.path` at runtime]: https://docs.python.org/3/library/site.html#module-site
+        let mut seen_paths = FxHashSet::with_capacity_and_hasher(static_paths.len(), FxBuildHasher);
+
+        static_paths.retain(|path| {
+            if let Some(path) = path.as_system_path() {
+                seen_paths.insert(path.to_path_buf())
+            } else {
+                true
+            }
+        });
+
+        Ok(SearchPaths {
+            static_paths,
+            site_packages,
+        })
+    }
+
+    pub(crate) fn iter<'a>(&'a self, db: &'a dyn Db) -> SearchPathIterator<'a> {
+        SearchPathIterator {
+            db,
+            static_paths: self.static_paths.iter(),
+            dynamic_paths: None,
+        }
+    }
 }
 
 /// Collect all dynamic search paths. For each `site-packages` path:
@@ -217,19 +246,20 @@ pub(crate) fn module_resolution_settings(db: &dyn Db) -> ModuleResolutionSetting
 /// module-resolution priority.
 #[salsa::tracked(return_ref)]
 pub(crate) fn dynamic_resolution_paths(db: &dyn Db) -> Vec<SearchPath> {
-    let ModuleResolutionSettings {
-        target_version: _,
-        static_search_paths,
-        site_packages_paths,
-    } = module_resolution_settings(db);
+    tracing::debug!("Resolving dynamic module resolution paths");
+
+    let SearchPaths {
+        static_paths,
+        site_packages,
+    } = Program::get(db).search_paths(db);
 
     let mut dynamic_paths = Vec::new();
 
-    if site_packages_paths.is_empty() {
+    if site_packages.is_empty() {
         return dynamic_paths;
     }
 
-    let mut existing_paths: FxHashSet<_> = static_search_paths
+    let mut existing_paths: FxHashSet<_> = static_paths
         .iter()
         .filter_map(|path| path.as_system_path())
         .map(Cow::Borrowed)
@@ -238,15 +268,19 @@ pub(crate) fn dynamic_resolution_paths(db: &dyn Db) -> Vec<SearchPath> {
     let files = db.files();
     let system = db.system();
 
-    for site_packages_dir in site_packages_paths {
+    for site_packages_search_path in site_packages {
+        let site_packages_dir = site_packages_search_path
+            .as_system_path()
+            .expect("Expected site package path to be a system path");
+
         if !existing_paths.insert(Cow::Borrowed(site_packages_dir)) {
             continue;
         }
-        let site_packages_root = files.try_add_root(
-            db.upcast(),
-            site_packages_dir,
-            FileRootKind::LibrarySearchPath,
-        );
+
+        let site_packages_root = files
+            .root(db.upcast(), site_packages_dir)
+            .expect("Site-package root to have been created.");
+
         // This query needs to be re-executed each time a `.pth` file
         // is added, modified or removed from the `site-packages` directory.
         // However, we don't use Salsa queries to read the source text of `.pth` files;
@@ -254,8 +288,7 @@ pub(crate) fn dynamic_resolution_paths(db: &dyn Db) -> Vec<SearchPath> {
         // site-package directory's revision.
         site_packages_root.revision(db.upcast());
 
-        dynamic_paths
-            .push(SearchPath::site_packages(system, site_packages_dir.to_owned()).unwrap());
+        dynamic_paths.push(site_packages_search_path.clone());
 
         // As well as modules installed directly into `site-packages`,
         // the directory may also contain `.pth` files.
@@ -263,22 +296,38 @@ pub(crate) fn dynamic_resolution_paths(db: &dyn Db) -> Vec<SearchPath> {
         // containing a (relative or absolute) path.
         // Each of these paths may point to an editable install of a package,
         // so should be considered an additional search path.
-        let Ok(pth_file_iterator) = PthFileIterator::new(db, site_packages_dir) else {
-            continue;
+        let pth_file_iterator = match PthFileIterator::new(db, site_packages_dir) {
+            Ok(iterator) => iterator,
+            Err(error) => {
+                tracing::warn!(
+                    "Failed to search for editable installation in {site_packages_dir}: {error}"
+                );
+                continue;
+            }
         };
 
         // The Python documentation specifies that `.pth` files in `site-packages`
         // are processed in alphabetical order, so collecting and then sorting is necessary.
         // https://docs.python.org/3/library/site.html#module-site
         let mut all_pth_files: Vec<PthFile> = pth_file_iterator.collect();
-        all_pth_files.sort_by(|a, b| a.path.cmp(&b.path));
+        all_pth_files.sort_unstable_by(|a, b| a.path.cmp(&b.path));
 
-        for pth_file in &all_pth_files {
-            for installation in pth_file.editable_installations() {
-                if existing_paths.insert(Cow::Owned(
-                    installation.as_system_path().unwrap().to_path_buf(),
-                )) {
-                    dynamic_paths.push(installation);
+        let installations = all_pth_files.iter().flat_map(PthFile::items);
+
+        for installation in installations {
+            if existing_paths.insert(Cow::Owned(installation.clone())) {
+                match SearchPath::editable(system, installation) {
+                    Ok(search_path) => {
+                        tracing::debug!(
+                            "Adding editable installation to module resolution path {path}",
+                            path = search_path.as_system_path().unwrap()
+                        );
+                        dynamic_paths.push(search_path);
+                    }
+
+                    Err(error) => {
+                        tracing::debug!("Skipping editable installation: {error}");
+                    }
                 }
             }
         }
@@ -324,7 +373,6 @@ impl<'db> FusedIterator for SearchPathIterator<'db> {}
 /// One or more lines in a `.pth` file may be a (relative or absolute)
 /// path that represents an editable installation of a package.
 struct PthFile<'db> {
-    system: &'db dyn System,
     path: SystemPathBuf,
     contents: String,
     site_packages: &'db SystemPath,
@@ -333,9 +381,8 @@ struct PthFile<'db> {
 impl<'db> PthFile<'db> {
     /// Yield paths in this `.pth` file that appear to represent editable installations,
     /// and should therefore be added as module-resolution search paths.
-    fn editable_installations(&'db self) -> impl Iterator<Item = SearchPath> + 'db {
+    fn items(&'db self) -> impl Iterator<Item = SystemPathBuf> + 'db {
         let PthFile {
-            system,
             path: _,
             contents,
             site_packages,
@@ -354,8 +401,8 @@ impl<'db> PthFile<'db> {
             {
                 return None;
             }
-            let possible_editable_install = SystemPath::absolute(line, site_packages);
-            SearchPath::editable(*system, possible_editable_install).ok()
+
+            Some(SystemPath::absolute(line, site_packages))
         })
     }
 }
@@ -404,12 +451,15 @@ impl<'db> Iterator for PthFileIterator<'db> {
                 continue;
             }
 
-            let Ok(contents) = db.system().read_to_string(&path) else {
-                continue;
+            let contents = match system.read_to_string(&path) {
+                Ok(contents) => contents,
+                Err(error) => {
+                    tracing::warn!("Failed to read .pth file '{path}': {error}");
+                    continue;
+                }
             };
 
             return Some(PthFile {
-                system,
                 path,
                 contents,
                 site_packages,
@@ -418,38 +468,6 @@ impl<'db> Iterator for PthFileIterator<'db> {
     }
 }
 
-/// Validated and normalized module-resolution settings.
-#[derive(Clone, Debug, PartialEq, Eq)]
-pub(crate) struct ModuleResolutionSettings {
-    target_version: TargetVersion,
-
-    /// Search paths that have been statically determined purely from reading Ruff's configuration settings.
-    /// These shouldn't ever change unless the config settings themselves change.
-    static_search_paths: Vec<SearchPath>,
-
-    /// site-packages paths are not included in the above field:
-    /// if there are multiple site-packages paths, editable installations can appear
-    /// *between* the site-packages paths on `sys.path` at runtime.
-    /// That means we can't know where a second or third `site-packages` path should sit
-    /// in terms of module-resolution priority until we've discovered the editable installs
-    /// for the first `site-packages` path
-    site_packages_paths: Vec<SystemPathBuf>,
-}
-
-impl ModuleResolutionSettings {
-    fn target_version(&self) -> TargetVersion {
-        self.target_version
-    }
-
-    pub(crate) fn search_paths<'db>(&'db self, db: &'db dyn Db) -> SearchPathIterator<'db> {
-        SearchPathIterator {
-            db,
-            static_paths: self.static_search_paths.iter(),
-            dynamic_paths: None,
-        }
-    }
-}
-
 /// A thin wrapper around `ModuleName` to make it a Salsa ingredient.
 ///
 /// This is needed because Salsa requires that all query arguments are salsa ingredients.
@@ -462,14 +480,13 @@ struct ModuleNameIngredient<'db> {
 /// Given a module name and a list of search paths in which to lookup modules,
 /// attempt to resolve the module name
 fn resolve_name(db: &dyn Db, name: &ModuleName) -> Option<(SearchPath, File, ModuleKind)> {
-    let resolver_settings = module_resolution_settings(db);
-    let target_version = resolver_settings.target_version();
+    let program = Program::get(db);
+    let target_version = program.target_version(db);
     let resolver_state = ResolverState::new(db, target_version);
-    let (_, minor_version) = target_version.as_tuple();
     let is_builtin_module =
-        ruff_python_stdlib::sys::is_builtin_module(minor_version, name.as_str());
+        ruff_python_stdlib::sys::is_builtin_module(target_version.minor, name.as_str());
 
-    for search_path in resolver_settings.search_paths(db) {
+    for search_path in search_paths(db) {
         // When a builtin module is imported, standard module resolution is bypassed:
         // the module name always resolves to the stdlib module,
         // even if there's a module of the same name in the first-party root
@@ -623,6 +640,8 @@ mod tests {
     use crate::module_name::ModuleName;
     use crate::module_resolver::module::ModuleKind;
     use crate::module_resolver::testing::{FileSpec, MockedTypeshed, TestCase, TestCaseBuilder};
+    use crate::ProgramSettings;
+    use crate::PythonVersion;
 
     use super::*;
 
@@ -677,7 +696,7 @@ mod tests {
         let TestCase { db, stdlib, .. } = TestCaseBuilder::new()
             .with_src_files(SRC)
             .with_custom_typeshed(TYPESHED)
-            .with_target_version(TargetVersion::Py38)
+            .with_target_version(PythonVersion::PY38)
             .build();
 
         let builtins_module_name = ModuleName::new_static("builtins").unwrap();
@@ -695,7 +714,7 @@ mod tests {
 
         let TestCase { db, stdlib, .. } = TestCaseBuilder::new()
             .with_custom_typeshed(TYPESHED)
-            .with_target_version(TargetVersion::Py38)
+            .with_target_version(PythonVersion::PY38)
             .build();
 
         let functools_module_name = ModuleName::new_static("functools").unwrap();
@@ -748,7 +767,7 @@ mod tests {
 
         let TestCase { db, stdlib, .. } = TestCaseBuilder::new()
             .with_custom_typeshed(TYPESHED)
-            .with_target_version(TargetVersion::Py38)
+            .with_target_version(PythonVersion::PY38)
             .build();
 
         let existing_modules = create_module_names(&["asyncio", "functools", "xml.etree"]);
@@ -793,7 +812,7 @@ mod tests {
 
         let TestCase { db, .. } = TestCaseBuilder::new()
             .with_custom_typeshed(TYPESHED)
-            .with_target_version(TargetVersion::Py38)
+            .with_target_version(PythonVersion::PY38)
             .build();
 
         let nonexisting_modules = create_module_names(&[
@@ -837,7 +856,7 @@ mod tests {
 
         let TestCase { db, stdlib, .. } = TestCaseBuilder::new()
             .with_custom_typeshed(TYPESHED)
-            .with_target_version(TargetVersion::Py39)
+            .with_target_version(PythonVersion::PY39)
             .build();
 
         let existing_modules = create_module_names(&[
@@ -879,7 +898,7 @@ mod tests {
 
         let TestCase { db, .. } = TestCaseBuilder::new()
             .with_custom_typeshed(TYPESHED)
-            .with_target_version(TargetVersion::Py39)
+            .with_target_version(PythonVersion::PY39)
             .build();
 
         let nonexisting_modules = create_module_names(&["importlib", "xml", "xml.etree"]);
@@ -903,7 +922,7 @@ mod tests {
         let TestCase { db, src, .. } = TestCaseBuilder::new()
             .with_src_files(SRC)
             .with_custom_typeshed(TYPESHED)
-            .with_target_version(TargetVersion::Py38)
+            .with_target_version(PythonVersion::PY38)
             .build();
 
         let functools_module_name = ModuleName::new_static("functools").unwrap();
@@ -927,7 +946,7 @@ mod tests {
     fn stdlib_uses_vendored_typeshed_when_no_custom_typeshed_supplied() {
         let TestCase { db, stdlib, .. } = TestCaseBuilder::new()
             .with_vendored_typeshed()
-            .with_target_version(TargetVersion::default())
+            .with_target_version(PythonVersion::default())
             .build();
 
         let pydoc_data_topics_name = ModuleName::new_static("pydoc_data.topics").unwrap();
@@ -1143,7 +1162,7 @@ mod tests {
     fn symlink() -> anyhow::Result<()> {
         use anyhow::Context;
 
-        use ruff_db::program::Program;
+        use crate::program::Program;
         use ruff_db::system::{OsSystem, SystemPath};
 
         use crate::db::tests::TestDb;
@@ -1173,14 +1192,19 @@ mod tests {
         std::fs::write(foo.as_std_path(), "")?;
         std::os::unix::fs::symlink(foo.as_std_path(), bar.as_std_path())?;
 
-        let search_paths = SearchPathSettings {
-            extra_paths: vec![],
-            src_root: src.clone(),
-            custom_typeshed: Some(custom_typeshed.clone()),
-            site_packages: vec![site_packages],
-        };
-
-        Program::new(&db, TargetVersion::Py38, search_paths);
+        Program::from_settings(
+            &db,
+            ProgramSettings {
+                target_version: PythonVersion::PY38,
+                search_paths: SearchPathSettings {
+                    extra_paths: vec![],
+                    src_root: src.clone(),
+                    custom_typeshed: Some(custom_typeshed.clone()),
+                    site_packages: vec![site_packages],
+                },
+            },
+        )
+        .context("Invalid program settings")?;
 
         let foo_module = resolve_module(&db, ModuleName::new_static("foo").unwrap()).unwrap();
         let bar_module = resolve_module(&db, ModuleName::new_static("bar").unwrap()).unwrap();
@@ -1214,7 +1238,7 @@ mod tests {
     fn deleting_an_unrelated_file_doesnt_change_module_resolution() {
         let TestCase { mut db, src, .. } = TestCaseBuilder::new()
             .with_src_files(&[("foo.py", "x = 1"), ("bar.py", "x = 2")])
-            .with_target_version(TargetVersion::Py38)
+            .with_target_version(PythonVersion::PY38)
             .build();
 
         let foo_module_name = ModuleName::new_static("foo").unwrap();
@@ -1302,7 +1326,7 @@ mod tests {
             ..
         } = TestCaseBuilder::new()
             .with_custom_typeshed(TYPESHED)
-            .with_target_version(TargetVersion::Py38)
+            .with_target_version(PythonVersion::PY38)
             .build();
 
         let functools_module_name = ModuleName::new_static("functools").unwrap();
@@ -1350,7 +1374,7 @@ mod tests {
             ..
         } = TestCaseBuilder::new()
             .with_custom_typeshed(TYPESHED)
-            .with_target_version(TargetVersion::Py38)
+            .with_target_version(PythonVersion::PY38)
             .build();
 
         let functools_module_name = ModuleName::new_static("functools").unwrap();
@@ -1390,7 +1414,7 @@ mod tests {
         } = TestCaseBuilder::new()
             .with_src_files(SRC)
             .with_custom_typeshed(TYPESHED)
-            .with_target_version(TargetVersion::Py38)
+            .with_target_version(PythonVersion::PY38)
             .build();
 
         let functools_module_name = ModuleName::new_static("functools").unwrap();
@@ -1644,8 +1668,7 @@ not_a_directory
             .with_site_packages_files(&[("_foo.pth", "/src")])
             .build();
 
-        let search_paths: Vec<&SearchPath> =
-            module_resolution_settings(&db).search_paths(&db).collect();
+        let search_paths: Vec<&SearchPath> = search_paths(&db).collect();
 
         assert!(search_paths.contains(
             &&SearchPath::first_party(db.system(), SystemPathBuf::from("/src")).unwrap()
@@ -1674,16 +1697,19 @@ not_a_directory
         ])
         .unwrap();
 
-        Program::new(
+        Program::from_settings(
             &db,
-            TargetVersion::default(),
-            SearchPathSettings {
-                extra_paths: vec![],
-                src_root: SystemPathBuf::from("/src"),
-                custom_typeshed: None,
-                site_packages: vec![venv_site_packages, system_site_packages],
+            ProgramSettings {
+                target_version: PythonVersion::default(),
+                search_paths: SearchPathSettings {
+                    extra_paths: vec![],
+                    src_root: SystemPathBuf::from("/src"),
+                    custom_typeshed: None,
+                    site_packages: vec![venv_site_packages, system_site_packages],
+                },
             },
-        );
+        )
+        .expect("Valid program settings");
 
         // The editable installs discovered from the `.pth` file in the first `site-packages` directory
         // take precedence over the second `site-packages` directory...

crates/red_knot_python_semantic/src/module_resolver/state.rs

@@ -1,17 +1,17 @@
-use ruff_db::program::TargetVersion;
 use ruff_db::vendored::VendoredFileSystem;
 
 use super::typeshed::LazyTypeshedVersions;
 use crate::db::Db;
+use crate::python_version::PythonVersion;
 
 pub(crate) struct ResolverState<'db> {
     pub(crate) db: &'db dyn Db,
     pub(crate) typeshed_versions: LazyTypeshedVersions<'db>,
-    pub(crate) target_version: TargetVersion,
+    pub(crate) target_version: PythonVersion,
 }
 
 impl<'db> ResolverState<'db> {
-    pub(crate) fn new(db: &'db dyn Db, target_version: TargetVersion) -> Self {
+    pub(crate) fn new(db: &'db dyn Db, target_version: PythonVersion) -> Self {
         Self {
             db,
             typeshed_versions: LazyTypeshedVersions::new(),

crates/red_knot_python_semantic/src/module_resolver/testing.rs

@@ -1,8 +1,10 @@
-use ruff_db::program::{Program, SearchPathSettings, TargetVersion};
 use ruff_db::system::{DbWithTestSystem, SystemPath, SystemPathBuf};
 use ruff_db::vendored::VendoredPathBuf;
 
 use crate::db::tests::TestDb;
+use crate::program::{Program, SearchPathSettings};
+use crate::python_version::PythonVersion;
+use crate::ProgramSettings;
 
 /// A test case for the module resolver.
 ///
@@ -16,7 +18,7 @@ pub(crate) struct TestCase<T> {
     // so this is a single directory instead of a `Vec` of directories,
     // like it is in `ruff_db::Program`.
     pub(crate) site_packages: SystemPathBuf,
-    pub(crate) target_version: TargetVersion,
+    pub(crate) target_version: PythonVersion,
 }
 
 /// A `(file_name, file_contents)` tuple
@@ -98,7 +100,7 @@ pub(crate) struct UnspecifiedTypeshed;
 /// to `()`.
 pub(crate) struct TestCaseBuilder<T> {
     typeshed_option: T,
-    target_version: TargetVersion,
+    target_version: PythonVersion,
     first_party_files: Vec<FileSpec>,
     site_packages_files: Vec<FileSpec>,
 }
@@ -117,7 +119,7 @@ impl<T> TestCaseBuilder<T> {
     }
 
     /// Specify the target Python version the module resolver should assume
-    pub(crate) fn with_target_version(mut self, target_version: TargetVersion) -> Self {
+    pub(crate) fn with_target_version(mut self, target_version: PythonVersion) -> Self {
         self.target_version = target_version;
         self
     }
@@ -144,7 +146,7 @@ impl TestCaseBuilder<UnspecifiedTypeshed> {
     pub(crate) fn new() -> TestCaseBuilder<UnspecifiedTypeshed> {
         Self {
             typeshed_option: UnspecifiedTypeshed,
-            target_version: TargetVersion::default(),
+            target_version: PythonVersion::default(),
             first_party_files: vec![],
             site_packages_files: vec![],
         }
@@ -219,16 +221,19 @@ impl TestCaseBuilder<MockedTypeshed> {
         let src = Self::write_mock_directory(&mut db, "/src", first_party_files);
         let typeshed = Self::build_typeshed_mock(&mut db, &typeshed_option);
 
-        Program::new(
+        Program::from_settings(
             &db,
-            target_version,
-            SearchPathSettings {
-                extra_paths: vec![],
-                src_root: src.clone(),
-                custom_typeshed: Some(typeshed.clone()),
-                site_packages: vec![site_packages.clone()],
+            ProgramSettings {
+                target_version,
+                search_paths: SearchPathSettings {
+                    extra_paths: vec![],
+                    src_root: src.clone(),
+                    custom_typeshed: Some(typeshed.clone()),
+                    site_packages: vec![site_packages.clone()],
+                },
             },
-        );
+        )
+        .expect("Valid program settings");
 
         TestCase {
             db,
@@ -272,16 +277,19 @@ impl TestCaseBuilder<VendoredTypeshed> {
             Self::write_mock_directory(&mut db, "/site-packages", site_packages_files);
         let src = Self::write_mock_directory(&mut db, "/src", first_party_files);
 
-        Program::new(
+        Program::from_settings(
             &db,
-            target_version,
-            SearchPathSettings {
-                extra_paths: vec![],
-                src_root: src.clone(),
-                custom_typeshed: None,
-                site_packages: vec![site_packages.clone()],
+            ProgramSettings {
+                target_version,
+                search_paths: SearchPathSettings {
+                    extra_paths: vec![],
+                    src_root: src.clone(),
+                    custom_typeshed: None,
+                    site_packages: vec![site_packages.clone()],
+                },
             },
-        );
+        )
+        .expect("Valid search path settings");
 
         TestCase {
             db,

crates/red_knot_python_semantic/src/module_resolver/typeshed/versions.rs

@@ -6,16 +6,15 @@ use std::ops::{RangeFrom, RangeInclusive};
 use std::str::FromStr;
 
 use once_cell::sync::Lazy;
-use ruff_db::program::TargetVersion;
 use ruff_db::system::SystemPath;
 use rustc_hash::FxHashMap;
 
 use ruff_db::files::{system_path_to_file, File};
 
+use super::vendored::vendored_typeshed_stubs;
 use crate::db::Db;
 use crate::module_name::ModuleName;
-
-use super::vendored::vendored_typeshed_stubs;
+use crate::python_version::PythonVersion;
 
 #[derive(Debug)]
 pub(crate) struct LazyTypeshedVersions<'db>(OnceCell<&'db TypeshedVersions>);
@@ -44,7 +43,7 @@ impl<'db> LazyTypeshedVersions<'db> {
         db: &'db dyn Db,
         module: &ModuleName,
         stdlib_root: Option<&SystemPath>,
-        target_version: TargetVersion,
+        target_version: PythonVersion,
     ) -> TypeshedVersionsQueryResult {
         let versions = self.0.get_or_init(|| {
             let versions_path = if let Some(system_path) = stdlib_root {
@@ -64,7 +63,7 @@ impl<'db> LazyTypeshedVersions<'db> {
             // Unwrapping here is not correct...
             parse_typeshed_versions(db, versions_file).as_ref().unwrap()
         });
-        versions.query_module(module, PyVersion::from(target_version))
+        versions.query_module(module, target_version)
     }
 }
 
@@ -178,7 +177,7 @@ impl TypeshedVersions {
     fn query_module(
         &self,
         module: &ModuleName,
-        target_version: PyVersion,
+        target_version: PythonVersion,
     ) -> TypeshedVersionsQueryResult {
         if let Some(range) = self.exact(module) {
             if range.contains(target_version) {
@@ -323,13 +322,13 @@ impl fmt::Display for TypeshedVersions {
 
 #[derive(Debug, Clone, Eq, PartialEq, Hash)]
 enum PyVersionRange {
-    AvailableFrom(RangeFrom<PyVersion>),
-    AvailableWithin(RangeInclusive<PyVersion>),
+    AvailableFrom(RangeFrom<PythonVersion>),
+    AvailableWithin(RangeInclusive<PythonVersion>),
 }
 
 impl PyVersionRange {
     #[must_use]
-    fn contains(&self, version: PyVersion) -> bool {
+    fn contains(&self, version: PythonVersion) -> bool {
         match self {
             Self::AvailableFrom(inner) => inner.contains(&version),
             Self::AvailableWithin(inner) => inner.contains(&version),
@@ -343,9 +342,14 @@ impl FromStr for PyVersionRange {
     fn from_str(s: &str) -> Result<Self, Self::Err> {
         let mut parts = s.split('-').map(str::trim);
         match (parts.next(), parts.next(), parts.next()) {
-            (Some(lower), Some(""), None) => Ok(Self::AvailableFrom((lower.parse()?)..)),
+            (Some(lower), Some(""), None) => {
+                let lower = PythonVersion::from_versions_file_string(lower)?;
+                Ok(Self::AvailableFrom(lower..))
+            }
             (Some(lower), Some(upper), None) => {
-                Ok(Self::AvailableWithin((lower.parse()?)..=(upper.parse()?)))
+                let lower = PythonVersion::from_versions_file_string(lower)?;
+                let upper = PythonVersion::from_versions_file_string(upper)?;
+                Ok(Self::AvailableWithin(lower..=upper))
             }
             _ => Err(TypeshedVersionsParseErrorKind::UnexpectedNumberOfHyphens),
         }
@@ -363,74 +367,20 @@ impl fmt::Display for PyVersionRange {
     }
 }
 
-#[derive(Debug, Clone, Copy, Eq, PartialEq, Ord, PartialOrd, Hash)]
-struct PyVersion {
-    major: u8,
-    minor: u8,
-}
-
-impl FromStr for PyVersion {
-    type Err = TypeshedVersionsParseErrorKind;
-
-    fn from_str(s: &str) -> Result<Self, Self::Err> {
+impl PythonVersion {
+    fn from_versions_file_string(s: &str) -> Result<Self, TypeshedVersionsParseErrorKind> {
         let mut parts = s.split('.').map(str::trim);
         let (Some(major), Some(minor), None) = (parts.next(), parts.next(), parts.next()) else {
             return Err(TypeshedVersionsParseErrorKind::UnexpectedNumberOfPeriods(
                 s.to_string(),
             ));
         };
-        let major = match u8::from_str(major) {
-            Ok(major) => major,
-            Err(err) => {
-                return Err(TypeshedVersionsParseErrorKind::IntegerParsingFailure {
-                    version: s.to_string(),
-                    err,
-                })
+        PythonVersion::try_from((major, minor)).map_err(|int_parse_error| {
+            TypeshedVersionsParseErrorKind::IntegerParsingFailure {
+                version: s.to_string(),
+                err: int_parse_error,
             }
-        };
-        let minor = match u8::from_str(minor) {
-            Ok(minor) => minor,
-            Err(err) => {
-                return Err(TypeshedVersionsParseErrorKind::IntegerParsingFailure {
-                    version: s.to_string(),
-                    err,
-                })
-            }
-        };
-        Ok(Self { major, minor })
-    }
-}
-
-impl fmt::Display for PyVersion {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        let PyVersion { major, minor } = self;
-        write!(f, "{major}.{minor}")
-    }
-}
-
-impl From<TargetVersion> for PyVersion {
-    fn from(value: TargetVersion) -> Self {
-        match value {
-            TargetVersion::Py37 => PyVersion { major: 3, minor: 7 },
-            TargetVersion::Py38 => PyVersion { major: 3, minor: 8 },
-            TargetVersion::Py39 => PyVersion { major: 3, minor: 9 },
-            TargetVersion::Py310 => PyVersion {
-                major: 3,
-                minor: 10,
-            },
-            TargetVersion::Py311 => PyVersion {
-                major: 3,
-                minor: 11,
-            },
-            TargetVersion::Py312 => PyVersion {
-                major: 3,
-                minor: 12,
-            },
-            TargetVersion::Py313 => PyVersion {
-                major: 3,
-                minor: 13,
-            },
-        }
+        })
     }
 }
 
@@ -440,7 +390,6 @@ mod tests {
     use std::path::Path;
 
     use insta::assert_snapshot;
-    use ruff_db::program::TargetVersion;
 
     use super::*;
 
@@ -478,27 +427,27 @@ mod tests {
 
         assert!(versions.contains_exact(&asyncio));
         assert_eq!(
-            versions.query_module(&asyncio, TargetVersion::Py310.into()),
+            versions.query_module(&asyncio, PythonVersion::PY310),
             TypeshedVersionsQueryResult::Exists
         );
 
         assert!(versions.contains_exact(&asyncio_staggered));
         assert_eq!(
-            versions.query_module(&asyncio_staggered, TargetVersion::Py38.into()),
+            versions.query_module(&asyncio_staggered, PythonVersion::PY38),
             TypeshedVersionsQueryResult::Exists
         );
         assert_eq!(
-            versions.query_module(&asyncio_staggered, TargetVersion::Py37.into()),
+            versions.query_module(&asyncio_staggered, PythonVersion::PY37),
             TypeshedVersionsQueryResult::DoesNotExist
         );
 
         assert!(versions.contains_exact(&audioop));
         assert_eq!(
-            versions.query_module(&audioop, TargetVersion::Py312.into()),
+            versions.query_module(&audioop, PythonVersion::PY312),
             TypeshedVersionsQueryResult::Exists
         );
         assert_eq!(
-            versions.query_module(&audioop, TargetVersion::Py313.into()),
+            versions.query_module(&audioop, PythonVersion::PY313),
             TypeshedVersionsQueryResult::DoesNotExist
         );
     }
@@ -590,15 +539,15 @@ foo: 3.8-   # trailing comment
 
         assert!(parsed_versions.contains_exact(&bar));
         assert_eq!(
-            parsed_versions.query_module(&bar, TargetVersion::Py37.into()),
+            parsed_versions.query_module(&bar, PythonVersion::PY37),
             TypeshedVersionsQueryResult::Exists
         );
         assert_eq!(
-            parsed_versions.query_module(&bar, TargetVersion::Py310.into()),
+            parsed_versions.query_module(&bar, PythonVersion::PY310),
             TypeshedVersionsQueryResult::Exists
         );
         assert_eq!(
-            parsed_versions.query_module(&bar, TargetVersion::Py311.into()),
+            parsed_versions.query_module(&bar, PythonVersion::PY311),
             TypeshedVersionsQueryResult::DoesNotExist
         );
     }
@@ -610,15 +559,15 @@ foo: 3.8-   # trailing comment
 
         assert!(parsed_versions.contains_exact(&foo));
         assert_eq!(
-            parsed_versions.query_module(&foo, TargetVersion::Py37.into()),
+            parsed_versions.query_module(&foo, PythonVersion::PY37),
             TypeshedVersionsQueryResult::DoesNotExist
         );
         assert_eq!(
-            parsed_versions.query_module(&foo, TargetVersion::Py38.into()),
+            parsed_versions.query_module(&foo, PythonVersion::PY38),
             TypeshedVersionsQueryResult::Exists
         );
         assert_eq!(
-            parsed_versions.query_module(&foo, TargetVersion::Py311.into()),
+            parsed_versions.query_module(&foo, PythonVersion::PY311),
             TypeshedVersionsQueryResult::Exists
         );
     }
@@ -630,15 +579,15 @@ foo: 3.8-   # trailing comment
 
         assert!(parsed_versions.contains_exact(&bar_baz));
         assert_eq!(
-            parsed_versions.query_module(&bar_baz, TargetVersion::Py37.into()),
+            parsed_versions.query_module(&bar_baz, PythonVersion::PY37),
             TypeshedVersionsQueryResult::Exists
         );
         assert_eq!(
-            parsed_versions.query_module(&bar_baz, TargetVersion::Py39.into()),
+            parsed_versions.query_module(&bar_baz, PythonVersion::PY39),
             TypeshedVersionsQueryResult::Exists
         );
         assert_eq!(
-            parsed_versions.query_module(&bar_baz, TargetVersion::Py310.into()),
+            parsed_versions.query_module(&bar_baz, PythonVersion::PY310),
             TypeshedVersionsQueryResult::DoesNotExist
         );
     }
@@ -650,15 +599,15 @@ foo: 3.8-   # trailing comment
 
         assert!(!parsed_versions.contains_exact(&bar_eggs));
         assert_eq!(
-            parsed_versions.query_module(&bar_eggs, TargetVersion::Py37.into()),
+            parsed_versions.query_module(&bar_eggs, PythonVersion::PY37),
             TypeshedVersionsQueryResult::MaybeExists
         );
         assert_eq!(
-            parsed_versions.query_module(&bar_eggs, TargetVersion::Py310.into()),
+            parsed_versions.query_module(&bar_eggs, PythonVersion::PY310),
             TypeshedVersionsQueryResult::MaybeExists
         );
         assert_eq!(
-            parsed_versions.query_module(&bar_eggs, TargetVersion::Py311.into()),
+            parsed_versions.query_module(&bar_eggs, PythonVersion::PY311),
             TypeshedVersionsQueryResult::DoesNotExist
         );
     }
@@ -670,11 +619,11 @@ foo: 3.8-   # trailing comment
 
         assert!(!parsed_versions.contains_exact(&spam));
         assert_eq!(
-            parsed_versions.query_module(&spam, TargetVersion::Py37.into()),
+            parsed_versions.query_module(&spam, PythonVersion::PY37),
             TypeshedVersionsQueryResult::DoesNotExist
         );
         assert_eq!(
-            parsed_versions.query_module(&spam, TargetVersion::Py313.into()),
+            parsed_versions.query_module(&spam, PythonVersion::PY313),
             TypeshedVersionsQueryResult::DoesNotExist
         );
     }

crates/red_knot_python_semantic/src/program.rs

@@ -0,0 +1,78 @@
+use crate::python_version::PythonVersion;
+use anyhow::Context;
+use salsa::Durability;
+use salsa::Setter;
+
+use ruff_db::system::SystemPathBuf;
+
+use crate::module_resolver::SearchPaths;
+use crate::Db;
+
+#[salsa::input(singleton)]
+pub struct Program {
+    pub target_version: PythonVersion,
+
+    #[default]
+    #[return_ref]
+    pub(crate) search_paths: SearchPaths,
+}
+
+impl Program {
+    pub fn from_settings(db: &dyn Db, settings: ProgramSettings) -> anyhow::Result<Self> {
+        let ProgramSettings {
+            target_version,
+            search_paths,
+        } = settings;
+
+        tracing::info!("Target version: {target_version}");
+
+        let search_paths = SearchPaths::from_settings(db, search_paths)
+            .with_context(|| "Invalid search path settings")?;
+
+        Ok(Program::builder(settings.target_version)
+            .durability(Durability::HIGH)
+            .search_paths(search_paths)
+            .new(db))
+    }
+
+    pub fn update_search_paths(
+        &self,
+        db: &mut dyn Db,
+        search_path_settings: SearchPathSettings,
+    ) -> anyhow::Result<()> {
+        let search_paths = SearchPaths::from_settings(db, search_path_settings)?;
+
+        if self.search_paths(db) != &search_paths {
+            tracing::debug!("Update search paths");
+            self.set_search_paths(db).to(search_paths);
+        }
+
+        Ok(())
+    }
+}
+
+#[derive(Debug, Eq, PartialEq)]
+pub struct ProgramSettings {
+    pub target_version: PythonVersion,
+    pub search_paths: SearchPathSettings,
+}
+
+/// Configures the search paths for module resolution.
+#[derive(Eq, PartialEq, Debug, Clone, Default)]
+pub struct SearchPathSettings {
+    /// List of user-provided paths that should take first priority in the module resolution.
+    /// Examples in other type checkers are mypy's MYPYPATH environment variable,
+    /// or pyright's stubPath configuration setting.
+    pub extra_paths: Vec<SystemPathBuf>,
+
+    /// The root of the workspace, used for finding first-party modules.
+    pub src_root: SystemPathBuf,
+
+    /// Optional path to a "custom typeshed" directory on disk for us to use for standard-library types.
+    /// If this is not provided, we will fallback to our vendored typeshed stubs for the stdlib,
+    /// bundled as a zip file in the binary
+    pub custom_typeshed: Option<SystemPathBuf>,
+
+    /// The path to the user's `site-packages` directory, where third-party packages from ``PyPI`` are installed.
+    pub site_packages: Vec<SystemPathBuf>,
+}

crates/red_knot_python_semantic/src/python_version.rs

@@ -0,0 +1,62 @@
+use std::fmt;
+
+/// Representation of a Python version.
+///
+/// Unlike the `TargetVersion` enums in the CLI crates,
+/// this does not necessarily represent a Python version that we actually support.
+#[derive(Debug, Clone, Copy, Eq, PartialEq, Ord, PartialOrd, Hash)]
+pub struct PythonVersion {
+    pub major: u8,
+    pub minor: u8,
+}
+
+impl PythonVersion {
+    pub const PY37: PythonVersion = PythonVersion { major: 3, minor: 7 };
+    pub const PY38: PythonVersion = PythonVersion { major: 3, minor: 8 };
+    pub const PY39: PythonVersion = PythonVersion { major: 3, minor: 9 };
+    pub const PY310: PythonVersion = PythonVersion {
+        major: 3,
+        minor: 10,
+    };
+    pub const PY311: PythonVersion = PythonVersion {
+        major: 3,
+        minor: 11,
+    };
+    pub const PY312: PythonVersion = PythonVersion {
+        major: 3,
+        minor: 12,
+    };
+    pub const PY313: PythonVersion = PythonVersion {
+        major: 3,
+        minor: 13,
+    };
+
+    pub fn free_threaded_build_available(self) -> bool {
+        self >= PythonVersion::PY313
+    }
+}
+
+impl Default for PythonVersion {
+    fn default() -> Self {
+        Self::PY38
+    }
+}
+
+impl TryFrom<(&str, &str)> for PythonVersion {
+    type Error = std::num::ParseIntError;
+
+    fn try_from(value: (&str, &str)) -> Result<Self, Self::Error> {
+        let (major, minor) = value;
+        Ok(Self {
+            major: major.parse()?,
+            minor: minor.parse()?,
+        })
+    }
+}
+
+impl fmt::Display for PythonVersion {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        let PythonVersion { major, minor } = self;
+        write!(f, "{major}.{minor}")
+    }
+}

crates/red_knot_python_semantic/src/semantic_index.rs

@@ -89,8 +89,6 @@ pub(crate) struct SemanticIndex<'db> {
     scopes: IndexVec<FileScopeId, Scope>,
 
     /// Map expressions to their corresponding scope.
-    /// We can't use [`ExpressionId`] here, because the challenge is how to get from
-    /// an [`ast::Expr`] to an [`ExpressionId`] (which requires knowing the scope).
     scopes_by_expression: FxHashMap<ExpressionNodeKey, FileScopeId>,
 
     /// Map from a node creating a definition to its definition.
@@ -118,7 +116,7 @@ pub(crate) struct SemanticIndex<'db> {
 impl<'db> SemanticIndex<'db> {
     /// Returns the symbol table for a specific scope.
     ///
-    /// Use the Salsa cached [`symbol_table`] query if you only need the
+    /// Use the Salsa cached [`symbol_table()`] query if you only need the
     /// symbol table for a single scope.
     pub(super) fn symbol_table(&self, scope_id: FileScopeId) -> Arc<SymbolTable> {
         self.symbol_tables[scope_id].clone()
@@ -126,7 +124,7 @@ impl<'db> SemanticIndex<'db> {
 
     /// Returns the use-def map for a specific scope.
     ///
-    /// Use the Salsa cached [`use_def_map`] query if you only need the
+    /// Use the Salsa cached [`use_def_map()`] query if you only need the
     /// use-def map for a single scope.
     pub(super) fn use_def_map(&self, scope_id: FileScopeId) -> Arc<UseDefMap> {
         self.use_def_maps[scope_id].clone()
@@ -309,6 +307,7 @@ mod tests {
     use ruff_db::parsed::parsed_module;
     use ruff_db::system::DbWithTestSystem;
     use ruff_python_ast as ast;
+    use ruff_text_size::{Ranged, TextRange};
 
     use crate::db::tests::TestDb;
     use crate::semantic_index::ast_ids::HasScopedUseId;
@@ -529,6 +528,235 @@ y = 2
         ));
     }
 
+    #[test]
+    fn function_parameter_symbols() {
+        let TestCase { db, file } = test_case(
+            "
+def f(a: str, /, b: str, c: int = 1, *args, d: int = 2, **kwargs):
+    pass
+",
+        );
+
+        let index = semantic_index(&db, file);
+        let global_table = symbol_table(&db, global_scope(&db, file));
+
+        assert_eq!(names(&global_table), vec!["f", "str", "int"]);
+
+        let [(function_scope_id, _function_scope)] = index
+            .child_scopes(FileScopeId::global())
+            .collect::<Vec<_>>()[..]
+        else {
+            panic!("Expected a function scope")
+        };
+
+        let function_table = index.symbol_table(function_scope_id);
+        assert_eq!(
+            names(&function_table),
+            vec!["a", "b", "c", "args", "d", "kwargs"],
+        );
+
+        let use_def = index.use_def_map(function_scope_id);
+        for name in ["a", "b", "c", "d"] {
+            let [definition] = use_def.public_definitions(
+                function_table
+                    .symbol_id_by_name(name)
+                    .expect("symbol exists"),
+            ) else {
+                panic!("Expected parameter definition for {name}");
+            };
+            assert!(matches!(
+                definition.node(&db),
+                DefinitionKind::ParameterWithDefault(_)
+            ));
+        }
+        for name in ["args", "kwargs"] {
+            let [definition] = use_def.public_definitions(
+                function_table
+                    .symbol_id_by_name(name)
+                    .expect("symbol exists"),
+            ) else {
+                panic!("Expected parameter definition for {name}");
+            };
+            assert!(matches!(definition.node(&db), DefinitionKind::Parameter(_)));
+        }
+    }
+
+    #[test]
+    fn lambda_parameter_symbols() {
+        let TestCase { db, file } = test_case("lambda a, b, c=1, *args, d=2, **kwargs: None");
+
+        let index = semantic_index(&db, file);
+        let global_table = symbol_table(&db, global_scope(&db, file));
+
+        assert!(names(&global_table).is_empty());
+
+        let [(lambda_scope_id, _lambda_scope)] = index
+            .child_scopes(FileScopeId::global())
+            .collect::<Vec<_>>()[..]
+        else {
+            panic!("Expected a lambda scope")
+        };
+
+        let lambda_table = index.symbol_table(lambda_scope_id);
+        assert_eq!(
+            names(&lambda_table),
+            vec!["a", "b", "c", "args", "d", "kwargs"],
+        );
+
+        let use_def = index.use_def_map(lambda_scope_id);
+        for name in ["a", "b", "c", "d"] {
+            let [definition] = use_def
+                .public_definitions(lambda_table.symbol_id_by_name(name).expect("symbol exists"))
+            else {
+                panic!("Expected parameter definition for {name}");
+            };
+            assert!(matches!(
+                definition.node(&db),
+                DefinitionKind::ParameterWithDefault(_)
+            ));
+        }
+        for name in ["args", "kwargs"] {
+            let [definition] = use_def
+                .public_definitions(lambda_table.symbol_id_by_name(name).expect("symbol exists"))
+            else {
+                panic!("Expected parameter definition for {name}");
+            };
+            assert!(matches!(definition.node(&db), DefinitionKind::Parameter(_)));
+        }
+    }
+
+    /// Test case to validate that the comprehension scope is correctly identified and that the target
+    /// variable is defined only in the comprehension scope and not in the global scope.
+    #[test]
+    fn comprehension_scope() {
+        let TestCase { db, file } = test_case(
+            "
+[x for x in iter1]
+",
+        );
+
+        let index = semantic_index(&db, file);
+        let global_table = index.symbol_table(FileScopeId::global());
+
+        assert_eq!(names(&global_table), vec!["iter1"]);
+
+        let [(comprehension_scope_id, comprehension_scope)] = index
+            .child_scopes(FileScopeId::global())
+            .collect::<Vec<_>>()[..]
+        else {
+            panic!("expected one child scope")
+        };
+
+        assert_eq!(comprehension_scope.kind(), ScopeKind::Comprehension);
+        assert_eq!(
+            comprehension_scope_id.to_scope_id(&db, file).name(&db),
+            "<listcomp>"
+        );
+
+        let comprehension_symbol_table = index.symbol_table(comprehension_scope_id);
+
+        assert_eq!(names(&comprehension_symbol_table), vec!["x"]);
+    }
+
+    /// Test case to validate that the `x` variable used in the comprehension is referencing the
+    /// `x` variable defined by the inner generator (`for x in iter2`) and not the outer one.
+    #[test]
+    fn multiple_generators() {
+        let TestCase { db, file } = test_case(
+            "
+[x for x in iter1 for x in iter2]
+",
+        );
+
+        let index = semantic_index(&db, file);
+        let [(comprehension_scope_id, _)] = index
+            .child_scopes(FileScopeId::global())
+            .collect::<Vec<_>>()[..]
+        else {
+            panic!("expected one child scope")
+        };
+
+        let use_def = index.use_def_map(comprehension_scope_id);
+
+        let module = parsed_module(&db, file).syntax();
+        let element = module.body[0]
+            .as_expr_stmt()
+            .unwrap()
+            .value
+            .as_list_comp_expr()
+            .unwrap()
+            .elt
+            .as_name_expr()
+            .unwrap();
+        let element_use_id =
+            element.scoped_use_id(&db, comprehension_scope_id.to_scope_id(&db, file));
+
+        let [definition] = use_def.use_definitions(element_use_id) else {
+            panic!("expected one definition")
+        };
+        let DefinitionKind::Comprehension(comprehension) = definition.node(&db) else {
+            panic!("expected generator definition")
+        };
+        let ast::Comprehension { target, .. } = comprehension.node();
+        let name = target.as_name_expr().unwrap().id().as_str();
+
+        assert_eq!(name, "x");
+        assert_eq!(target.range(), TextRange::new(23.into(), 24.into()));
+    }
+
+    /// Test case to validate that the nested comprehension creates a new scope which is a child of
+    /// the outer comprehension scope and the variables are correctly defined in the respective
+    /// scopes.
+    #[test]
+    fn nested_generators() {
+        let TestCase { db, file } = test_case(
+            "
+[{x for x in iter2} for y in iter1]
+",
+        );
+
+        let index = semantic_index(&db, file);
+        let global_table = index.symbol_table(FileScopeId::global());
+
+        assert_eq!(names(&global_table), vec!["iter1"]);
+
+        let [(comprehension_scope_id, comprehension_scope)] = index
+            .child_scopes(FileScopeId::global())
+            .collect::<Vec<_>>()[..]
+        else {
+            panic!("expected one child scope")
+        };
+
+        assert_eq!(comprehension_scope.kind(), ScopeKind::Comprehension);
+        assert_eq!(
+            comprehension_scope_id.to_scope_id(&db, file).name(&db),
+            "<listcomp>"
+        );
+
+        let comprehension_symbol_table = index.symbol_table(comprehension_scope_id);
+
+        assert_eq!(names(&comprehension_symbol_table), vec!["y", "iter2"]);
+
+        let [(inner_comprehension_scope_id, inner_comprehension_scope)] = index
+            .child_scopes(comprehension_scope_id)
+            .collect::<Vec<_>>()[..]
+        else {
+            panic!("expected one inner generator scope")
+        };
+
+        assert_eq!(inner_comprehension_scope.kind(), ScopeKind::Comprehension);
+        assert_eq!(
+            inner_comprehension_scope_id
+                .to_scope_id(&db, file)
+                .name(&db),
+            "<setcomp>"
+        );
+
+        let inner_comprehension_symbol_table = index.symbol_table(inner_comprehension_scope_id);
+
+        assert_eq!(names(&inner_comprehension_symbol_table), vec!["x"]);
+    }
+
     #[test]
     fn dupes() {
         let TestCase { db, file } = test_case(

crates/red_knot_python_semantic/src/semantic_index/ast_ids.rs

@@ -26,9 +26,9 @@ use crate::Db;
 /// ```
 #[derive(Debug)]
 pub(crate) struct AstIds {
-    /// Maps expressions to their expression id. Uses `NodeKey` because it avoids cloning [`Parsed`].
+    /// Maps expressions to their expression id.
     expressions_map: FxHashMap<ExpressionNodeKey, ScopedExpressionId>,
-    /// Maps expressions which "use" a symbol (that is, [`ExprName`]) to a use id.
+    /// Maps expressions which "use" a symbol (that is, [`ast::ExprName`]) to a use id.
     uses_map: FxHashMap<ExpressionNodeKey, ScopedUseId>,
 }
 

crates/red_knot_python_semantic/src/semantic_index/builder.rs

@@ -13,8 +13,8 @@ use crate::ast_node_ref::AstNodeRef;
 use crate::semantic_index::ast_ids::node_key::ExpressionNodeKey;
 use crate::semantic_index::ast_ids::AstIdsBuilder;
 use crate::semantic_index::definition::{
-    AssignmentDefinitionNodeRef, Definition, DefinitionNodeKey, DefinitionNodeRef,
-    ImportFromDefinitionNodeRef,
+    AssignmentDefinitionNodeRef, ComprehensionDefinitionNodeRef, Definition, DefinitionNodeKey,
+    DefinitionNodeRef, ImportFromDefinitionNodeRef,
 };
 use crate::semantic_index::expression::Expression;
 use crate::semantic_index::symbol::{
@@ -174,7 +174,7 @@ impl<'db> SemanticIndexBuilder<'db> {
         symbol: ScopedSymbolId,
         definition_node: impl Into<DefinitionNodeRef<'a>>,
     ) -> Definition<'db> {
-        let definition_node = definition_node.into();
+        let definition_node: DefinitionNodeRef<'_> = definition_node.into();
         let definition = Definition::new(
             self.db,
             self.file,
@@ -258,6 +258,49 @@ impl<'db> SemanticIndexBuilder<'db> {
         nested_scope
     }
 
+    /// Visit a list of [`Comprehension`] nodes, assumed to be the "generators" that compose a
+    /// comprehension (that is, the `for x in y` and `for y in z` parts of `x for x in y for y in z`.)
+    ///
+    /// [`Comprehension`]: ast::Comprehension
+    fn visit_generators(&mut self, scope: NodeWithScopeRef, generators: &'db [ast::Comprehension]) {
+        let mut generators_iter = generators.iter();
+
+        let Some(generator) = generators_iter.next() else {
+            unreachable!("Expression must contain at least one generator");
+        };
+
+        // The `iter` of the first generator is evaluated in the outer scope, while all subsequent
+        // nodes are evaluated in the inner scope.
+        self.visit_expr(&generator.iter);
+        self.push_scope(scope);
+
+        self.current_assignment = Some(CurrentAssignment::Comprehension {
+            node: generator,
+            first: true,
+        });
+        self.visit_expr(&generator.target);
+        self.current_assignment = None;
+
+        for expr in &generator.ifs {
+            self.visit_expr(expr);
+        }
+
+        for generator in generators_iter {
+            self.visit_expr(&generator.iter);
+
+            self.current_assignment = Some(CurrentAssignment::Comprehension {
+                node: generator,
+                first: false,
+            });
+            self.visit_expr(&generator.target);
+            self.current_assignment = None;
+
+            for expr in &generator.ifs {
+                self.visit_expr(expr);
+            }
+        }
+    }
+
     pub(super) fn build(mut self) -> SemanticIndex<'db> {
         let module = self.module;
         self.visit_body(module.suite());
@@ -325,6 +368,16 @@ where
                     .add_or_update_symbol(function_def.name.id.clone(), SymbolFlags::IS_DEFINED);
                 self.add_definition(symbol, function_def);
 
+                // The default value of the parameters needs to be evaluated in the
+                // enclosing scope.
+                for default in function_def
+                    .parameters
+                    .iter_non_variadic_params()
+                    .filter_map(|param| param.default.as_deref())
+                {
+                    self.visit_expr(default);
+                }
+
                 self.with_type_params(
                     NodeWithScopeRef::FunctionTypeParameters(function_def),
                     function_def.type_params.as_deref(),
@@ -335,6 +388,16 @@ where
                         }
 
                         builder.push_scope(NodeWithScopeRef::Function(function_def));
+
+                        // Add symbols and definitions for the parameters to the function scope.
+                        for parameter in &*function_def.parameters {
+                            let symbol = builder.add_or_update_symbol(
+                                parameter.name().id().clone(),
+                                SymbolFlags::IS_DEFINED,
+                            );
+                            builder.add_definition(symbol, parameter);
+                        }
+
                         builder.visit_body(&function_def.body);
                         builder.pop_scope()
                     },
@@ -476,8 +539,7 @@ where
         self.current_ast_ids().record_expression(expr);
 
         match expr {
-            ast::Expr::Name(name_node) => {
-                let ast::ExprName { id, ctx, .. } = name_node;
+            ast::Expr::Name(name_node @ ast::ExprName { id, ctx, .. }) => {
                 let flags = match ctx {
                     ast::ExprContext::Load => SymbolFlags::IS_USED,
                     ast::ExprContext::Store => SymbolFlags::IS_DEFINED,
@@ -500,8 +562,17 @@ where
                             self.add_definition(symbol, ann_assign);
                         }
                         Some(CurrentAssignment::Named(named)) => {
+                            // TODO(dhruvmanila): If the current scope is a comprehension, then the
+                            // named expression is implicitly nonlocal. This is yet to be
+                            // implemented.
                             self.add_definition(symbol, named);
                         }
+                        Some(CurrentAssignment::Comprehension { node, first }) => {
+                            self.add_definition(
+                                symbol,
+                                ComprehensionDefinitionNodeRef { node, first },
+                            );
+                        }
                         None => {}
                     }
                 }
@@ -523,11 +594,30 @@ where
             }
             ast::Expr::Lambda(lambda) => {
                 if let Some(parameters) = &lambda.parameters {
+                    // The default value of the parameters needs to be evaluated in the
+                    // enclosing scope.
+                    for default in parameters
+                        .iter_non_variadic_params()
+                        .filter_map(|param| param.default.as_deref())
+                    {
+                        self.visit_expr(default);
+                    }
                     self.visit_parameters(parameters);
                 }
                 self.push_scope(NodeWithScopeRef::Lambda(lambda));
+
+                // Add symbols and definitions for the parameters to the lambda scope.
+                if let Some(parameters) = &lambda.parameters {
+                    for parameter in &**parameters {
+                        let symbol = self.add_or_update_symbol(
+                            parameter.name().id().clone(),
+                            SymbolFlags::IS_DEFINED,
+                        );
+                        self.add_definition(symbol, parameter);
+                    }
+                }
+
                 self.visit_expr(lambda.body.as_ref());
-                self.pop_scope();
             }
             ast::Expr::If(ast::ExprIf {
                 body, test, orelse, ..
@@ -543,10 +633,74 @@ where
                 self.visit_expr(orelse);
                 self.flow_merge(&post_body);
             }
+            ast::Expr::ListComp(
+                list_comprehension @ ast::ExprListComp {
+                    elt, generators, ..
+                },
+            ) => {
+                self.visit_generators(
+                    NodeWithScopeRef::ListComprehension(list_comprehension),
+                    generators,
+                );
+                self.visit_expr(elt);
+            }
+            ast::Expr::SetComp(
+                set_comprehension @ ast::ExprSetComp {
+                    elt, generators, ..
+                },
+            ) => {
+                self.visit_generators(
+                    NodeWithScopeRef::SetComprehension(set_comprehension),
+                    generators,
+                );
+                self.visit_expr(elt);
+            }
+            ast::Expr::Generator(
+                generator @ ast::ExprGenerator {
+                    elt, generators, ..
+                },
+            ) => {
+                self.visit_generators(NodeWithScopeRef::GeneratorExpression(generator), generators);
+                self.visit_expr(elt);
+            }
+            ast::Expr::DictComp(
+                dict_comprehension @ ast::ExprDictComp {
+                    key,
+                    value,
+                    generators,
+                    ..
+                },
+            ) => {
+                self.visit_generators(
+                    NodeWithScopeRef::DictComprehension(dict_comprehension),
+                    generators,
+                );
+                self.visit_expr(key);
+                self.visit_expr(value);
+            }
             _ => {
                 walk_expr(self, expr);
             }
         }
+
+        if matches!(
+            expr,
+            ast::Expr::Lambda(_)
+                | ast::Expr::ListComp(_)
+                | ast::Expr::SetComp(_)
+                | ast::Expr::Generator(_)
+                | ast::Expr::DictComp(_)
+        ) {
+            self.pop_scope();
+        }
+    }
+
+    fn visit_parameters(&mut self, parameters: &'ast ruff_python_ast::Parameters) {
+        // Intentionally avoid walking default expressions, as we handle them in the enclosing
+        // scope.
+        for parameter in parameters.iter().map(ast::AnyParameterRef::as_parameter) {
+            self.visit_parameter(parameter);
+        }
     }
 }
 
@@ -555,6 +709,10 @@ enum CurrentAssignment<'a> {
     Assign(&'a ast::StmtAssign),
     AnnAssign(&'a ast::StmtAnnAssign),
     Named(&'a ast::ExprNamed),
+    Comprehension {
+        node: &'a ast::Comprehension,
+        first: bool,
+    },
 }
 
 impl<'a> From<&'a ast::StmtAssign> for CurrentAssignment<'a> {

crates/red_knot_python_semantic/src/semantic_index/definition.rs

@@ -44,6 +44,8 @@ pub(crate) enum DefinitionNodeRef<'a> {
     NamedExpression(&'a ast::ExprNamed),
     Assignment(AssignmentDefinitionNodeRef<'a>),
     AnnotatedAssignment(&'a ast::StmtAnnAssign),
+    Comprehension(ComprehensionDefinitionNodeRef<'a>),
+    Parameter(ast::AnyParameterRef<'a>),
 }
 
 impl<'a> From<&'a ast::StmtFunctionDef> for DefinitionNodeRef<'a> {
@@ -88,6 +90,18 @@ impl<'a> From<AssignmentDefinitionNodeRef<'a>> for DefinitionNodeRef<'a> {
     }
 }
 
+impl<'a> From<ComprehensionDefinitionNodeRef<'a>> for DefinitionNodeRef<'a> {
+    fn from(node: ComprehensionDefinitionNodeRef<'a>) -> Self {
+        Self::Comprehension(node)
+    }
+}
+
+impl<'a> From<ast::AnyParameterRef<'a>> for DefinitionNodeRef<'a> {
+    fn from(node: ast::AnyParameterRef<'a>) -> Self {
+        Self::Parameter(node)
+    }
+}
+
 #[derive(Copy, Clone, Debug)]
 pub(crate) struct ImportFromDefinitionNodeRef<'a> {
     pub(crate) node: &'a ast::StmtImportFrom,
@@ -100,6 +114,12 @@ pub(crate) struct AssignmentDefinitionNodeRef<'a> {
     pub(crate) target: &'a ast::ExprName,
 }
 
+#[derive(Copy, Clone, Debug)]
+pub(crate) struct ComprehensionDefinitionNodeRef<'a> {
+    pub(crate) node: &'a ast::Comprehension,
+    pub(crate) first: bool,
+}
+
 impl DefinitionNodeRef<'_> {
     #[allow(unsafe_code)]
     pub(super) unsafe fn into_owned(self, parsed: ParsedModule) -> DefinitionKind {
@@ -131,6 +151,20 @@ impl DefinitionNodeRef<'_> {
             DefinitionNodeRef::AnnotatedAssignment(assign) => {
                 DefinitionKind::AnnotatedAssignment(AstNodeRef::new(parsed, assign))
             }
+            DefinitionNodeRef::Comprehension(ComprehensionDefinitionNodeRef { node, first }) => {
+                DefinitionKind::Comprehension(ComprehensionDefinitionKind {
+                    node: AstNodeRef::new(parsed, node),
+                    first,
+                })
+            }
+            DefinitionNodeRef::Parameter(parameter) => match parameter {
+                ast::AnyParameterRef::Variadic(parameter) => {
+                    DefinitionKind::Parameter(AstNodeRef::new(parsed, parameter))
+                }
+                ast::AnyParameterRef::NonVariadic(parameter) => {
+                    DefinitionKind::ParameterWithDefault(AstNodeRef::new(parsed, parameter))
+                }
+            },
         }
     }
 
@@ -148,6 +182,11 @@ impl DefinitionNodeRef<'_> {
                 target,
             }) => target.into(),
             Self::AnnotatedAssignment(node) => node.into(),
+            Self::Comprehension(ComprehensionDefinitionNodeRef { node, first: _ }) => node.into(),
+            Self::Parameter(node) => match node {
+                ast::AnyParameterRef::Variadic(parameter) => parameter.into(),
+                ast::AnyParameterRef::NonVariadic(parameter) => parameter.into(),
+            },
         }
     }
 }
@@ -161,6 +200,25 @@ pub enum DefinitionKind {
     NamedExpression(AstNodeRef<ast::ExprNamed>),
     Assignment(AssignmentDefinitionKind),
     AnnotatedAssignment(AstNodeRef<ast::StmtAnnAssign>),
+    Comprehension(ComprehensionDefinitionKind),
+    Parameter(AstNodeRef<ast::Parameter>),
+    ParameterWithDefault(AstNodeRef<ast::ParameterWithDefault>),
+}
+
+#[derive(Clone, Debug)]
+pub struct ComprehensionDefinitionKind {
+    node: AstNodeRef<ast::Comprehension>,
+    first: bool,
+}
+
+impl ComprehensionDefinitionKind {
+    pub(crate) fn node(&self) -> &ast::Comprehension {
+        self.node.node()
+    }
+
+    pub(crate) fn is_first(&self) -> bool {
+        self.first
+    }
 }
 
 #[derive(Clone, Debug)]
@@ -230,3 +288,21 @@ impl From<&ast::StmtAnnAssign> for DefinitionNodeKey {
         Self(NodeKey::from_node(node))
     }
 }
+
+impl From<&ast::Comprehension> for DefinitionNodeKey {
+    fn from(node: &ast::Comprehension) -> Self {
+        Self(NodeKey::from_node(node))
+    }
+}
+
+impl From<&ast::Parameter> for DefinitionNodeKey {
+    fn from(node: &ast::Parameter) -> Self {
+        Self(NodeKey::from_node(node))
+    }
+}
+
+impl From<&ast::ParameterWithDefault> for DefinitionNodeKey {
+    fn from(node: &ast::ParameterWithDefault) -> Self {
+        Self(NodeKey::from_node(node))
+    }
+}

crates/red_knot_python_semantic/src/semantic_index/symbol.rs

@@ -114,6 +114,10 @@ impl<'db> ScopeId<'db> {
             NodeWithScopeKind::ClassTypeParameters(_)
                 | NodeWithScopeKind::FunctionTypeParameters(_)
                 | NodeWithScopeKind::Function(_)
+                | NodeWithScopeKind::ListComprehension(_)
+                | NodeWithScopeKind::SetComprehension(_)
+                | NodeWithScopeKind::DictComprehension(_)
+                | NodeWithScopeKind::GeneratorExpression(_)
         )
     }
 
@@ -127,6 +131,10 @@ impl<'db> ScopeId<'db> {
             NodeWithScopeKind::Function(function)
             | NodeWithScopeKind::FunctionTypeParameters(function) => function.name.as_str(),
             NodeWithScopeKind::Lambda(_) => "<lambda>",
+            NodeWithScopeKind::ListComprehension(_) => "<listcomp>",
+            NodeWithScopeKind::SetComprehension(_) => "<setcomp>",
+            NodeWithScopeKind::DictComprehension(_) => "<dictcomp>",
+            NodeWithScopeKind::GeneratorExpression(_) => "<generator>",
         }
     }
 }
@@ -170,6 +178,13 @@ pub enum ScopeKind {
     Annotation,
     Class,
     Function,
+    Comprehension,
+}
+
+impl ScopeKind {
+    pub const fn is_comprehension(self) -> bool {
+        matches!(self, ScopeKind::Comprehension)
+    }
 }
 
 /// Symbol table for a specific [`Scope`].
@@ -300,6 +315,10 @@ pub(crate) enum NodeWithScopeRef<'a> {
     Lambda(&'a ast::ExprLambda),
     FunctionTypeParameters(&'a ast::StmtFunctionDef),
     ClassTypeParameters(&'a ast::StmtClassDef),
+    ListComprehension(&'a ast::ExprListComp),
+    SetComprehension(&'a ast::ExprSetComp),
+    DictComprehension(&'a ast::ExprDictComp),
+    GeneratorExpression(&'a ast::ExprGenerator),
 }
 
 impl NodeWithScopeRef<'_> {
@@ -326,6 +345,18 @@ impl NodeWithScopeRef<'_> {
             NodeWithScopeRef::ClassTypeParameters(class) => {
                 NodeWithScopeKind::ClassTypeParameters(AstNodeRef::new(module, class))
             }
+            NodeWithScopeRef::ListComprehension(comprehension) => {
+                NodeWithScopeKind::ListComprehension(AstNodeRef::new(module, comprehension))
+            }
+            NodeWithScopeRef::SetComprehension(comprehension) => {
+                NodeWithScopeKind::SetComprehension(AstNodeRef::new(module, comprehension))
+            }
+            NodeWithScopeRef::DictComprehension(comprehension) => {
+                NodeWithScopeKind::DictComprehension(AstNodeRef::new(module, comprehension))
+            }
+            NodeWithScopeRef::GeneratorExpression(generator) => {
+                NodeWithScopeKind::GeneratorExpression(AstNodeRef::new(module, generator))
+            }
         }
     }
 
@@ -337,6 +368,10 @@ impl NodeWithScopeRef<'_> {
             NodeWithScopeRef::Lambda(_) => ScopeKind::Function,
             NodeWithScopeRef::FunctionTypeParameters(_)
             | NodeWithScopeRef::ClassTypeParameters(_) => ScopeKind::Annotation,
+            NodeWithScopeRef::ListComprehension(_)
+            | NodeWithScopeRef::SetComprehension(_)
+            | NodeWithScopeRef::DictComprehension(_)
+            | NodeWithScopeRef::GeneratorExpression(_) => ScopeKind::Comprehension,
         }
     }
 
@@ -356,6 +391,18 @@ impl NodeWithScopeRef<'_> {
             NodeWithScopeRef::ClassTypeParameters(class) => {
                 NodeWithScopeKey::ClassTypeParameters(NodeKey::from_node(class))
             }
+            NodeWithScopeRef::ListComprehension(comprehension) => {
+                NodeWithScopeKey::ListComprehension(NodeKey::from_node(comprehension))
+            }
+            NodeWithScopeRef::SetComprehension(comprehension) => {
+                NodeWithScopeKey::SetComprehension(NodeKey::from_node(comprehension))
+            }
+            NodeWithScopeRef::DictComprehension(comprehension) => {
+                NodeWithScopeKey::DictComprehension(NodeKey::from_node(comprehension))
+            }
+            NodeWithScopeRef::GeneratorExpression(generator) => {
+                NodeWithScopeKey::GeneratorExpression(NodeKey::from_node(generator))
+            }
         }
     }
 }
@@ -369,6 +416,10 @@ pub enum NodeWithScopeKind {
     Function(AstNodeRef<ast::StmtFunctionDef>),
     FunctionTypeParameters(AstNodeRef<ast::StmtFunctionDef>),
     Lambda(AstNodeRef<ast::ExprLambda>),
+    ListComprehension(AstNodeRef<ast::ExprListComp>),
+    SetComprehension(AstNodeRef<ast::ExprSetComp>),
+    DictComprehension(AstNodeRef<ast::ExprDictComp>),
+    GeneratorExpression(AstNodeRef<ast::ExprGenerator>),
 }
 
 #[derive(Copy, Clone, Debug, Eq, PartialEq, Hash)]
@@ -379,4 +430,8 @@ pub(crate) enum NodeWithScopeKey {
     Function(NodeKey),
     FunctionTypeParameters(NodeKey),
     Lambda(NodeKey),
+    ListComprehension(NodeKey),
+    SetComprehension(NodeKey),
+    DictComprehension(NodeKey),
+    GeneratorExpression(NodeKey),
 }

crates/red_knot_python_semantic/src/semantic_model.rs

@@ -1,6 +1,8 @@
-use ruff_db::files::File;
+use ruff_db::files::{File, FilePath};
+use ruff_db::source::line_index;
 use ruff_python_ast as ast;
 use ruff_python_ast::{Expr, ExpressionRef, StmtClassDef};
+use ruff_source_file::LineIndex;
 
 use crate::module_name::ModuleName;
 use crate::module_resolver::{resolve_module, Module};
@@ -25,6 +27,14 @@ impl<'db> SemanticModel<'db> {
         self.db
     }
 
+    pub fn file_path(&self) -> &FilePath {
+        self.file.path(self.db)
+    }
+
+    pub fn line_index(&self) -> LineIndex {
+        line_index(self.db.upcast(), self.file)
+    }
+
     pub fn resolve_module(&self, module_name: ModuleName) -> Option<Module> {
         resolve_module(self.db, module_name)
     }
@@ -165,34 +175,38 @@ impl HasTy for ast::Alias {
 mod tests {
     use ruff_db::files::system_path_to_file;
     use ruff_db::parsed::parsed_module;
-    use ruff_db::program::{Program, SearchPathSettings, TargetVersion};
     use ruff_db::system::{DbWithTestSystem, SystemPathBuf};
 
     use crate::db::tests::TestDb;
+    use crate::program::{Program, SearchPathSettings};
+    use crate::python_version::PythonVersion;
     use crate::types::Type;
-    use crate::{HasTy, SemanticModel};
+    use crate::{HasTy, ProgramSettings, SemanticModel};
 
-    fn setup_db() -> TestDb {
-        let db = TestDb::new();
-        Program::new(
+    fn setup_db<'a>(files: impl IntoIterator<Item = (&'a str, &'a str)>) -> anyhow::Result<TestDb> {
+        let mut db = TestDb::new();
+        db.write_files(files)?;
+
+        Program::from_settings(
             &db,
-            TargetVersion::Py38,
-            SearchPathSettings {
-                extra_paths: vec![],
-                src_root: SystemPathBuf::from("/src"),
-                site_packages: vec![],
-                custom_typeshed: None,
+            ProgramSettings {
+                target_version: PythonVersion::default(),
+                search_paths: SearchPathSettings {
+                    extra_paths: vec![],
+                    src_root: SystemPathBuf::from("/src"),
+                    site_packages: vec![],
+                    custom_typeshed: None,
+                },
             },
-        );
+        )?;
 
-        db
+        Ok(db)
     }
 
     #[test]
     fn function_ty() -> anyhow::Result<()> {
-        let mut db = setup_db();
+        let db = setup_db([("/src/foo.py", "def test(): pass")])?;
 
-        db.write_file("/src/foo.py", "def test(): pass")?;
         let foo = system_path_to_file(&db, "/src/foo.py").unwrap();
 
         let ast = parsed_module(&db, foo);
@@ -208,9 +222,8 @@ mod tests {
 
     #[test]
     fn class_ty() -> anyhow::Result<()> {
-        let mut db = setup_db();
+        let db = setup_db([("/src/foo.py", "class Test: pass")])?;
 
-        db.write_file("/src/foo.py", "class Test: pass")?;
         let foo = system_path_to_file(&db, "/src/foo.py").unwrap();
 
         let ast = parsed_module(&db, foo);
@@ -226,12 +239,11 @@ mod tests {
 
     #[test]
     fn alias_ty() -> anyhow::Result<()> {
-        let mut db = setup_db();
-
-        db.write_files([
+        let db = setup_db([
             ("/src/foo.py", "class Test: pass"),
             ("/src/bar.py", "from foo import Test"),
         ])?;
+
         let bar = system_path_to_file(&db, "/src/bar.py").unwrap();
 
         let ast = parsed_module(&db, bar);

crates/red_knot_python_semantic/src/types.rs

@@ -7,9 +7,11 @@ use crate::semantic_index::symbol::{ScopeId, ScopedSymbolId};
 use crate::semantic_index::{global_scope, symbol_table, use_def_map};
 use crate::{Db, FxOrderSet};
 
+mod builder;
 mod display;
 mod infer;
 
+pub(crate) use self::builder::UnionBuilder;
 pub(crate) use self::infer::{infer_definition_types, infer_scope_types};
 
 /// Infer the public type of a symbol (its type as seen from outside its scope).
@@ -91,14 +93,14 @@ pub(crate) fn definitions_ty<'db>(
     };
 
     if let Some(second) = all_types.next() {
-        let mut builder = UnionTypeBuilder::new(db);
+        let mut builder = UnionBuilder::new(db);
         builder = builder.add(first).add(second);
 
         for variant in all_types {
             builder = builder.add(variant);
         }
 
-        Type::Union(builder.build())
+        builder.build()
     } else {
         first
     }
@@ -111,13 +113,13 @@ pub enum Type<'db> {
     Any,
     /// the empty set of values
     Never,
-    /// unknown type (no annotation)
+    /// unknown type (either no annotation, or some kind of type error)
     /// equivalent to Any, or possibly to object in strict mode
     Unknown,
     /// name does not exist or is not bound to any value (this represents an error, but with some
     /// leniency options it could be silently resolved to Unknown in some cases)
     Unbound,
-    /// the None object (TODO remove this in favor of Instance(types.NoneType)
+    /// the None object -- TODO remove this in favor of Instance(types.NoneType)
     None,
     /// a specific function object
     Function(FunctionType<'db>),
@@ -127,8 +129,11 @@ pub enum Type<'db> {
     Class(ClassType<'db>),
     /// the set of Python objects with the given class in their __class__'s method resolution order
     Instance(ClassType<'db>),
+    /// the set of objects in any of the types in the union
     Union(UnionType<'db>),
+    /// the set of objects in all of the types in the intersection
     Intersection(IntersectionType<'db>),
+    /// An integer literal
     IntLiteral(i64),
     /// A boolean literal, either `True` or `False`.
     BooleanLiteral(bool),
@@ -144,6 +149,35 @@ impl<'db> Type<'db> {
         matches!(self, Type::Unknown)
     }
 
+    pub const fn is_never(&self) -> bool {
+        matches!(self, Type::Never)
+    }
+
+    pub fn may_be_unbound(&self, db: &'db dyn Db) -> bool {
+        match self {
+            Type::Unbound => true,
+            Type::Union(union) => union.contains(db, Type::Unbound),
+            // Unbound can't appear in an intersection, because an intersection with Unbound
+            // simplifies to just Unbound.
+            _ => false,
+        }
+    }
+
+    #[must_use]
+    pub fn replace_unbound_with(&self, db: &'db dyn Db, replacement: Type<'db>) -> Type<'db> {
+        match self {
+            Type::Unbound => replacement,
+            Type::Union(union) => union
+                .elements(db)
+                .into_iter()
+                .fold(UnionBuilder::new(db), |builder, ty| {
+                    builder.add(ty.replace_unbound_with(db, replacement))
+                })
+                .build(),
+            ty => *ty,
+        }
+    }
+
     #[must_use]
     pub fn member(&self, db: &'db dyn Db, name: &Name) -> Type<'db> {
         match self {
@@ -159,15 +193,13 @@ impl<'db> Type<'db> {
                 // TODO MRO? get_own_instance_member, get_instance_member
                 todo!("attribute lookup on Instance type")
             }
-            Type::Union(union) => Type::Union(
-                union
-                    .elements(db)
-                    .iter()
-                    .fold(UnionTypeBuilder::new(db), |builder, element_ty| {
-                        builder.add(element_ty.member(db, name))
-                    })
-                    .build(),
-            ),
+            Type::Union(union) => union
+                .elements(db)
+                .iter()
+                .fold(UnionBuilder::new(db), |builder, element_ty| {
+                    builder.add(element_ty.member(db, name))
+                })
+                .build(),
             Type::Intersection(_) => {
                 // TODO perform the get_member on each type in the intersection
                 // TODO return the intersection of those results
@@ -251,7 +283,7 @@ impl<'db> ClassType<'db> {
 
 #[salsa::interned]
 pub struct UnionType<'db> {
-    /// the union type includes values in any of these types
+    /// The union type includes values in any of these types.
     elements: FxOrderSet<Type<'db>>,
 }
 
@@ -261,48 +293,15 @@ impl<'db> UnionType<'db> {
     }
 }
 
-struct UnionTypeBuilder<'db> {
-    elements: FxOrderSet<Type<'db>>,
-    db: &'db dyn Db,
-}
-
-impl<'db> UnionTypeBuilder<'db> {
-    fn new(db: &'db dyn Db) -> Self {
-        Self {
-            db,
-            elements: FxOrderSet::default(),
-        }
-    }
-
-    /// Adds a type to this union.
-    fn add(mut self, ty: Type<'db>) -> Self {
-        match ty {
-            Type::Union(union) => {
-                self.elements.extend(&union.elements(self.db));
-            }
-            _ => {
-                self.elements.insert(ty);
-            }
-        }
-
-        self
-    }
-
-    fn build(self) -> UnionType<'db> {
-        UnionType::new(self.db, self.elements)
-    }
-}
-
-// Negation types aren't expressible in annotations, and are most likely to arise from type
-// narrowing along with intersections (e.g. `if not isinstance(...)`), so we represent them
-// directly in intersections rather than as a separate type. This sacrifices some efficiency in the
-// case where a Not appears outside an intersection (unclear when that could even happen, but we'd
-// have to represent it as a single-element intersection if it did) in exchange for better
-// efficiency in the within-intersection case.
 #[salsa::interned]
 pub struct IntersectionType<'db> {
-    // the intersection type includes only values in all of these types
+    /// The intersection type includes only values in all of these types.
     positive: FxOrderSet<Type<'db>>,
-    // the intersection type does not include any value in any of these types
+
+    /// The intersection type does not include any value in any of these types.
+    ///
+    /// Negation types aren't expressible in annotations, and are most likely to arise from type
+    /// narrowing along with intersections (e.g. `if not isinstance(...)`), so we represent them
+    /// directly in intersections rather than as a separate type.
     negative: FxOrderSet<Type<'db>>,
 }

crates/red_knot_python_semantic/src/types/builder.rs

@@ -0,0 +1,456 @@
+//! Smart builders for union and intersection types.
+//!
+//! Invariants we maintain here:
+//!   * No single-element union types (should just be the contained type instead.)
+//!   * No single-positive-element intersection types. Single-negative-element are OK, we don't
+//!     have a standalone negation type so there's no other representation for this.
+//!   * The same type should never appear more than once in a union or intersection. (This should
+//!     be expanded to cover subtyping -- see below -- but for now we only implement it for type
+//!     identity.)
+//!   * Disjunctive normal form (DNF): the tree of unions and intersections can never be deeper
+//!     than a union-of-intersections. Unions cannot contain other unions (the inner union just
+//!     flattens into the outer one), intersections cannot contain other intersections (also
+//!     flattens), and intersections cannot contain unions (the intersection distributes over the
+//!     union, inverting it into a union-of-intersections).
+//!
+//! The implication of these invariants is that a [`UnionBuilder`] does not necessarily build a
+//! [`Type::Union`]. For example, if only one type is added to the [`UnionBuilder`], `build()` will
+//! just return that type directly. The same is true for [`IntersectionBuilder`]; for example, if a
+//! union type is added to the intersection, it will distribute and [`IntersectionBuilder::build`]
+//! may end up returning a [`Type::Union`] of intersections.
+//!
+//! In the future we should have these additional invariants, but they aren't implemented yet:
+//!   * No type in a union can be a subtype of any other type in the union (just eliminate the
+//!     subtype from the union).
+//!   * No type in an intersection can be a supertype of any other type in the intersection (just
+//!     eliminate the supertype from the intersection).
+//!   * An intersection containing two non-overlapping types should simplify to [`Type::Never`].
+use crate::types::{IntersectionType, Type, UnionType};
+use crate::{Db, FxOrderSet};
+
+pub(crate) struct UnionBuilder<'db> {
+    elements: FxOrderSet<Type<'db>>,
+    db: &'db dyn Db,
+}
+
+impl<'db> UnionBuilder<'db> {
+    pub(crate) fn new(db: &'db dyn Db) -> Self {
+        Self {
+            db,
+            elements: FxOrderSet::default(),
+        }
+    }
+
+    /// Adds a type to this union.
+    pub(crate) fn add(mut self, ty: Type<'db>) -> Self {
+        match ty {
+            Type::Union(union) => {
+                self.elements.extend(&union.elements(self.db));
+            }
+            Type::Never => {}
+            _ => {
+                self.elements.insert(ty);
+            }
+        }
+
+        self
+    }
+
+    pub(crate) fn build(self) -> Type<'db> {
+        match self.elements.len() {
+            0 => Type::Never,
+            1 => self.elements[0],
+            _ => Type::Union(UnionType::new(self.db, self.elements)),
+        }
+    }
+}
+
+#[allow(unused)]
+#[derive(Clone)]
+pub(crate) struct IntersectionBuilder<'db> {
+    // Really this builds a union-of-intersections, because we always keep our set-theoretic types
+    // in disjunctive normal form (DNF), a union of intersections. In the simplest case there's
+    // just a single intersection in this vector, and we are building a single intersection type,
+    // but if a union is added to the intersection, we'll distribute ourselves over that union and
+    // create a union of intersections.
+    intersections: Vec<InnerIntersectionBuilder<'db>>,
+    db: &'db dyn Db,
+}
+
+impl<'db> IntersectionBuilder<'db> {
+    #[allow(dead_code)]
+    fn new(db: &'db dyn Db) -> Self {
+        Self {
+            db,
+            intersections: vec![InnerIntersectionBuilder::new()],
+        }
+    }
+
+    fn empty(db: &'db dyn Db) -> Self {
+        Self {
+            db,
+            intersections: vec![],
+        }
+    }
+
+    #[allow(dead_code)]
+    fn add_positive(mut self, ty: Type<'db>) -> Self {
+        if let Type::Union(union) = ty {
+            // Distribute ourself over this union: for each union element, clone ourself and
+            // intersect with that union element, then create a new union-of-intersections with all
+            // of those sub-intersections in it. E.g. if `self` is a simple intersection `T1 & T2`
+            // and we add `T3 | T4` to the intersection, we don't get `T1 & T2 & (T3 | T4)` (that's
+            // not in DNF), we distribute the union and get `(T1 & T3) | (T2 & T3) | (T1 & T4) |
+            // (T2 & T4)`. If `self` is already a union-of-intersections `(T1 & T2) | (T3 & T4)`
+            // and we add `T5 | T6` to it, that flattens all the way out to `(T1 & T2 & T5) | (T1 &
+            // T2 & T6) | (T3 & T4 & T5) ...` -- you get the idea.
+            union
+                .elements(self.db)
+                .iter()
+                .map(|elem| self.clone().add_positive(*elem))
+                .fold(IntersectionBuilder::empty(self.db), |mut builder, sub| {
+                    builder.intersections.extend(sub.intersections);
+                    builder
+                })
+        } else {
+            // If we are already a union-of-intersections, distribute the new intersected element
+            // across all of those intersections.
+            for inner in &mut self.intersections {
+                inner.add_positive(self.db, ty);
+            }
+            self
+        }
+    }
+
+    #[allow(dead_code)]
+    fn add_negative(mut self, ty: Type<'db>) -> Self {
+        // See comments above in `add_positive`; this is just the negated version.
+        if let Type::Union(union) = ty {
+            union
+                .elements(self.db)
+                .iter()
+                .map(|elem| self.clone().add_negative(*elem))
+                .fold(IntersectionBuilder::empty(self.db), |mut builder, sub| {
+                    builder.intersections.extend(sub.intersections);
+                    builder
+                })
+        } else {
+            for inner in &mut self.intersections {
+                inner.add_negative(self.db, ty);
+            }
+            self
+        }
+    }
+
+    #[allow(dead_code)]
+    fn build(mut self) -> Type<'db> {
+        // Avoid allocating the UnionBuilder unnecessarily if we have just one intersection:
+        if self.intersections.len() == 1 {
+            self.intersections.pop().unwrap().build(self.db)
+        } else {
+            let mut builder = UnionBuilder::new(self.db);
+            for inner in self.intersections {
+                builder = builder.add(inner.build(self.db));
+            }
+            builder.build()
+        }
+    }
+}
+
+#[allow(unused)]
+#[derive(Debug, Clone, Default)]
+struct InnerIntersectionBuilder<'db> {
+    positive: FxOrderSet<Type<'db>>,
+    negative: FxOrderSet<Type<'db>>,
+}
+
+impl<'db> InnerIntersectionBuilder<'db> {
+    fn new() -> Self {
+        Self::default()
+    }
+
+    /// Adds a positive type to this intersection.
+    fn add_positive(&mut self, db: &'db dyn Db, ty: Type<'db>) {
+        match ty {
+            Type::Intersection(inter) => {
+                let pos = inter.positive(db);
+                let neg = inter.negative(db);
+                self.positive.extend(pos.difference(&self.negative));
+                self.negative.extend(neg.difference(&self.positive));
+                self.positive.retain(|elem| !neg.contains(elem));
+                self.negative.retain(|elem| !pos.contains(elem));
+            }
+            _ => {
+                if !self.negative.remove(&ty) {
+                    self.positive.insert(ty);
+                };
+            }
+        }
+    }
+
+    /// Adds a negative type to this intersection.
+    fn add_negative(&mut self, db: &'db dyn Db, ty: Type<'db>) {
+        // TODO Any/Unknown actually should not self-cancel
+        match ty {
+            Type::Intersection(intersection) => {
+                let pos = intersection.negative(db);
+                let neg = intersection.positive(db);
+                self.positive.extend(pos.difference(&self.negative));
+                self.negative.extend(neg.difference(&self.positive));
+                self.positive.retain(|elem| !neg.contains(elem));
+                self.negative.retain(|elem| !pos.contains(elem));
+            }
+            Type::Never => {}
+            Type::Unbound => {}
+            _ => {
+                if !self.positive.remove(&ty) {
+                    self.negative.insert(ty);
+                };
+            }
+        }
+    }
+
+    fn simplify(&mut self) {
+        // TODO this should be generalized based on subtyping, for now we just handle a few cases
+
+        // Never is a subtype of all types
+        if self.positive.contains(&Type::Never) {
+            self.positive.retain(Type::is_never);
+            self.negative.clear();
+        }
+
+        if self.positive.contains(&Type::Unbound) {
+            self.positive.retain(Type::is_unbound);
+            self.negative.clear();
+        }
+    }
+
+    fn build(mut self, db: &'db dyn Db) -> Type<'db> {
+        self.simplify();
+        match (self.positive.len(), self.negative.len()) {
+            (0, 0) => Type::Never,
+            (1, 0) => self.positive[0],
+            _ => {
+                self.positive.shrink_to_fit();
+                self.negative.shrink_to_fit();
+                Type::Intersection(IntersectionType::new(db, self.positive, self.negative))
+            }
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::{IntersectionBuilder, IntersectionType, Type, UnionBuilder, UnionType};
+    use crate::db::tests::TestDb;
+
+    fn setup_db() -> TestDb {
+        TestDb::new()
+    }
+
+    impl<'db> UnionType<'db> {
+        fn elements_vec(self, db: &'db TestDb) -> Vec<Type<'db>> {
+            self.elements(db).into_iter().collect()
+        }
+    }
+
+    #[test]
+    fn build_union() {
+        let db = setup_db();
+        let t0 = Type::IntLiteral(0);
+        let t1 = Type::IntLiteral(1);
+        let Type::Union(union) = UnionBuilder::new(&db).add(t0).add(t1).build() else {
+            panic!("expected a union");
+        };
+
+        assert_eq!(union.elements_vec(&db), &[t0, t1]);
+    }
+
+    #[test]
+    fn build_union_single() {
+        let db = setup_db();
+        let t0 = Type::IntLiteral(0);
+        let ty = UnionBuilder::new(&db).add(t0).build();
+
+        assert_eq!(ty, t0);
+    }
+
+    #[test]
+    fn build_union_empty() {
+        let db = setup_db();
+        let ty = UnionBuilder::new(&db).build();
+
+        assert_eq!(ty, Type::Never);
+    }
+
+    #[test]
+    fn build_union_never() {
+        let db = setup_db();
+        let t0 = Type::IntLiteral(0);
+        let ty = UnionBuilder::new(&db).add(t0).add(Type::Never).build();
+
+        assert_eq!(ty, t0);
+    }
+
+    #[test]
+    fn build_union_flatten() {
+        let db = setup_db();
+        let t0 = Type::IntLiteral(0);
+        let t1 = Type::IntLiteral(1);
+        let t2 = Type::IntLiteral(2);
+        let u1 = UnionBuilder::new(&db).add(t0).add(t1).build();
+        let Type::Union(union) = UnionBuilder::new(&db).add(u1).add(t2).build() else {
+            panic!("expected a union");
+        };
+
+        assert_eq!(union.elements_vec(&db), &[t0, t1, t2]);
+    }
+
+    impl<'db> IntersectionType<'db> {
+        fn pos_vec(self, db: &'db TestDb) -> Vec<Type<'db>> {
+            self.positive(db).into_iter().collect()
+        }
+
+        fn neg_vec(self, db: &'db TestDb) -> Vec<Type<'db>> {
+            self.negative(db).into_iter().collect()
+        }
+    }
+
+    #[test]
+    fn build_intersection() {
+        let db = setup_db();
+        let t0 = Type::IntLiteral(0);
+        let ta = Type::Any;
+        let Type::Intersection(inter) = IntersectionBuilder::new(&db)
+            .add_positive(ta)
+            .add_negative(t0)
+            .build()
+        else {
+            panic!("expected to be an intersection");
+        };
+
+        assert_eq!(inter.pos_vec(&db), &[ta]);
+        assert_eq!(inter.neg_vec(&db), &[t0]);
+    }
+
+    #[test]
+    fn build_intersection_flatten_positive() {
+        let db = setup_db();
+        let ta = Type::Any;
+        let t1 = Type::IntLiteral(1);
+        let t2 = Type::IntLiteral(2);
+        let i0 = IntersectionBuilder::new(&db)
+            .add_positive(ta)
+            .add_negative(t1)
+            .build();
+        let Type::Intersection(inter) = IntersectionBuilder::new(&db)
+            .add_positive(t2)
+            .add_positive(i0)
+            .build()
+        else {
+            panic!("expected to be an intersection");
+        };
+
+        assert_eq!(inter.pos_vec(&db), &[t2, ta]);
+        assert_eq!(inter.neg_vec(&db), &[t1]);
+    }
+
+    #[test]
+    fn build_intersection_flatten_negative() {
+        let db = setup_db();
+        let ta = Type::Any;
+        let t1 = Type::IntLiteral(1);
+        let t2 = Type::IntLiteral(2);
+        let i0 = IntersectionBuilder::new(&db)
+            .add_positive(ta)
+            .add_negative(t1)
+            .build();
+        let Type::Intersection(inter) = IntersectionBuilder::new(&db)
+            .add_positive(t2)
+            .add_negative(i0)
+            .build()
+        else {
+            panic!("expected to be an intersection");
+        };
+
+        assert_eq!(inter.pos_vec(&db), &[t2, t1]);
+        assert_eq!(inter.neg_vec(&db), &[ta]);
+    }
+
+    #[test]
+    fn intersection_distributes_over_union() {
+        let db = setup_db();
+        let t0 = Type::IntLiteral(0);
+        let t1 = Type::IntLiteral(1);
+        let ta = Type::Any;
+        let u0 = UnionBuilder::new(&db).add(t0).add(t1).build();
+
+        let Type::Union(union) = IntersectionBuilder::new(&db)
+            .add_positive(ta)
+            .add_positive(u0)
+            .build()
+        else {
+            panic!("expected a union");
+        };
+        let [Type::Intersection(i0), Type::Intersection(i1)] = union.elements_vec(&db)[..] else {
+            panic!("expected a union of two intersections");
+        };
+        assert_eq!(i0.pos_vec(&db), &[ta, t0]);
+        assert_eq!(i1.pos_vec(&db), &[ta, t1]);
+    }
+
+    #[test]
+    fn build_intersection_self_negation() {
+        let db = setup_db();
+        let ty = IntersectionBuilder::new(&db)
+            .add_positive(Type::None)
+            .add_negative(Type::None)
+            .build();
+
+        assert_eq!(ty, Type::Never);
+    }
+
+    #[test]
+    fn build_intersection_simplify_negative_never() {
+        let db = setup_db();
+        let ty = IntersectionBuilder::new(&db)
+            .add_positive(Type::None)
+            .add_negative(Type::Never)
+            .build();
+
+        assert_eq!(ty, Type::None);
+    }
+
+    #[test]
+    fn build_intersection_simplify_positive_never() {
+        let db = setup_db();
+        let ty = IntersectionBuilder::new(&db)
+            .add_positive(Type::None)
+            .add_positive(Type::Never)
+            .build();
+
+        assert_eq!(ty, Type::Never);
+    }
+
+    #[test]
+    fn build_intersection_simplify_positive_unbound() {
+        let db = setup_db();
+        let ty = IntersectionBuilder::new(&db)
+            .add_positive(Type::Unbound)
+            .add_positive(Type::IntLiteral(1))
+            .build();
+
+        assert_eq!(ty, Type::Unbound);
+    }
+
+    #[test]
+    fn build_intersection_simplify_negative_unbound() {
+        let db = setup_db();
+        let ty = IntersectionBuilder::new(&db)
+            .add_negative(Type::Unbound)
+            .add_positive(Type::IntLiteral(1))
+            .build();
+
+        assert_eq!(ty, Type::IntLiteral(1));
+    }
+}

crates/red_knot_python_semantic/src/types/infer.rs

@@ -20,6 +20,8 @@
 //!
 //! Inferring types at any of the three region granularities returns a [`TypeInference`], which
 //! holds types for every [`Definition`] and expression within the inferred region.
+use std::num::NonZeroU32;
+
 use rustc_hash::FxHashMap;
 use salsa;
 use salsa::plumbing::AsId;
@@ -31,7 +33,7 @@ use ruff_python_ast::{ExprContext, TypeParams};
 
 use crate::builtins::builtins_scope;
 use crate::module_name::ModuleName;
-use crate::module_resolver::resolve_module;
+use crate::module_resolver::{file_to_module, resolve_module};
 use crate::semantic_index::ast_ids::{HasScopedAstId, HasScopedUseId, ScopedExpressionId};
 use crate::semantic_index::definition::{Definition, DefinitionKind, DefinitionNodeKey};
 use crate::semantic_index::expression::Expression;
@@ -40,7 +42,7 @@ use crate::semantic_index::symbol::{FileScopeId, NodeWithScopeKind, NodeWithScop
 use crate::semantic_index::SemanticIndex;
 use crate::types::{
     builtins_symbol_ty_by_name, definitions_ty, global_symbol_ty_by_name, ClassType, FunctionType,
-    Name, Type, UnionTypeBuilder,
+    Name, Type, UnionBuilder,
 };
 use crate::Db;
 
@@ -61,7 +63,7 @@ pub(crate) fn infer_scope_types<'db>(db: &'db dyn Db, scope: ScopeId<'db>) -> Ty
     TypeInferenceBuilder::new(db, InferenceRegion::Scope(scope), index).finish()
 }
 
-/// Cycle recovery for [`infer_definition_types`]: for now, just [`Type::Unknown`]
+/// Cycle recovery for [`infer_definition_types()`]: for now, just [`Type::Unknown`]
 /// TODO fixpoint iteration
 fn infer_definition_types_cycle_recovery<'db>(
     _db: &'db dyn Db,
@@ -260,6 +262,18 @@ impl<'db> TypeInferenceBuilder<'db> {
             NodeWithScopeKind::FunctionTypeParameters(function) => {
                 self.infer_function_type_params(function.node());
             }
+            NodeWithScopeKind::ListComprehension(comprehension) => {
+                self.infer_list_comprehension_expression_scope(comprehension.node());
+            }
+            NodeWithScopeKind::SetComprehension(comprehension) => {
+                self.infer_set_comprehension_expression_scope(comprehension.node());
+            }
+            NodeWithScopeKind::DictComprehension(comprehension) => {
+                self.infer_dict_comprehension_expression_scope(comprehension.node());
+            }
+            NodeWithScopeKind::GeneratorExpression(generator) => {
+                self.infer_generator_expression_scope(generator.node());
+            }
         }
     }
 
@@ -288,6 +302,19 @@ impl<'db> TypeInferenceBuilder<'db> {
             DefinitionKind::NamedExpression(named_expression) => {
                 self.infer_named_expression_definition(named_expression.node(), definition);
             }
+            DefinitionKind::Comprehension(comprehension) => {
+                self.infer_comprehension_definition(
+                    comprehension.node(),
+                    comprehension.is_first(),
+                    definition,
+                );
+            }
+            DefinitionKind::Parameter(parameter) => {
+                self.infer_parameter_definition(parameter, definition);
+            }
+            DefinitionKind::ParameterWithDefault(parameter_with_default) => {
+                self.infer_parameter_with_default_definition(parameter_with_default, definition);
+            }
         }
     }
 
@@ -402,6 +429,13 @@ impl<'db> TypeInferenceBuilder<'db> {
             .map(|decorator| self.infer_decorator(decorator))
             .collect();
 
+        for default in parameters
+            .iter_non_variadic_params()
+            .filter_map(|param| param.default.as_deref())
+        {
+            self.infer_expression(default);
+        }
+
         // If there are type params, parameters and returns are evaluated in that scope.
         if type_params.is_none() {
             self.infer_parameters(parameters);
@@ -439,10 +473,12 @@ impl<'db> TypeInferenceBuilder<'db> {
         let ast::ParameterWithDefault {
             range: _,
             parameter,
-            default,
+            default: _,
         } = parameter_with_default;
-        self.infer_parameter(parameter);
-        self.infer_optional_expression(default.as_deref());
+
+        self.infer_optional_expression(parameter.annotation.as_deref());
+
+        self.infer_definition(parameter_with_default);
     }
 
     fn infer_parameter(&mut self, parameter: &ast::Parameter) {
@@ -451,7 +487,29 @@ impl<'db> TypeInferenceBuilder<'db> {
             name: _,
             annotation,
         } = parameter;
+
         self.infer_optional_expression(annotation.as_deref());
+
+        self.infer_definition(parameter);
+    }
+
+    fn infer_parameter_with_default_definition(
+        &mut self,
+        _parameter_with_default: &ast::ParameterWithDefault,
+        definition: Definition<'db>,
+    ) {
+        // TODO(dhruvmanila): Infer types from annotation or default expression
+        self.types.definitions.insert(definition, Type::Unknown);
+    }
+
+    fn infer_parameter_definition(
+        &mut self,
+        _parameter: &ast::Parameter,
+        definition: Definition<'db>,
+    ) {
+        // TODO(dhruvmanila): Annotation expression is resolved at the enclosing scope, infer the
+        // parameter type from there
+        self.types.definitions.insert(definition, Type::Unknown);
     }
 
     fn infer_class_definition_statement(&mut self, class: &ast::StmtClassDef) {
@@ -766,7 +824,7 @@ impl<'db> TypeInferenceBuilder<'db> {
             asname: _,
         } = alias;
 
-        let module_ty = self.module_ty_from_name(name);
+        let module_ty = self.module_ty_from_name(ModuleName::new(name));
         self.types.definitions.insert(definition, module_ty);
     }
 
@@ -804,27 +862,82 @@ impl<'db> TypeInferenceBuilder<'db> {
         self.infer_optional_expression(cause.as_deref());
     }
 
+    /// Given a `from .foo import bar` relative import, resolve the relative module
+    /// we're importing `bar` from into an absolute [`ModuleName`]
+    /// using the name of the module we're currently analyzing.
+    ///
+    /// - `level` is the number of dots at the beginning of the relative module name:
+    ///   - `from .foo.bar import baz` => `level == 1`
+    ///   - `from ...foo.bar import baz` => `level == 3`
+    /// - `tail` is the relative module name stripped of all leading dots:
+    ///   - `from .foo import bar` => `tail == "foo"`
+    ///   - `from ..foo.bar import baz` => `tail == "foo.bar"`
+    fn relative_module_name(&self, tail: Option<&str>, level: NonZeroU32) -> Option<ModuleName> {
+        let Some(module) = file_to_module(self.db, self.file) else {
+            tracing::debug!("Failed to resolve file {:?} to a module", self.file);
+            return None;
+        };
+        let mut level = level.get();
+        if module.kind().is_package() {
+            level -= 1;
+        }
+        let mut module_name = module.name().to_owned();
+        for _ in 0..level {
+            module_name = module_name.parent()?;
+        }
+        if let Some(tail) = tail {
+            if let Some(valid_tail) = ModuleName::new(tail) {
+                module_name.extend(&valid_tail);
+            } else {
+                tracing::debug!("Failed to resolve relative import due to invalid syntax");
+                return None;
+            }
+        }
+        Some(module_name)
+    }
+
     fn infer_import_from_definition(
         &mut self,
         import_from: &ast::StmtImportFrom,
         alias: &ast::Alias,
         definition: Definition<'db>,
     ) {
-        let ast::StmtImportFrom { module, .. } = import_from;
-        let module_ty = if let Some(module) = module {
-            self.module_ty_from_name(module)
+        // TODO:
+        // - Absolute `*` imports (`from collections import *`)
+        // - Relative `*` imports (`from ...foo import *`)
+        // - Submodule imports (`from collections import abc`,
+        //   where `abc` is a submodule of the `collections` package)
+        //
+        // For the last item, see the currently skipped tests
+        // `follow_relative_import_bare_to_module()` and
+        // `follow_nonexistent_import_bare_to_module()`.
+        let ast::StmtImportFrom { module, level, .. } = import_from;
+        tracing::trace!("Resolving imported object {alias:?} from statement {import_from:?}");
+        let module_name = if let Some(level) = NonZeroU32::new(*level) {
+            self.relative_module_name(module.as_deref(), level)
         } else {
-            // TODO support relative imports
-            Type::Unknown
+            let module_name = module
+                .as_ref()
+                .expect("Non-relative import should always have a non-None `module`!");
+            ModuleName::new(module_name)
         };
 
+        let module_ty = self.module_ty_from_name(module_name);
+
         let ast::Alias {
             range: _,
             name,
             asname: _,
         } = alias;
 
-        let ty = module_ty.member(self.db, &Name::new(&name.id));
+        // If a symbol is unbound in the module the symbol was originally defined in,
+        // when we're trying to import the symbol from that module into "our" module,
+        // the runtime error will occur immediately (rather than when the symbol is *used*,
+        // as would be the case for a symbol with type `Unbound`), so it's appropriate to
+        // think of the type of the imported symbol as `Unknown` rather than `Unbound`
+        let ty = module_ty
+            .member(self.db, &Name::new(&name.id))
+            .replace_unbound_with(self.db, Type::Unknown);
 
         self.types.definitions.insert(definition, ty);
     }
@@ -840,11 +953,10 @@ impl<'db> TypeInferenceBuilder<'db> {
         }
     }
 
-    fn module_ty_from_name(&self, name: &ast::Identifier) -> Type<'db> {
-        let module = ModuleName::new(&name.id).and_then(|name| resolve_module(self.db, name));
-        module
-            .map(|module| Type::Module(module.file()))
-            .unwrap_or(Type::Unbound)
+    fn module_ty_from_name(&self, module_name: Option<ModuleName>) -> Type<'db> {
+        module_name
+            .and_then(|module_name| resolve_module(self.db, module_name))
+            .map_or(Type::Unknown, |module| Type::Module(module.file()))
     }
 
     fn infer_decorator(&mut self, decorator: &ast::Decorator) -> Type<'db> {
@@ -923,7 +1035,6 @@ impl<'db> TypeInferenceBuilder<'db> {
         ty
     }
 
-    #[allow(clippy::unused_self)]
     fn infer_number_literal_expression(&mut self, literal: &ast::ExprNumberLiteral) -> Type<'db> {
         let ast::ExprNumberLiteral { range: _, value } = literal;
 
@@ -1054,18 +1165,24 @@ impl<'db> TypeInferenceBuilder<'db> {
         builtins_symbol_ty_by_name(self.db, "dict").instance()
     }
 
+    /// Infer the type of the `iter` expression of the first comprehension.
+    fn infer_first_comprehension_iter(&mut self, comprehensions: &[ast::Comprehension]) {
+        let mut generators_iter = comprehensions.iter();
+        let Some(first_generator) = generators_iter.next() else {
+            unreachable!("Comprehension must contain at least one generator");
+        };
+        self.infer_expression(&first_generator.iter);
+    }
+
     fn infer_generator_expression(&mut self, generator: &ast::ExprGenerator) -> Type<'db> {
         let ast::ExprGenerator {
             range: _,
-            elt,
+            elt: _,
             generators,
             parenthesized: _,
         } = generator;
 
-        self.infer_expression(elt);
-        for generator in generators {
-            self.infer_comprehension(generator);
-        }
+        self.infer_first_comprehension_iter(generators);
 
         // TODO generator type
         Type::Unknown
@@ -1074,20 +1191,71 @@ impl<'db> TypeInferenceBuilder<'db> {
     fn infer_list_comprehension_expression(&mut self, listcomp: &ast::ExprListComp) -> Type<'db> {
         let ast::ExprListComp {
             range: _,
-            elt,
+            elt: _,
             generators,
         } = listcomp;
 
-        self.infer_expression(elt);
-        for generator in generators {
-            self.infer_comprehension(generator);
-        }
+        self.infer_first_comprehension_iter(generators);
 
         // TODO list type
         Type::Unknown
     }
 
     fn infer_dict_comprehension_expression(&mut self, dictcomp: &ast::ExprDictComp) -> Type<'db> {
+        let ast::ExprDictComp {
+            range: _,
+            key: _,
+            value: _,
+            generators,
+        } = dictcomp;
+
+        self.infer_first_comprehension_iter(generators);
+
+        // TODO dict type
+        Type::Unknown
+    }
+
+    fn infer_set_comprehension_expression(&mut self, setcomp: &ast::ExprSetComp) -> Type<'db> {
+        let ast::ExprSetComp {
+            range: _,
+            elt: _,
+            generators,
+        } = setcomp;
+
+        self.infer_first_comprehension_iter(generators);
+
+        // TODO set type
+        Type::Unknown
+    }
+
+    fn infer_generator_expression_scope(&mut self, generator: &ast::ExprGenerator) {
+        let ast::ExprGenerator {
+            range: _,
+            elt,
+            generators,
+            parenthesized: _,
+        } = generator;
+
+        self.infer_expression(elt);
+        for comprehension in generators {
+            self.infer_comprehension(comprehension);
+        }
+    }
+
+    fn infer_list_comprehension_expression_scope(&mut self, listcomp: &ast::ExprListComp) {
+        let ast::ExprListComp {
+            range: _,
+            elt,
+            generators,
+        } = listcomp;
+
+        self.infer_expression(elt);
+        for comprehension in generators {
+            self.infer_comprehension(comprehension);
+        }
+    }
+
+    fn infer_dict_comprehension_expression_scope(&mut self, dictcomp: &ast::ExprDictComp) {
         let ast::ExprDictComp {
             range: _,
             key,
@@ -1097,46 +1265,51 @@ impl<'db> TypeInferenceBuilder<'db> {
 
         self.infer_expression(key);
         self.infer_expression(value);
-        for generator in generators {
-            self.infer_comprehension(generator);
+        for comprehension in generators {
+            self.infer_comprehension(comprehension);
         }
-
-        // TODO dict type
-        Type::Unknown
     }
 
-    fn infer_set_comprehension_expression(&mut self, setcomp: &ast::ExprSetComp) -> Type<'db> {
+    fn infer_set_comprehension_expression_scope(&mut self, setcomp: &ast::ExprSetComp) {
         let ast::ExprSetComp {
             range: _,
             elt,
             generators,
         } = setcomp;
-        self.infer_expression(elt);
-        for generator in generators {
-            self.infer_comprehension(generator);
-        }
 
-        // TODO set type
-        Type::Unknown
+        self.infer_expression(elt);
+        for comprehension in generators {
+            self.infer_comprehension(comprehension);
+        }
     }
 
-    fn infer_comprehension(&mut self, comprehension: &ast::Comprehension) -> Type<'db> {
+    fn infer_comprehension(&mut self, comprehension: &ast::Comprehension) {
+        self.infer_definition(comprehension);
+        for expr in &comprehension.ifs {
+            self.infer_expression(expr);
+        }
+    }
+
+    fn infer_comprehension_definition(
+        &mut self,
+        comprehension: &ast::Comprehension,
+        is_first: bool,
+        definition: Definition<'db>,
+    ) {
         let ast::Comprehension {
             range: _,
             target,
             iter,
-            ifs,
+            ifs: _,
             is_async: _,
         } = comprehension;
 
-        self.infer_expression(target);
-        self.infer_expression(iter);
-        for if_clause in ifs {
-            self.infer_expression(if_clause);
+        if !is_first {
+            self.infer_expression(iter);
         }
-
-        // TODO comprehension type
-        Type::Unknown
+        // TODO(dhruvmanila): The target type should be inferred based on the iter type instead.
+        let target_ty = self.infer_expression(target);
+        self.types.definitions.insert(definition, target_ty);
     }
 
     fn infer_named_expression(&mut self, named: &ast::ExprNamed) -> Type<'db> {
@@ -1179,12 +1352,10 @@ impl<'db> TypeInferenceBuilder<'db> {
         let body_ty = self.infer_expression(body);
         let orelse_ty = self.infer_expression(orelse);
 
-        let union = UnionTypeBuilder::new(self.db)
+        UnionBuilder::new(self.db)
             .add(body_ty)
             .add(orelse_ty)
-            .build();
-
-        Type::Union(union)
+            .build()
     }
 
     fn infer_lambda_body(&mut self, lambda_expression: &ast::ExprLambda) {
@@ -1199,6 +1370,13 @@ impl<'db> TypeInferenceBuilder<'db> {
         } = lambda_expression;
 
         if let Some(parameters) = parameters {
+            for default in parameters
+                .iter_non_variadic_params()
+                .filter_map(|param| param.default.as_deref())
+            {
+                self.infer_expression(default);
+            }
+
             self.infer_parameters(parameters);
         }
 
@@ -1276,18 +1454,22 @@ impl<'db> TypeInferenceBuilder<'db> {
                     let symbol = symbols.symbol_by_name(id).unwrap();
                     if !symbol.is_defined() || !self.scope.is_function_like(self.db) {
                         // implicit global
-                        let mut unbound_ty = if file_scope_id == FileScopeId::global() {
+                        let unbound_ty = if file_scope_id == FileScopeId::global() {
                             Type::Unbound
                         } else {
                             global_symbol_ty_by_name(self.db, self.file, id)
                         };
                         // fallback to builtins
-                        if matches!(unbound_ty, Type::Unbound)
+                        if unbound_ty.may_be_unbound(self.db)
                             && Some(self.scope) != builtins_scope(self.db)
                         {
-                            unbound_ty = builtins_symbol_ty_by_name(self.db, id);
+                            Some(unbound_ty.replace_unbound_with(
+                                self.db,
+                                builtins_symbol_ty_by_name(self.db, id),
+                            ))
+                        } else {
+                            Some(unbound_ty)
                         }
-                        Some(unbound_ty)
                     } else {
                         Some(Type::Unbound)
                     }
@@ -1494,53 +1676,73 @@ impl<'db> TypeInferenceBuilder<'db> {
 
 #[cfg(test)]
 mod tests {
+    use anyhow::Context;
     use ruff_db::files::{system_path_to_file, File};
     use ruff_db::parsed::parsed_module;
-    use ruff_db::program::{Program, SearchPathSettings, TargetVersion};
     use ruff_db::system::{DbWithTestSystem, SystemPathBuf};
     use ruff_db::testing::assert_function_query_was_not_run;
     use ruff_python_ast::name::Name;
 
     use crate::builtins::builtins_scope;
     use crate::db::tests::TestDb;
+    use crate::program::{Program, SearchPathSettings};
+    use crate::python_version::PythonVersion;
     use crate::semantic_index::definition::Definition;
     use crate::semantic_index::symbol::FileScopeId;
     use crate::semantic_index::{global_scope, semantic_index, symbol_table, use_def_map};
     use crate::types::{global_symbol_ty_by_name, infer_definition_types, symbol_ty_by_name, Type};
-    use crate::{HasTy, SemanticModel};
+    use crate::{HasTy, ProgramSettings, SemanticModel};
 
     fn setup_db() -> TestDb {
         let db = TestDb::new();
 
-        Program::new(
+        let src_root = SystemPathBuf::from("/src");
+        db.memory_file_system()
+            .create_directory_all(&src_root)
+            .unwrap();
+
+        Program::from_settings(
             &db,
-            TargetVersion::Py38,
-            SearchPathSettings {
-                extra_paths: Vec::new(),
-                src_root: SystemPathBuf::from("/src"),
-                site_packages: vec![],
-                custom_typeshed: None,
+            ProgramSettings {
+                target_version: PythonVersion::default(),
+                search_paths: SearchPathSettings {
+                    extra_paths: Vec::new(),
+                    src_root,
+                    site_packages: vec![],
+                    custom_typeshed: None,
+                },
             },
-        );
+        )
+        .expect("Valid search path settings");
 
         db
     }
 
-    fn setup_db_with_custom_typeshed(typeshed: &str) -> TestDb {
-        let db = TestDb::new();
+    fn setup_db_with_custom_typeshed<'a>(
+        typeshed: &str,
+        files: impl IntoIterator<Item = (&'a str, &'a str)>,
+    ) -> anyhow::Result<TestDb> {
+        let mut db = TestDb::new();
+        let src_root = SystemPathBuf::from("/src");
 
-        Program::new(
+        db.write_files(files)
+            .context("Failed to write test files")?;
+
+        Program::from_settings(
             &db,
-            TargetVersion::Py38,
-            SearchPathSettings {
-                extra_paths: Vec::new(),
-                src_root: SystemPathBuf::from("/src"),
-                site_packages: vec![],
-                custom_typeshed: Some(SystemPathBuf::from(typeshed)),
+            ProgramSettings {
+                target_version: PythonVersion::default(),
+                search_paths: SearchPathSettings {
+                    extra_paths: Vec::new(),
+                    src_root,
+                    site_packages: vec![],
+                    custom_typeshed: Some(SystemPathBuf::from(typeshed)),
+                },
             },
-        );
+        )
+        .context("Failed to create Program")?;
 
-        db
+        Ok(db)
     }
 
     fn assert_public_ty(db: &TestDb, file_name: &str, symbol_name: &str, expected: &str) {
@@ -1564,6 +1766,166 @@ mod tests {
         Ok(())
     }
 
+    #[test]
+    fn follow_relative_import_simple() -> anyhow::Result<()> {
+        let mut db = setup_db();
+
+        db.write_files([
+            ("src/package/__init__.py", ""),
+            ("src/package/foo.py", "X = 42"),
+            ("src/package/bar.py", "from .foo import X"),
+        ])?;
+
+        assert_public_ty(&db, "src/package/bar.py", "X", "Literal[42]");
+
+        Ok(())
+    }
+
+    #[test]
+    fn follow_nonexistent_relative_import_simple() -> anyhow::Result<()> {
+        let mut db = setup_db();
+
+        db.write_files([
+            ("src/package/__init__.py", ""),
+            ("src/package/bar.py", "from .foo import X"),
+        ])?;
+
+        assert_public_ty(&db, "src/package/bar.py", "X", "Unknown");
+
+        Ok(())
+    }
+
+    #[test]
+    fn follow_relative_import_dotted() -> anyhow::Result<()> {
+        let mut db = setup_db();
+
+        db.write_files([
+            ("src/package/__init__.py", ""),
+            ("src/package/foo/bar/baz.py", "X = 42"),
+            ("src/package/bar.py", "from .foo.bar.baz import X"),
+        ])?;
+
+        assert_public_ty(&db, "src/package/bar.py", "X", "Literal[42]");
+
+        Ok(())
+    }
+
+    #[test]
+    fn follow_relative_import_bare_to_package() -> anyhow::Result<()> {
+        let mut db = setup_db();
+
+        db.write_files([
+            ("src/package/__init__.py", "X = 42"),
+            ("src/package/bar.py", "from . import X"),
+        ])?;
+
+        assert_public_ty(&db, "src/package/bar.py", "X", "Literal[42]");
+
+        Ok(())
+    }
+
+    #[test]
+    fn follow_nonexistent_relative_import_bare_to_package() -> anyhow::Result<()> {
+        let mut db = setup_db();
+        db.write_files([("src/package/bar.py", "from . import X")])?;
+        assert_public_ty(&db, "src/package/bar.py", "X", "Unknown");
+        Ok(())
+    }
+
+    #[ignore = "TODO: Submodule imports possibly not supported right now?"]
+    #[test]
+    fn follow_relative_import_bare_to_module() -> anyhow::Result<()> {
+        let mut db = setup_db();
+
+        db.write_files([
+            ("src/package/__init__.py", ""),
+            ("src/package/foo.py", "X = 42"),
+            ("src/package/bar.py", "from . import foo; y = foo.X"),
+        ])?;
+
+        assert_public_ty(&db, "src/package/bar.py", "y", "Literal[42]");
+
+        Ok(())
+    }
+
+    #[ignore = "TODO: Submodule imports possibly not supported right now?"]
+    #[test]
+    fn follow_nonexistent_import_bare_to_module() -> anyhow::Result<()> {
+        let mut db = setup_db();
+
+        db.write_files([
+            ("src/package/__init__.py", ""),
+            ("src/package/bar.py", "from . import foo"),
+        ])?;
+
+        assert_public_ty(&db, "src/package/bar.py", "foo", "Unknown");
+
+        Ok(())
+    }
+
+    #[test]
+    fn follow_relative_import_from_dunder_init() -> anyhow::Result<()> {
+        let mut db = setup_db();
+
+        db.write_files([
+            ("src/package/__init__.py", "from .foo import X"),
+            ("src/package/foo.py", "X = 42"),
+        ])?;
+
+        assert_public_ty(&db, "src/package/__init__.py", "X", "Literal[42]");
+
+        Ok(())
+    }
+
+    #[test]
+    fn follow_nonexistent_relative_import_from_dunder_init() -> anyhow::Result<()> {
+        let mut db = setup_db();
+        db.write_files([("src/package/__init__.py", "from .foo import X")])?;
+        assert_public_ty(&db, "src/package/__init__.py", "X", "Unknown");
+        Ok(())
+    }
+
+    #[test]
+    fn follow_very_relative_import() -> anyhow::Result<()> {
+        let mut db = setup_db();
+
+        db.write_files([
+            ("src/package/__init__.py", ""),
+            ("src/package/foo.py", "X = 42"),
+            (
+                "src/package/subpackage/subsubpackage/bar.py",
+                "from ...foo import X",
+            ),
+        ])?;
+
+        assert_public_ty(
+            &db,
+            "src/package/subpackage/subsubpackage/bar.py",
+            "X",
+            "Literal[42]",
+        );
+
+        Ok(())
+    }
+
+    #[test]
+    fn imported_unbound_symbol_is_unknown() -> anyhow::Result<()> {
+        let mut db = setup_db();
+
+        db.write_files([
+            ("src/package/__init__.py", ""),
+            ("src/package/foo.py", "x"),
+            ("src/package/bar.py", "from package.foo import x"),
+        ])?;
+
+        // the type as seen from external modules (`Unknown`)
+        // is different from the type inside the module itself (`Unbound`):
+        assert_public_ty(&db, "src/package/foo.py", "x", "Unbound");
+        assert_public_ty(&db, "src/package/bar.py", "x", "Unknown");
+
+        Ok(())
+    }
+
     #[test]
     fn resolve_base_class_by_name() -> anyhow::Result<()> {
         let mut db = setup_db();
@@ -2065,6 +2427,38 @@ mod tests {
         Ok(())
     }
 
+    #[test]
+    fn conditionally_global_or_builtin() -> anyhow::Result<()> {
+        let mut db = setup_db();
+
+        db.write_dedented(
+            "/src/a.py",
+            "
+            if flag:
+                copyright = 1
+            def f():
+                y = copyright
+            ",
+        )?;
+
+        let file = system_path_to_file(&db, "src/a.py").expect("Expected file to exist.");
+        let index = semantic_index(&db, file);
+        let function_scope = index
+            .child_scopes(FileScopeId::global())
+            .next()
+            .unwrap()
+            .0
+            .to_scope_id(&db, file);
+        let y_ty = symbol_ty_by_name(&db, function_scope, "y");
+
+        assert_eq!(
+            y_ty.display(&db).to_string(),
+            "Literal[1] | Literal[copyright]"
+        );
+
+        Ok(())
+    }
+
     /// Class name lookups do fall back to globals, but the public type never does.
     #[test]
     fn unbound_class_local() -> anyhow::Result<()> {
@@ -2130,16 +2524,17 @@ mod tests {
 
     #[test]
     fn builtin_symbol_custom_stdlib() -> anyhow::Result<()> {
-        let mut db = setup_db_with_custom_typeshed("/typeshed");
-
-        db.write_files([
-            ("/src/a.py", "c = copyright"),
-            (
-                "/typeshed/stdlib/builtins.pyi",
-                "def copyright() -> None: ...",
-            ),
-            ("/typeshed/stdlib/VERSIONS", "builtins: 3.8-"),
-        ])?;
+        let db = setup_db_with_custom_typeshed(
+            "/typeshed",
+            [
+                ("/src/a.py", "c = copyright"),
+                (
+                    "/typeshed/stdlib/builtins.pyi",
+                    "def copyright() -> None: ...",
+                ),
+                ("/typeshed/stdlib/VERSIONS", "builtins: 3.8-"),
+            ],
+        )?;
 
         assert_public_ty(&db, "/src/a.py", "c", "Literal[copyright]");
 
@@ -2159,13 +2554,14 @@ mod tests {
 
     #[test]
     fn unknown_builtin_later_defined() -> anyhow::Result<()> {
-        let mut db = setup_db_with_custom_typeshed("/typeshed");
-
-        db.write_files([
-            ("/src/a.py", "x = foo"),
-            ("/typeshed/stdlib/builtins.pyi", "foo = bar; bar = 1"),
-            ("/typeshed/stdlib/VERSIONS", "builtins: 3.8-"),
-        ])?;
+        let db = setup_db_with_custom_typeshed(
+            "/typeshed",
+            [
+                ("/src/a.py", "x = foo"),
+                ("/typeshed/stdlib/builtins.pyi", "foo = bar; bar = 1"),
+                ("/typeshed/stdlib/VERSIONS", "builtins: 3.8-"),
+            ],
+        )?;
 
         assert_public_ty(&db, "/src/a.py", "x", "Unbound");
 

crates/red_knot_python_semantic/vendor/typeshed/source_commit.txt

@@ -1 +1 @@
-4ef2d66663fc080fefa379e6ae5fc45d4f8b54eb
+1ace5718deaf3041f8e3d1dc9c9e8a8e830e517f

crates/red_knot_python_semantic/vendor/typeshed/stdlib/_ast.pyi

@@ -753,9 +753,11 @@ class Constant(expr):
         __match_args__ = ("value", "kind")
     value: Any  # None, str, bytes, bool, int, float, complex, Ellipsis
     kind: str | None
-    # Aliases for value, for backwards compatibility
-    s: Any
-    n: int | float | complex
+    if sys.version_info < (3, 14):
+        # Aliases for value, for backwards compatibility
+        s: Any
+        n: int | float | complex
+
     def __init__(self, value: Any, kind: str | None = None, **kwargs: Unpack[_Attributes]) -> None: ...
 
 class NamedExpr(expr):

crates/red_knot_python_semantic/vendor/typeshed/stdlib/_collections_abc.pyi

@@ -1,13 +1,12 @@
 import sys
 from abc import abstractmethod
 from types import MappingProxyType
-from typing import (  # noqa: Y022,Y038,Y057
+from typing import (  # noqa: Y022,Y038
     AbstractSet as Set,
     AsyncGenerator as AsyncGenerator,
     AsyncIterable as AsyncIterable,
     AsyncIterator as AsyncIterator,
     Awaitable as Awaitable,
-    ByteString as ByteString,
     Callable as Callable,
     Collection as Collection,
     Container as Container,
@@ -59,8 +58,12 @@ __all__ = [
     "ValuesView",
     "Sequence",
     "MutableSequence",
-    "ByteString",
 ]
+if sys.version_info < (3, 14):
+    from typing import ByteString as ByteString  # noqa: Y057
+
+    __all__ += ["ByteString"]
+
 if sys.version_info >= (3, 12):
     __all__ += ["Buffer"]
 

crates/red_knot_python_semantic/vendor/typeshed/stdlib/_ctypes.pyi

@@ -51,8 +51,8 @@ class _CDataMeta(type):
     # By default mypy complains about the following two methods, because strictly speaking cls
     # might not be a Type[_CT]. However this can never actually happen, because the only class that
     # uses _CDataMeta as its metaclass is _CData. So it's safe to ignore the errors here.
-    def __mul__(cls: type[_CT], other: int) -> type[Array[_CT]]: ...  # type: ignore[misc]
-    def __rmul__(cls: type[_CT], other: int) -> type[Array[_CT]]: ...  # type: ignore[misc]
+    def __mul__(cls: type[_CT], other: int) -> type[Array[_CT]]: ...  # type: ignore[misc]  # pyright: ignore[reportGeneralTypeIssues]
+    def __rmul__(cls: type[_CT], other: int) -> type[Array[_CT]]: ...  # type: ignore[misc]  # pyright: ignore[reportGeneralTypeIssues]
 
 class _CData(metaclass=_CDataMeta):
     _b_base_: int

crates/red_knot_python_semantic/vendor/typeshed/stdlib/argparse.pyi

@@ -357,7 +357,17 @@ class Action(_AttributeHolder):
 
 if sys.version_info >= (3, 12):
     class BooleanOptionalAction(Action):
-        if sys.version_info >= (3, 13):
+        if sys.version_info >= (3, 14):
+            def __init__(
+                self,
+                option_strings: Sequence[str],
+                dest: str,
+                default: bool | None = None,
+                required: bool = False,
+                help: str | None = None,
+                deprecated: bool = False,
+            ) -> None: ...
+        elif sys.version_info >= (3, 13):
             @overload
             def __init__(
                 self,

crates/red_knot_python_semantic/vendor/typeshed/stdlib/ast.pyi

@@ -10,27 +10,28 @@ class _ABC(type):
     if sys.version_info >= (3, 9):
         def __init__(cls, *args: Unused) -> None: ...
 
-@deprecated("Replaced by ast.Constant; removal scheduled for Python 3.14")
-class Num(Constant, metaclass=_ABC):
-    value: int | float | complex
+if sys.version_info < (3, 14):
+    @deprecated("Replaced by ast.Constant; removed in Python 3.14")
+    class Num(Constant, metaclass=_ABC):
+        value: int | float | complex
 
-@deprecated("Replaced by ast.Constant; removal scheduled for Python 3.14")
-class Str(Constant, metaclass=_ABC):
-    value: str
-    # Aliases for value, for backwards compatibility
-    s: str
+    @deprecated("Replaced by ast.Constant; removed in Python 3.14")
+    class Str(Constant, metaclass=_ABC):
+        value: str
+        # Aliases for value, for backwards compatibility
+        s: str
 
-@deprecated("Replaced by ast.Constant; removal scheduled for Python 3.14")
-class Bytes(Constant, metaclass=_ABC):
-    value: bytes
-    # Aliases for value, for backwards compatibility
-    s: bytes
+    @deprecated("Replaced by ast.Constant; removed in Python 3.14")
+    class Bytes(Constant, metaclass=_ABC):
+        value: bytes
+        # Aliases for value, for backwards compatibility
+        s: bytes
 
-@deprecated("Replaced by ast.Constant; removal scheduled for Python 3.14")
-class NameConstant(Constant, metaclass=_ABC): ...
+    @deprecated("Replaced by ast.Constant; removed in Python 3.14")
+    class NameConstant(Constant, metaclass=_ABC): ...
 
-@deprecated("Replaced by ast.Constant; removal scheduled for Python 3.14")
-class Ellipsis(Constant, metaclass=_ABC): ...
+    @deprecated("Replaced by ast.Constant; removed in Python 3.14")
+    class Ellipsis(Constant, metaclass=_ABC): ...
 
 if sys.version_info >= (3, 9):
     class slice(AST): ...

crates/red_knot_python_semantic/vendor/typeshed/stdlib/asyncio/tasks.pyi

@@ -151,13 +151,13 @@ if sys.version_info >= (3, 10):
     @overload
     def gather(*coros_or_futures: _FutureLike[_T], return_exceptions: Literal[False] = False) -> Future[list[_T]]: ...  # type: ignore[overload-overlap]
     @overload
-    def gather(coro_or_future1: _FutureLike[_T1], /, *, return_exceptions: bool) -> Future[tuple[_T1 | BaseException]]: ...  # type: ignore[overload-overlap]
+    def gather(coro_or_future1: _FutureLike[_T1], /, *, return_exceptions: bool) -> Future[tuple[_T1 | BaseException]]: ...
     @overload
-    def gather(  # type: ignore[overload-overlap]
+    def gather(
         coro_or_future1: _FutureLike[_T1], coro_or_future2: _FutureLike[_T2], /, *, return_exceptions: bool
     ) -> Future[tuple[_T1 | BaseException, _T2 | BaseException]]: ...
     @overload
-    def gather(  # type: ignore[overload-overlap]
+    def gather(
         coro_or_future1: _FutureLike[_T1],
         coro_or_future2: _FutureLike[_T2],
         coro_or_future3: _FutureLike[_T3],
@@ -166,7 +166,7 @@ if sys.version_info >= (3, 10):
         return_exceptions: bool,
     ) -> Future[tuple[_T1 | BaseException, _T2 | BaseException, _T3 | BaseException]]: ...
     @overload
-    def gather(  # type: ignore[overload-overlap]
+    def gather(
         coro_or_future1: _FutureLike[_T1],
         coro_or_future2: _FutureLike[_T2],
         coro_or_future3: _FutureLike[_T3],
@@ -176,7 +176,7 @@ if sys.version_info >= (3, 10):
         return_exceptions: bool,
     ) -> Future[tuple[_T1 | BaseException, _T2 | BaseException, _T3 | BaseException, _T4 | BaseException]]: ...
     @overload
-    def gather(  # type: ignore[overload-overlap]
+    def gather(
         coro_or_future1: _FutureLike[_T1],
         coro_or_future2: _FutureLike[_T2],
         coro_or_future3: _FutureLike[_T3],
@@ -189,7 +189,7 @@ if sys.version_info >= (3, 10):
         tuple[_T1 | BaseException, _T2 | BaseException, _T3 | BaseException, _T4 | BaseException, _T5 | BaseException]
     ]: ...
     @overload
-    def gather(  # type: ignore[overload-overlap]
+    def gather(
         coro_or_future1: _FutureLike[_T1],
         coro_or_future2: _FutureLike[_T2],
         coro_or_future3: _FutureLike[_T3],

crates/red_knot_python_semantic/vendor/typeshed/stdlib/asyncio/unix_events.pyi

@@ -159,7 +159,7 @@ if sys.platform != "win32":
 
     class _UnixSelectorEventLoop(BaseSelectorEventLoop):
         if sys.version_info >= (3, 13):
-            async def create_unix_server(  # type: ignore[override]
+            async def create_unix_server(
                 self,
                 protocol_factory: _ProtocolFactory,
                 path: StrPath | None = None,

crates/red_knot_python_semantic/vendor/typeshed/stdlib/builtins.pyi

@@ -1744,7 +1744,7 @@ _SupportsSumNoDefaultT = TypeVar("_SupportsSumNoDefaultT", bound=_SupportsSumWit
 # without creating many false-positive errors (see #7578).
 # Instead, we special-case the most common examples of this: bool and literal integers.
 @overload
-def sum(iterable: Iterable[bool | _LiteralInteger], /, start: int = 0) -> int: ...  # type: ignore[overload-overlap]
+def sum(iterable: Iterable[bool | _LiteralInteger], /, start: int = 0) -> int: ...
 @overload
 def sum(iterable: Iterable[_SupportsSumNoDefaultT], /) -> _SupportsSumNoDefaultT | Literal[0]: ...
 @overload
@@ -1752,9 +1752,8 @@ def sum(iterable: Iterable[_AddableT1], /, start: _AddableT2) -> _AddableT1 | _A
 
 # The argument to `vars()` has to have a `__dict__` attribute, so the second overload can't be annotated with `object`
 # (A "SupportsDunderDict" protocol doesn't work)
-# Use a type: ignore to make complaints about overlapping overloads go away
 @overload
-def vars(object: type, /) -> types.MappingProxyType[str, Any]: ...  # type: ignore[overload-overlap]
+def vars(object: type, /) -> types.MappingProxyType[str, Any]: ...
 @overload
 def vars(object: Any = ..., /) -> dict[str, Any]: ...
 

crates/red_knot_python_semantic/vendor/typeshed/stdlib/contextlib.pyi

@@ -55,6 +55,7 @@ class AbstractAsyncContextManager(Protocol[_T_co, _ExitT_co]):
     ) -> _ExitT_co: ...
 
 class ContextDecorator:
+    def _recreate_cm(self) -> Self: ...
     def __call__(self, func: _F) -> _F: ...
 
 class _GeneratorContextManager(AbstractContextManager[_T_co, bool | None], ContextDecorator):
@@ -80,6 +81,7 @@ if sys.version_info >= (3, 10):
     _AF = TypeVar("_AF", bound=Callable[..., Awaitable[Any]])
 
     class AsyncContextDecorator:
+        def _recreate_cm(self) -> Self: ...
         def __call__(self, func: _AF) -> _AF: ...
 
     class _AsyncGeneratorContextManager(AbstractAsyncContextManager[_T_co, bool | None], AsyncContextDecorator):

crates/red_knot_python_semantic/vendor/typeshed/stdlib/ctypes/_endian.pyi

@@ -1,12 +1,5 @@
 import sys
-from _ctypes import RTLD_GLOBAL as RTLD_GLOBAL, RTLD_LOCAL as RTLD_LOCAL, Structure, Union
-from ctypes import DEFAULT_MODE as DEFAULT_MODE, cdll as cdll, pydll as pydll, pythonapi as pythonapi
-
-if sys.version_info >= (3, 12):
-    from _ctypes import SIZEOF_TIME_T as SIZEOF_TIME_T
-
-if sys.platform == "win32":
-    from ctypes import oledll as oledll, windll as windll
+from ctypes import Structure, Union
 
 # At runtime, the native endianness is an alias for Structure,
 # while the other is a subclass with a metaclass added in.

crates/red_knot_python_semantic/vendor/typeshed/stdlib/dataclasses.pyi

@@ -5,7 +5,7 @@ from _typeshed import DataclassInstance
 from builtins import type as Type  # alias to avoid name clashes with fields named "type"
 from collections.abc import Callable, Iterable, Mapping
 from typing import Any, Generic, Literal, Protocol, TypeVar, overload
-from typing_extensions import TypeAlias, TypeIs
+from typing_extensions import Never, TypeAlias, TypeIs
 
 if sys.version_info >= (3, 9):
     from types import GenericAlias
@@ -213,6 +213,10 @@ else:
     ) -> Any: ...
 
 def fields(class_or_instance: DataclassInstance | type[DataclassInstance]) -> tuple[Field[Any], ...]: ...
+
+# HACK: `obj: Never` typing matches if object argument is using `Any` type.
+@overload
+def is_dataclass(obj: Never) -> TypeIs[DataclassInstance | type[DataclassInstance]]: ...  # type: ignore[narrowed-type-not-subtype]  # pyright: ignore[reportGeneralTypeIssues]
 @overload
 def is_dataclass(obj: type) -> TypeIs[type[DataclassInstance]]: ...
 @overload

crates/red_knot_python_semantic/vendor/typeshed/stdlib/distutils/cmd.pyi

@@ -1,6 +1,26 @@
 from _typeshed import BytesPath, Incomplete, StrOrBytesPath, StrPath, Unused
 from abc import abstractmethod
 from collections.abc import Callable, Iterable
+from distutils.command.bdist import bdist
+from distutils.command.bdist_dumb import bdist_dumb
+from distutils.command.bdist_rpm import bdist_rpm
+from distutils.command.build import build
+from distutils.command.build_clib import build_clib
+from distutils.command.build_ext import build_ext
+from distutils.command.build_py import build_py
+from distutils.command.build_scripts import build_scripts
+from distutils.command.check import check
+from distutils.command.clean import clean
+from distutils.command.config import config
+from distutils.command.install import install
+from distutils.command.install_data import install_data
+from distutils.command.install_egg_info import install_egg_info
+from distutils.command.install_headers import install_headers
+from distutils.command.install_lib import install_lib
+from distutils.command.install_scripts import install_scripts
+from distutils.command.register import register
+from distutils.command.sdist import sdist
+from distutils.command.upload import upload
 from distutils.dist import Distribution
 from distutils.file_util import _BytesPathT, _StrPathT
 from typing import Any, ClassVar, Literal, TypeVar, overload
@@ -28,8 +48,108 @@ class Command:
     def ensure_dirname(self, option: str) -> None: ...
     def get_command_name(self) -> str: ...
     def set_undefined_options(self, src_cmd: str, *option_pairs: tuple[str, str]) -> None: ...
+    # NOTE: This list comes directly from the distutils/command folder. Minus bdist_msi and bdist_wininst.
+    @overload
+    def get_finalized_command(self, command: Literal["bdist"], create: bool | Literal[0, 1] = 1) -> bdist: ...
+    @overload
+    def get_finalized_command(self, command: Literal["bdist_dumb"], create: bool | Literal[0, 1] = 1) -> bdist_dumb: ...
+    @overload
+    def get_finalized_command(self, command: Literal["bdist_rpm"], create: bool | Literal[0, 1] = 1) -> bdist_rpm: ...
+    @overload
+    def get_finalized_command(self, command: Literal["build"], create: bool | Literal[0, 1] = 1) -> build: ...
+    @overload
+    def get_finalized_command(self, command: Literal["build_clib"], create: bool | Literal[0, 1] = 1) -> build_clib: ...
+    @overload
+    def get_finalized_command(self, command: Literal["build_ext"], create: bool | Literal[0, 1] = 1) -> build_ext: ...
+    @overload
+    def get_finalized_command(self, command: Literal["build_py"], create: bool | Literal[0, 1] = 1) -> build_py: ...
+    @overload
+    def get_finalized_command(self, command: Literal["build_scripts"], create: bool | Literal[0, 1] = 1) -> build_scripts: ...
+    @overload
+    def get_finalized_command(self, command: Literal["check"], create: bool | Literal[0, 1] = 1) -> check: ...
+    @overload
+    def get_finalized_command(self, command: Literal["clean"], create: bool | Literal[0, 1] = 1) -> clean: ...
+    @overload
+    def get_finalized_command(self, command: Literal["config"], create: bool | Literal[0, 1] = 1) -> config: ...
+    @overload
+    def get_finalized_command(self, command: Literal["install"], create: bool | Literal[0, 1] = 1) -> install: ...
+    @overload
+    def get_finalized_command(self, command: Literal["install_data"], create: bool | Literal[0, 1] = 1) -> install_data: ...
+    @overload
+    def get_finalized_command(
+        self, command: Literal["install_egg_info"], create: bool | Literal[0, 1] = 1
+    ) -> install_egg_info: ...
+    @overload
+    def get_finalized_command(self, command: Literal["install_headers"], create: bool | Literal[0, 1] = 1) -> install_headers: ...
+    @overload
+    def get_finalized_command(self, command: Literal["install_lib"], create: bool | Literal[0, 1] = 1) -> install_lib: ...
+    @overload
+    def get_finalized_command(self, command: Literal["install_scripts"], create: bool | Literal[0, 1] = 1) -> install_scripts: ...
+    @overload
+    def get_finalized_command(self, command: Literal["register"], create: bool | Literal[0, 1] = 1) -> register: ...
+    @overload
+    def get_finalized_command(self, command: Literal["sdist"], create: bool | Literal[0, 1] = 1) -> sdist: ...
+    @overload
+    def get_finalized_command(self, command: Literal["upload"], create: bool | Literal[0, 1] = 1) -> upload: ...
+    @overload
     def get_finalized_command(self, command: str, create: bool | Literal[0, 1] = 1) -> Command: ...
     @overload
+    def reinitialize_command(self, command: Literal["bdist"], reinit_subcommands: bool | Literal[0, 1] = 0) -> bdist: ...
+    @overload
+    def reinitialize_command(
+        self, command: Literal["bdist_dumb"], reinit_subcommands: bool | Literal[0, 1] = 0
+    ) -> bdist_dumb: ...
+    @overload
+    def reinitialize_command(self, command: Literal["bdist_rpm"], reinit_subcommands: bool | Literal[0, 1] = 0) -> bdist_rpm: ...
+    @overload
+    def reinitialize_command(self, command: Literal["build"], reinit_subcommands: bool | Literal[0, 1] = 0) -> build: ...
+    @overload
+    def reinitialize_command(
+        self, command: Literal["build_clib"], reinit_subcommands: bool | Literal[0, 1] = 0
+    ) -> build_clib: ...
+    @overload
+    def reinitialize_command(self, command: Literal["build_ext"], reinit_subcommands: bool | Literal[0, 1] = 0) -> build_ext: ...
+    @overload
+    def reinitialize_command(self, command: Literal["build_py"], reinit_subcommands: bool | Literal[0, 1] = 0) -> build_py: ...
+    @overload
+    def reinitialize_command(
+        self, command: Literal["build_scripts"], reinit_subcommands: bool | Literal[0, 1] = 0
+    ) -> build_scripts: ...
+    @overload
+    def reinitialize_command(self, command: Literal["check"], reinit_subcommands: bool | Literal[0, 1] = 0) -> check: ...
+    @overload
+    def reinitialize_command(self, command: Literal["clean"], reinit_subcommands: bool | Literal[0, 1] = 0) -> clean: ...
+    @overload
+    def reinitialize_command(self, command: Literal["config"], reinit_subcommands: bool | Literal[0, 1] = 0) -> config: ...
+    @overload
+    def reinitialize_command(self, command: Literal["install"], reinit_subcommands: bool | Literal[0, 1] = 0) -> install: ...
+    @overload
+    def reinitialize_command(
+        self, command: Literal["install_data"], reinit_subcommands: bool | Literal[0, 1] = 0
+    ) -> install_data: ...
+    @overload
+    def reinitialize_command(
+        self, command: Literal["install_egg_info"], reinit_subcommands: bool | Literal[0, 1] = 0
+    ) -> install_egg_info: ...
+    @overload
+    def reinitialize_command(
+        self, command: Literal["install_headers"], reinit_subcommands: bool | Literal[0, 1] = 0
+    ) -> install_headers: ...
+    @overload
+    def reinitialize_command(
+        self, command: Literal["install_lib"], reinit_subcommands: bool | Literal[0, 1] = 0
+    ) -> install_lib: ...
+    @overload
+    def reinitialize_command(
+        self, command: Literal["install_scripts"], reinit_subcommands: bool | Literal[0, 1] = 0
+    ) -> install_scripts: ...
+    @overload
+    def reinitialize_command(self, command: Literal["register"], reinit_subcommands: bool | Literal[0, 1] = 0) -> register: ...
+    @overload
+    def reinitialize_command(self, command: Literal["sdist"], reinit_subcommands: bool | Literal[0, 1] = 0) -> sdist: ...
+    @overload
+    def reinitialize_command(self, command: Literal["upload"], reinit_subcommands: bool | Literal[0, 1] = 0) -> upload: ...
+    @overload
     def reinitialize_command(self, command: str, reinit_subcommands: bool | Literal[0, 1] = 0) -> Command: ...
     @overload
     def reinitialize_command(self, command: _CommandT, reinit_subcommands: bool | Literal[0, 1] = 0) -> _CommandT: ...

crates/red_knot_python_semantic/vendor/typeshed/stdlib/distutils/command/__init__.pyi

@@ -0,0 +1,48 @@
+import sys
+
+from . import (
+    bdist,
+    bdist_dumb,
+    bdist_rpm,
+    build,
+    build_clib,
+    build_ext,
+    build_py,
+    build_scripts,
+    check,
+    clean,
+    install,
+    install_data,
+    install_headers,
+    install_lib,
+    install_scripts,
+    register,
+    sdist,
+    upload,
+)
+
+__all__ = [
+    "build",
+    "build_py",
+    "build_ext",
+    "build_clib",
+    "build_scripts",
+    "clean",
+    "install",
+    "install_lib",
+    "install_headers",
+    "install_scripts",
+    "install_data",
+    "sdist",
+    "register",
+    "bdist",
+    "bdist_dumb",
+    "bdist_rpm",
+    "check",
+    "upload",
+]
+
+if sys.version_info < (3, 10):
+    from . import bdist_wininst
+
+    __all__ += ["bdist_wininst"]

crates/red_knot_python_semantic/vendor/typeshed/stdlib/distutils/dist.pyi

@@ -1,6 +1,26 @@
 from _typeshed import Incomplete, StrOrBytesPath, StrPath, SupportsWrite
 from collections.abc import Iterable, MutableMapping
 from distutils.cmd import Command
+from distutils.command.bdist import bdist
+from distutils.command.bdist_dumb import bdist_dumb
+from distutils.command.bdist_rpm import bdist_rpm
+from distutils.command.build import build
+from distutils.command.build_clib import build_clib
+from distutils.command.build_ext import build_ext
+from distutils.command.build_py import build_py
+from distutils.command.build_scripts import build_scripts
+from distutils.command.check import check
+from distutils.command.clean import clean
+from distutils.command.config import config
+from distutils.command.install import install
+from distutils.command.install_data import install_data
+from distutils.command.install_egg_info import install_egg_info
+from distutils.command.install_headers import install_headers
+from distutils.command.install_lib import install_lib
+from distutils.command.install_scripts import install_scripts
+from distutils.command.register import register
+from distutils.command.sdist import sdist
+from distutils.command.upload import upload
 from re import Pattern
 from typing import IO, ClassVar, Literal, TypeVar, overload
 from typing_extensions import TypeAlias
@@ -63,10 +83,6 @@ class Distribution:
     def __init__(self, attrs: MutableMapping[str, Incomplete] | None = None) -> None: ...
     def get_option_dict(self, command: str) -> dict[str, tuple[str, str]]: ...
     def parse_config_files(self, filenames: Iterable[str] | None = None) -> None: ...
-    @overload
-    def get_command_obj(self, command: str, create: Literal[1, True] = 1) -> Command: ...
-    @overload
-    def get_command_obj(self, command: str, create: Literal[0, False]) -> Command | None: ...
     global_options: ClassVar[_OptionsList]
     common_usage: ClassVar[str]
     display_options: ClassVar[_OptionsList]
@@ -108,8 +124,137 @@ class Distribution:
     def print_commands(self) -> None: ...
     def get_command_list(self): ...
     def get_command_packages(self): ...
+    # NOTE: This list comes directly from the distutils/command folder. Minus bdist_msi and bdist_wininst.
+    @overload
+    def get_command_obj(self, command: Literal["bdist"], create: Literal[1, True] = 1) -> bdist: ...
+    @overload
+    def get_command_obj(self, command: Literal["bdist_dumb"], create: Literal[1, True] = 1) -> bdist_dumb: ...
+    @overload
+    def get_command_obj(self, command: Literal["bdist_rpm"], create: Literal[1, True] = 1) -> bdist_rpm: ...
+    @overload
+    def get_command_obj(self, command: Literal["build"], create: Literal[1, True] = 1) -> build: ...
+    @overload
+    def get_command_obj(self, command: Literal["build_clib"], create: Literal[1, True] = 1) -> build_clib: ...
+    @overload
+    def get_command_obj(self, command: Literal["build_ext"], create: Literal[1, True] = 1) -> build_ext: ...
+    @overload
+    def get_command_obj(self, command: Literal["build_py"], create: Literal[1, True] = 1) -> build_py: ...
+    @overload
+    def get_command_obj(self, command: Literal["build_scripts"], create: Literal[1, True] = 1) -> build_scripts: ...
+    @overload
+    def get_command_obj(self, command: Literal["check"], create: Literal[1, True] = 1) -> check: ...
+    @overload
+    def get_command_obj(self, command: Literal["clean"], create: Literal[1, True] = 1) -> clean: ...
+    @overload
+    def get_command_obj(self, command: Literal["config"], create: Literal[1, True] = 1) -> config: ...
+    @overload
+    def get_command_obj(self, command: Literal["install"], create: Literal[1, True] = 1) -> install: ...
+    @overload
+    def get_command_obj(self, command: Literal["install_data"], create: Literal[1, True] = 1) -> install_data: ...
+    @overload
+    def get_command_obj(self, command: Literal["install_egg_info"], create: Literal[1, True] = 1) -> install_egg_info: ...
+    @overload
+    def get_command_obj(self, command: Literal["install_headers"], create: Literal[1, True] = 1) -> install_headers: ...
+    @overload
+    def get_command_obj(self, command: Literal["install_lib"], create: Literal[1, True] = 1) -> install_lib: ...
+    @overload
+    def get_command_obj(self, command: Literal["install_scripts"], create: Literal[1, True] = 1) -> install_scripts: ...
+    @overload
+    def get_command_obj(self, command: Literal["register"], create: Literal[1, True] = 1) -> register: ...
+    @overload
+    def get_command_obj(self, command: Literal["sdist"], create: Literal[1, True] = 1) -> sdist: ...
+    @overload
+    def get_command_obj(self, command: Literal["upload"], create: Literal[1, True] = 1) -> upload: ...
+    @overload
+    def get_command_obj(self, command: str, create: Literal[1, True] = 1) -> Command: ...
+    # Not replicating the overloads for "Command | None", user may use "isinstance"
+    @overload
+    def get_command_obj(self, command: str, create: Literal[0, False]) -> Command | None: ...
+    @overload
+    def get_command_class(self, command: Literal["bdist"]) -> type[bdist]: ...
+    @overload
+    def get_command_class(self, command: Literal["bdist_dumb"]) -> type[bdist_dumb]: ...
+    @overload
+    def get_command_class(self, command: Literal["bdist_rpm"]) -> type[bdist_rpm]: ...
+    @overload
+    def get_command_class(self, command: Literal["build"]) -> type[build]: ...
+    @overload
+    def get_command_class(self, command: Literal["build_clib"]) -> type[build_clib]: ...
+    @overload
+    def get_command_class(self, command: Literal["build_ext"]) -> type[build_ext]: ...
+    @overload
+    def get_command_class(self, command: Literal["build_py"]) -> type[build_py]: ...
+    @overload
+    def get_command_class(self, command: Literal["build_scripts"]) -> type[build_scripts]: ...
+    @overload
+    def get_command_class(self, command: Literal["check"]) -> type[check]: ...
+    @overload
+    def get_command_class(self, command: Literal["clean"]) -> type[clean]: ...
+    @overload
+    def get_command_class(self, command: Literal["config"]) -> type[config]: ...
+    @overload
+    def get_command_class(self, command: Literal["install"]) -> type[install]: ...
+    @overload
+    def get_command_class(self, command: Literal["install_data"]) -> type[install_data]: ...
+    @overload
+    def get_command_class(self, command: Literal["install_egg_info"]) -> type[install_egg_info]: ...
+    @overload
+    def get_command_class(self, command: Literal["install_headers"]) -> type[install_headers]: ...
+    @overload
+    def get_command_class(self, command: Literal["install_lib"]) -> type[install_lib]: ...
+    @overload
+    def get_command_class(self, command: Literal["install_scripts"]) -> type[install_scripts]: ...
+    @overload
+    def get_command_class(self, command: Literal["register"]) -> type[register]: ...
+    @overload
+    def get_command_class(self, command: Literal["sdist"]) -> type[sdist]: ...
+    @overload
+    def get_command_class(self, command: Literal["upload"]) -> type[upload]: ...
+    @overload
     def get_command_class(self, command: str) -> type[Command]: ...
     @overload
+    def reinitialize_command(self, command: Literal["bdist"], reinit_subcommands: bool = False) -> bdist: ...
+    @overload
+    def reinitialize_command(self, command: Literal["bdist_dumb"], reinit_subcommands: bool = False) -> bdist_dumb: ...
+    @overload
+    def reinitialize_command(self, command: Literal["bdist_rpm"], reinit_subcommands: bool = False) -> bdist_rpm: ...
+    @overload
+    def reinitialize_command(self, command: Literal["build"], reinit_subcommands: bool = False) -> build: ...
+    @overload
+    def reinitialize_command(self, command: Literal["build_clib"], reinit_subcommands: bool = False) -> build_clib: ...
+    @overload
+    def reinitialize_command(self, command: Literal["build_ext"], reinit_subcommands: bool = False) -> build_ext: ...
+    @overload
+    def reinitialize_command(self, command: Literal["build_py"], reinit_subcommands: bool = False) -> build_py: ...
+    @overload
+    def reinitialize_command(self, command: Literal["build_scripts"], reinit_subcommands: bool = False) -> build_scripts: ...
+    @overload
+    def reinitialize_command(self, command: Literal["check"], reinit_subcommands: bool = False) -> check: ...
+    @overload
+    def reinitialize_command(self, command: Literal["clean"], reinit_subcommands: bool = False) -> clean: ...
+    @overload
+    def reinitialize_command(self, command: Literal["config"], reinit_subcommands: bool = False) -> config: ...
+    @overload
+    def reinitialize_command(self, command: Literal["install"], reinit_subcommands: bool = False) -> install: ...
+    @overload
+    def reinitialize_command(self, command: Literal["install_data"], reinit_subcommands: bool = False) -> install_data: ...
+    @overload
+    def reinitialize_command(
+        self, command: Literal["install_egg_info"], reinit_subcommands: bool = False
+    ) -> install_egg_info: ...
+    @overload
+    def reinitialize_command(self, command: Literal["install_headers"], reinit_subcommands: bool = False) -> install_headers: ...
+    @overload
+    def reinitialize_command(self, command: Literal["install_lib"], reinit_subcommands: bool = False) -> install_lib: ...
+    @overload
+    def reinitialize_command(self, command: Literal["install_scripts"], reinit_subcommands: bool = False) -> install_scripts: ...
+    @overload
+    def reinitialize_command(self, command: Literal["register"], reinit_subcommands: bool = False) -> register: ...
+    @overload
+    def reinitialize_command(self, command: Literal["sdist"], reinit_subcommands: bool = False) -> sdist: ...
+    @overload
+    def reinitialize_command(self, command: Literal["upload"], reinit_subcommands: bool = False) -> upload: ...
+    @overload
     def reinitialize_command(self, command: str, reinit_subcommands: bool = False) -> Command: ...
     @overload
     def reinitialize_command(self, command: _CommandT, reinit_subcommands: bool = False) -> _CommandT: ...

crates/red_knot_python_semantic/vendor/typeshed/stdlib/email/utils.pyi

@@ -66,7 +66,10 @@ def mktime_tz(data: _PDTZ) -> int: ...
 def formatdate(timeval: float | None = None, localtime: bool = False, usegmt: bool = False) -> str: ...
 def format_datetime(dt: datetime.datetime, usegmt: bool = False) -> str: ...
 
-if sys.version_info >= (3, 12):
+if sys.version_info >= (3, 14):
+    def localtime(dt: datetime.datetime | None = None) -> datetime.datetime: ...
+
+elif sys.version_info >= (3, 12):
     @overload
     def localtime(dt: datetime.datetime | None = None) -> datetime.datetime: ...
     @overload

crates/red_knot_python_semantic/vendor/typeshed/stdlib/filecmp.pyi

@@ -17,13 +17,24 @@ def cmpfiles(
 ) -> tuple[list[AnyStr], list[AnyStr], list[AnyStr]]: ...
 
 class dircmp(Generic[AnyStr]):
-    def __init__(
-        self,
-        a: GenericPath[AnyStr],
-        b: GenericPath[AnyStr],
-        ignore: Sequence[AnyStr] | None = None,
-        hide: Sequence[AnyStr] | None = None,
-    ) -> None: ...
+    if sys.version_info >= (3, 13):
+        def __init__(
+            self,
+            a: GenericPath[AnyStr],
+            b: GenericPath[AnyStr],
+            ignore: Sequence[AnyStr] | None = None,
+            hide: Sequence[AnyStr] | None = None,
+            *,
+            shallow: bool = True,
+        ) -> None: ...
+    else:
+        def __init__(
+            self,
+            a: GenericPath[AnyStr],
+            b: GenericPath[AnyStr],
+            ignore: Sequence[AnyStr] | None = None,
+            hide: Sequence[AnyStr] | None = None,
+        ) -> None: ...
     left: AnyStr
     right: AnyStr
     hide: Sequence[AnyStr]

crates/red_knot_python_semantic/vendor/typeshed/stdlib/importlib/metadata/__init__.pyi

@@ -155,7 +155,7 @@ if sys.version_info >= (3, 10) and sys.version_info < (3, 12):
         @property
         def names(self) -> set[str]: ...
         @overload
-        def select(self) -> Self: ...  # type: ignore[misc]
+        def select(self) -> Self: ...
         @overload
         def select(
             self,
@@ -277,7 +277,7 @@ if sys.version_info >= (3, 12):
 
 elif sys.version_info >= (3, 10):
     @overload
-    def entry_points() -> SelectableGroups: ...  # type: ignore[overload-overlap]
+    def entry_points() -> SelectableGroups: ...
     @overload
     def entry_points(
         *, name: str = ..., value: str = ..., group: str = ..., module: str = ..., attr: str = ..., extras: list[str] = ...

crates/red_knot_python_semantic/vendor/typeshed/stdlib/lib2to3/fixes/fix_unicode.pyi

@@ -6,7 +6,7 @@ from ..pytree import Node
 
 class FixUnicode(fixer_base.BaseFix):
     BM_compatible: ClassVar[Literal[True]]
-    PATTERN: ClassVar[Literal["STRING | 'unicode' | 'unichr'"]]  # type: ignore[name-defined]  # Name "STRING" is not defined
+    PATTERN: ClassVar[str]
     unicode_literals: bool
     def start_tree(self, tree: Node, filename: StrPath) -> None: ...
     def transform(self, node, results): ...

crates/red_knot_python_semantic/vendor/typeshed/stdlib/logging/__init__.pyi

@@ -55,10 +55,9 @@ __all__ = [
     "setLogRecordFactory",
     "lastResort",
     "raiseExceptions",
+    "warn",
 ]
 
-if sys.version_info < (3, 13):
-    __all__ += ["warn"]
 if sys.version_info >= (3, 11):
     __all__ += ["getLevelNamesMapping"]
 if sys.version_info >= (3, 12):
@@ -157,17 +156,16 @@ class Logger(Filterer):
         stacklevel: int = 1,
         extra: Mapping[str, object] | None = None,
     ) -> None: ...
-    if sys.version_info < (3, 13):
-        def warn(
-            self,
-            msg: object,
-            *args: object,
-            exc_info: _ExcInfoType = None,
-            stack_info: bool = False,
-            stacklevel: int = 1,
-            extra: Mapping[str, object] | None = None,
-        ) -> None: ...
-
+    @deprecated("Deprecated; use warning() instead.")
+    def warn(
+        self,
+        msg: object,
+        *args: object,
+        exc_info: _ExcInfoType = None,
+        stack_info: bool = False,
+        stacklevel: int = 1,
+        extra: Mapping[str, object] | None = None,
+    ) -> None: ...
     def error(
         self,
         msg: object,
@@ -412,18 +410,17 @@ class LoggerAdapter(Generic[_L]):
         extra: Mapping[str, object] | None = None,
         **kwargs: object,
     ) -> None: ...
-    if sys.version_info < (3, 13):
-        def warn(
-            self,
-            msg: object,
-            *args: object,
-            exc_info: _ExcInfoType = None,
-            stack_info: bool = False,
-            stacklevel: int = 1,
-            extra: Mapping[str, object] | None = None,
-            **kwargs: object,
-        ) -> None: ...
-
+    @deprecated("Deprecated; use warning() instead.")
+    def warn(
+        self,
+        msg: object,
+        *args: object,
+        exc_info: _ExcInfoType = None,
+        stack_info: bool = False,
+        stacklevel: int = 1,
+        extra: Mapping[str, object] | None = None,
+        **kwargs: object,
+    ) -> None: ...
     def error(
         self,
         msg: object,
@@ -523,17 +520,15 @@ def warning(
     stacklevel: int = 1,
     extra: Mapping[str, object] | None = None,
 ) -> None: ...
-
-if sys.version_info < (3, 13):
-    def warn(
-        msg: object,
-        *args: object,
-        exc_info: _ExcInfoType = None,
-        stack_info: bool = False,
-        stacklevel: int = 1,
-        extra: Mapping[str, object] | None = None,
-    ) -> None: ...
-
+@deprecated("Deprecated; use warning() instead.")
+def warn(
+    msg: object,
+    *args: object,
+    exc_info: _ExcInfoType = None,
+    stack_info: bool = False,
+    stacklevel: int = 1,
+    extra: Mapping[str, object] | None = None,
+) -> None: ...
 def error(
     msg: object,
     *args: object,

crates/red_knot_python_semantic/vendor/typeshed/stdlib/multiprocessing/sharedctypes.pyi

@@ -73,7 +73,7 @@ def copy(obj: _CT) -> _CT: ...
 @overload
 def synchronized(obj: _SimpleCData[_T], lock: _LockLike | None = None, ctx: Any | None = None) -> Synchronized[_T]: ...
 @overload
-def synchronized(obj: ctypes.Array[c_char], lock: _LockLike | None = None, ctx: Any | None = None) -> SynchronizedString: ...  # type: ignore
+def synchronized(obj: ctypes.Array[c_char], lock: _LockLike | None = None, ctx: Any | None = None) -> SynchronizedString: ...
 @overload
 def synchronized(
     obj: ctypes.Array[_SimpleCData[_T]], lock: _LockLike | None = None, ctx: Any | None = None
@@ -115,12 +115,12 @@ class SynchronizedArray(SynchronizedBase[ctypes.Array[_SimpleCData[_T]]], Generi
 class SynchronizedString(SynchronizedArray[bytes]):
     @overload  # type: ignore[override]
     def __getitem__(self, i: slice) -> bytes: ...
-    @overload  # type: ignore[override]
+    @overload
     def __getitem__(self, i: int) -> bytes: ...
     @overload  # type: ignore[override]
     def __setitem__(self, i: slice, value: bytes) -> None: ...
-    @overload  # type: ignore[override]
-    def __setitem__(self, i: int, value: bytes) -> None: ...  # type: ignore[override]
+    @overload
+    def __setitem__(self, i: int, value: bytes) -> None: ...
     def __getslice__(self, start: int, stop: int) -> bytes: ...  # type: ignore[override]
     def __setslice__(self, start: int, stop: int, values: bytes) -> None: ...  # type: ignore[override]
 

crates/red_knot_python_semantic/vendor/typeshed/stdlib/pathlib.pyi

@@ -159,6 +159,20 @@ class Path(PurePath):
     def lchmod(self, mode: int) -> None: ...
     def lstat(self) -> stat_result: ...
     def mkdir(self, mode: int = 0o777, parents: bool = False, exist_ok: bool = False) -> None: ...
+
+    if sys.version_info >= (3, 14):
+        def copy(self, target: StrPath, *, follow_symlinks: bool = True, preserve_metadata: bool = False) -> None: ...
+        def copytree(
+            self,
+            target: StrPath,
+            *,
+            follow_symlinks: bool = True,
+            preserve_metadata: bool = False,
+            dirs_exist_ok: bool = False,
+            ignore: Callable[[Self], bool] | None = None,
+            on_error: Callable[[OSError], object] | None = None,
+        ) -> None: ...
+
     # Adapted from builtins.open
     # Text mode: always returns a TextIOWrapper
     # The Traversable .open in stdlib/importlib/abc.pyi should be kept in sync with this.
@@ -232,10 +246,18 @@ class Path(PurePath):
     if sys.version_info >= (3, 9):
         def readlink(self) -> Self: ...
 
-    def rename(self, target: str | PurePath) -> Self: ...
-    def replace(self, target: str | PurePath) -> Self: ...
+    if sys.version_info >= (3, 10):
+        def rename(self, target: StrPath) -> Self: ...
+        def replace(self, target: StrPath) -> Self: ...
+    else:
+        def rename(self, target: str | PurePath) -> Self: ...
+        def replace(self, target: str | PurePath) -> Self: ...
+
     def resolve(self, strict: bool = False) -> Self: ...
     def rmdir(self) -> None: ...
+    if sys.version_info >= (3, 14):
+        def delete(self, ignore_errors: bool = False, on_error: Callable[[OSError], object] | None = None) -> None: ...
+
     def symlink_to(self, target: StrOrBytesPath, target_is_directory: bool = False) -> None: ...
     if sys.version_info >= (3, 10):
         def hardlink_to(self, target: StrOrBytesPath) -> None: ...
@@ -266,6 +288,9 @@ class Path(PurePath):
             self, top_down: bool = ..., on_error: Callable[[OSError], object] | None = ..., follow_symlinks: bool = ...
         ) -> Iterator[tuple[Self, list[str], list[str]]]: ...
 
+    if sys.version_info >= (3, 14):
+        def rmtree(self, ignore_errors: bool = False, on_error: Callable[[OSError], object] | None = None) -> None: ...
+
 class PosixPath(Path, PurePosixPath): ...
 class WindowsPath(Path, PureWindowsPath): ...
 

crates/red_knot_python_semantic/vendor/typeshed/stdlib/pdb.pyi

@@ -84,7 +84,7 @@ class Pdb(Bdb, Cmd):
         def _runscript(self, filename: str) -> None: ...
 
     if sys.version_info >= (3, 13):
-        def completedefault(self, text: str, line: str, begidx: int, endidx: int) -> list[str]: ...  # type: ignore[override]
+        def completedefault(self, text: str, line: str, begidx: int, endidx: int) -> list[str]: ...
 
     def do_commands(self, arg: str) -> bool | None: ...
     def do_break(self, arg: str, temporary: bool = ...) -> bool | None: ...

crates/red_knot_python_semantic/vendor/typeshed/stdlib/pty.pyi

@@ -1,7 +1,7 @@
 import sys
 from collections.abc import Callable, Iterable
 from typing import Final
-from typing_extensions import TypeAlias
+from typing_extensions import TypeAlias, deprecated
 
 if sys.platform != "win32":
     __all__ = ["openpty", "fork", "spawn"]
@@ -13,7 +13,12 @@ if sys.platform != "win32":
 
     CHILD: Final = 0
     def openpty() -> tuple[int, int]: ...
-    def master_open() -> tuple[int, str]: ...  # deprecated, use openpty()
-    def slave_open(tty_name: str) -> int: ...  # deprecated, use openpty()
+
+    if sys.version_info < (3, 14):
+        @deprecated("Deprecated in 3.12, to be removed in 3.14; use openpty() instead")
+        def master_open() -> tuple[int, str]: ...
+        @deprecated("Deprecated in 3.12, to be removed in 3.14; use openpty() instead")
+        def slave_open(tty_name: str) -> int: ...
+
     def fork() -> tuple[int, int]: ...
     def spawn(argv: str | Iterable[str], master_read: _Reader = ..., stdin_read: _Reader = ...) -> int: ...

crates/red_knot_python_semantic/vendor/typeshed/stdlib/re.pyi

@@ -74,7 +74,7 @@ class Match(Generic[AnyStr]):
     @overload
     def expand(self: Match[str], template: str) -> str: ...
     @overload
-    def expand(self: Match[bytes], template: ReadableBuffer) -> bytes: ...  # type: ignore[overload-overlap]
+    def expand(self: Match[bytes], template: ReadableBuffer) -> bytes: ...
     @overload
     def expand(self, template: AnyStr) -> AnyStr: ...
     # group() returns "AnyStr" or "AnyStr | None", depending on the pattern.
@@ -124,19 +124,21 @@ class Pattern(Generic[AnyStr]):
     @overload
     def search(self: Pattern[str], string: str, pos: int = 0, endpos: int = sys.maxsize) -> Match[str] | None: ...
     @overload
-    def search(self: Pattern[bytes], string: ReadableBuffer, pos: int = 0, endpos: int = sys.maxsize) -> Match[bytes] | None: ...  # type: ignore[overload-overlap]
+    def search(self: Pattern[bytes], string: ReadableBuffer, pos: int = 0, endpos: int = sys.maxsize) -> Match[bytes] | None: ...
     @overload
     def search(self, string: AnyStr, pos: int = 0, endpos: int = sys.maxsize) -> Match[AnyStr] | None: ...
     @overload
     def match(self: Pattern[str], string: str, pos: int = 0, endpos: int = sys.maxsize) -> Match[str] | None: ...
     @overload
-    def match(self: Pattern[bytes], string: ReadableBuffer, pos: int = 0, endpos: int = sys.maxsize) -> Match[bytes] | None: ...  # type: ignore[overload-overlap]
+    def match(self: Pattern[bytes], string: ReadableBuffer, pos: int = 0, endpos: int = sys.maxsize) -> Match[bytes] | None: ...
     @overload
     def match(self, string: AnyStr, pos: int = 0, endpos: int = sys.maxsize) -> Match[AnyStr] | None: ...
     @overload
     def fullmatch(self: Pattern[str], string: str, pos: int = 0, endpos: int = sys.maxsize) -> Match[str] | None: ...
     @overload
-    def fullmatch(self: Pattern[bytes], string: ReadableBuffer, pos: int = 0, endpos: int = sys.maxsize) -> Match[bytes] | None: ...  # type: ignore[overload-overlap]
+    def fullmatch(
+        self: Pattern[bytes], string: ReadableBuffer, pos: int = 0, endpos: int = sys.maxsize
+    ) -> Match[bytes] | None: ...
     @overload
     def fullmatch(self, string: AnyStr, pos: int = 0, endpos: int = sys.maxsize) -> Match[AnyStr] | None: ...
     @overload
@@ -155,13 +157,15 @@ class Pattern(Generic[AnyStr]):
     @overload
     def finditer(self: Pattern[str], string: str, pos: int = 0, endpos: int = sys.maxsize) -> Iterator[Match[str]]: ...
     @overload
-    def finditer(self: Pattern[bytes], string: ReadableBuffer, pos: int = 0, endpos: int = sys.maxsize) -> Iterator[Match[bytes]]: ...  # type: ignore[overload-overlap]
+    def finditer(
+        self: Pattern[bytes], string: ReadableBuffer, pos: int = 0, endpos: int = sys.maxsize
+    ) -> Iterator[Match[bytes]]: ...
     @overload
     def finditer(self, string: AnyStr, pos: int = 0, endpos: int = sys.maxsize) -> Iterator[Match[AnyStr]]: ...
     @overload
     def sub(self: Pattern[str], repl: str | Callable[[Match[str]], str], string: str, count: int = 0) -> str: ...
     @overload
-    def sub(  # type: ignore[overload-overlap]
+    def sub(
         self: Pattern[bytes],
         repl: ReadableBuffer | Callable[[Match[bytes]], ReadableBuffer],
         string: ReadableBuffer,
@@ -172,7 +176,7 @@ class Pattern(Generic[AnyStr]):
     @overload
     def subn(self: Pattern[str], repl: str | Callable[[Match[str]], str], string: str, count: int = 0) -> tuple[str, int]: ...
     @overload
-    def subn(  # type: ignore[overload-overlap]
+    def subn(
         self: Pattern[bytes],
         repl: ReadableBuffer | Callable[[Match[bytes]], ReadableBuffer],
         string: ReadableBuffer,

crates/red_knot_python_semantic/vendor/typeshed/stdlib/sqlite3/dbapi2.pyi

@@ -29,7 +29,10 @@ def DateFromTicks(ticks: float) -> Date: ...
 def TimeFromTicks(ticks: float) -> Time: ...
 def TimestampFromTicks(ticks: float) -> Timestamp: ...
 
-version_info: tuple[int, int, int]
+if sys.version_info < (3, 14):
+    # Deprecated in 3.12, removed in 3.14.
+    version_info: tuple[int, int, int]
+
 sqlite_version_info: tuple[int, int, int]
 Binary = memoryview
 
@@ -90,7 +93,10 @@ SQLITE_UPDATE: Final[int]
 adapters: dict[tuple[type[Any], type[Any]], _Adapter[Any]]
 converters: dict[str, _Converter]
 sqlite_version: str
-version: str
+
+if sys.version_info < (3, 14):
+    # Deprecated in 3.12, removed in 3.14.
+    version: str
 
 if sys.version_info >= (3, 11):
     SQLITE_ABORT: Final[int]

crates/red_knot_python_semantic/vendor/typeshed/stdlib/symtable.pyi

@@ -2,6 +2,7 @@ import sys
 from _collections_abc import dict_keys
 from collections.abc import Sequence
 from typing import Any
+from typing_extensions import deprecated
 
 __all__ = ["symtable", "SymbolTable", "Class", "Function", "Symbol"]
 
@@ -51,7 +52,9 @@ class Function(SymbolTable):
     def get_nonlocals(self) -> tuple[str, ...]: ...
 
 class Class(SymbolTable):
-    def get_methods(self) -> tuple[str, ...]: ...
+    if sys.version_info < (3, 16):
+        @deprecated("deprecated in Python 3.14, will be removed in Python 3.16")
+        def get_methods(self) -> tuple[str, ...]: ...
 
 class Symbol:
     def __init__(

crates/red_knot_python_semantic/vendor/typeshed/stdlib/tarfile.pyi

@@ -423,7 +423,7 @@ class TarInfo:
     name: str
     path: str
     size: int
-    mtime: int
+    mtime: int | float
     chksum: int
     devmajor: int
     devminor: int

crates/red_knot_python_semantic/vendor/typeshed/stdlib/tempfile.pyi

@@ -463,7 +463,7 @@ class TemporaryDirectory(Generic[AnyStr]):
 
 # The overloads overlap, but they should still work fine.
 @overload
-def mkstemp(  # type: ignore[overload-overlap]
+def mkstemp(
     suffix: str | None = None, prefix: str | None = None, dir: StrPath | None = None, text: bool = False
 ) -> tuple[int, str]: ...
 @overload
@@ -473,7 +473,7 @@ def mkstemp(
 
 # The overloads overlap, but they should still work fine.
 @overload
-def mkdtemp(suffix: str | None = None, prefix: str | None = None, dir: StrPath | None = None) -> str: ...  # type: ignore[overload-overlap]
+def mkdtemp(suffix: str | None = None, prefix: str | None = None, dir: StrPath | None = None) -> str: ...
 @overload
 def mkdtemp(suffix: bytes | None = None, prefix: bytes | None = None, dir: BytesPath | None = None) -> bytes: ...
 def mktemp(suffix: str = "", prefix: str = "tmp", dir: StrPath | None = None) -> str: ...

crates/red_knot_python_semantic/vendor/typeshed/stdlib/tkinter/__init__.pyi

@@ -2148,11 +2148,12 @@ class Listbox(Widget, XView, YView):
         selectborderwidth: _ScreenUnits = 0,
         selectforeground: str = ...,
         # from listbox man page: "The value of the [selectmode] option may be
-        # arbitrary, but the default bindings expect it to be ..."
+        # arbitrary, but the default bindings expect it to be either single,
+        # browse, multiple, or extended"
         #
         # I have never seen anyone setting this to something else than what
         # "the default bindings expect", but let's support it anyway.
-        selectmode: str = "browse",
+        selectmode: str | Literal["single", "browse", "multiple", "extended"] = "browse",  # noqa: Y051
         setgrid: bool = False,
         state: Literal["normal", "disabled"] = "normal",
         takefocus: _TakeFocusValue = "",
@@ -2187,7 +2188,7 @@ class Listbox(Widget, XView, YView):
         selectbackground: str = ...,
         selectborderwidth: _ScreenUnits = ...,
         selectforeground: str = ...,
-        selectmode: str = ...,
+        selectmode: str | Literal["single", "browse", "multiple", "extended"] = ...,  # noqa: Y051
         setgrid: bool = ...,
         state: Literal["normal", "disabled"] = ...,
         takefocus: _TakeFocusValue = ...,
@@ -2907,6 +2908,9 @@ class Scrollbar(Widget):
     def set(self, first: float | str, last: float | str) -> None: ...
 
 _TextIndex: TypeAlias = _tkinter.Tcl_Obj | str | float | Misc
+_WhatToCount: TypeAlias = Literal[
+    "chars", "displaychars", "displayindices", "displaylines", "indices", "lines", "xpixels", "ypixels"
+]
 
 class Text(Widget, XView, YView):
     def __init__(
@@ -3021,7 +3025,27 @@ class Text(Widget, XView, YView):
     config = configure
     def bbox(self, index: _TextIndex) -> tuple[int, int, int, int] | None: ...  # type: ignore[override]
     def compare(self, index1: _TextIndex, op: Literal["<", "<=", "==", ">=", ">", "!="], index2: _TextIndex) -> bool: ...
-    def count(self, index1, index2, *args): ...  # TODO
+    @overload
+    def count(self, index1: _TextIndex, index2: _TextIndex) -> tuple[int] | None: ...
+    @overload
+    def count(self, index1: _TextIndex, index2: _TextIndex, arg: _WhatToCount | Literal["update"], /) -> tuple[int] | None: ...
+    @overload
+    def count(self, index1: _TextIndex, index2: _TextIndex, arg1: Literal["update"], arg2: _WhatToCount, /) -> int | None: ...
+    @overload
+    def count(self, index1: _TextIndex, index2: _TextIndex, arg1: _WhatToCount, arg2: Literal["update"], /) -> int | None: ...
+    @overload
+    def count(self, index1: _TextIndex, index2: _TextIndex, arg1: _WhatToCount, arg2: _WhatToCount, /) -> tuple[int, int]: ...
+    @overload
+    def count(
+        self,
+        index1: _TextIndex,
+        index2: _TextIndex,
+        arg1: _WhatToCount | Literal["update"],
+        arg2: _WhatToCount | Literal["update"],
+        arg3: _WhatToCount | Literal["update"],
+        /,
+        *args: _WhatToCount | Literal["update"],
+    ) -> tuple[int, ...]: ...
     @overload
     def debug(self, boolean: None = None) -> bool: ...
     @overload
@@ -3564,7 +3588,7 @@ class Spinbox(Widget, XView):
     def scan_dragto(self, x): ...
     def selection(self, *args) -> tuple[int, ...]: ...
     def selection_adjust(self, index): ...
-    def selection_clear(self): ...
+    def selection_clear(self): ...  # type: ignore[override]
     def selection_element(self, element: Incomplete | None = None): ...
     def selection_from(self, index: int) -> None: ...
     def selection_present(self) -> None: ...

crates/red_knot_python_semantic/vendor/typeshed/stdlib/tkinter/ttk.pyi

@@ -1040,7 +1040,7 @@ class Treeview(Widget, tkinter.XView, tkinter.YView):
     @overload
     def heading(self, column: str | int, option: str) -> Any: ...
     @overload
-    def heading(self, column: str | int, option: None = None) -> _TreeviewHeaderDict: ...  # type: ignore[overload-overlap]
+    def heading(self, column: str | int, option: None = None) -> _TreeviewHeaderDict: ...
     @overload
     def heading(
         self,
@@ -1052,7 +1052,8 @@ class Treeview(Widget, tkinter.XView, tkinter.YView):
         anchor: tkinter._Anchor = ...,
         command: str | Callable[[], object] = ...,
     ) -> None: ...
-    def identify(self, component, x, y): ...  # Internal Method. Leave untyped
+    # Internal Method. Leave untyped:
+    def identify(self, component, x, y): ...  # type: ignore[override]
     def identify_row(self, y: int) -> str: ...
     def identify_column(self, x: int) -> str: ...
     def identify_region(self, x: int, y: int) -> Literal["heading", "separator", "tree", "cell", "nothing"]: ...
@@ -1084,7 +1085,7 @@ class Treeview(Widget, tkinter.XView, tkinter.YView):
     @overload
     def item(self, item: str | int, option: str) -> Any: ...
     @overload
-    def item(self, item: str | int, option: None = None) -> _TreeviewItemDict: ...  # type: ignore[overload-overlap]
+    def item(self, item: str | int, option: None = None) -> _TreeviewItemDict: ...
     @overload
     def item(
         self,

crates/red_knot_python_semantic/vendor/typeshed/stdlib/turtle.pyi

@@ -338,7 +338,7 @@ class TPen:
     def isvisible(self) -> bool: ...
     # Note: signatures 1 and 2 overlap unsafely when no arguments are provided
     @overload
-    def pen(self) -> _PenState: ...  # type: ignore[overload-overlap]
+    def pen(self) -> _PenState: ...
     @overload
     def pen(
         self,
@@ -384,7 +384,7 @@ class RawTurtle(TPen, TNavigator):
     def shape(self, name: str) -> None: ...
     # Unsafely overlaps when no arguments are provided
     @overload
-    def shapesize(self) -> tuple[float, float, float]: ...  # type: ignore[overload-overlap]
+    def shapesize(self) -> tuple[float, float, float]: ...
     @overload
     def shapesize(
         self, stretch_wid: float | None = None, stretch_len: float | None = None, outline: float | None = None
@@ -395,7 +395,7 @@ class RawTurtle(TPen, TNavigator):
     def shearfactor(self, shear: float) -> None: ...
     # Unsafely overlaps when no arguments are provided
     @overload
-    def shapetransform(self) -> tuple[float, float, float, float]: ...  # type: ignore[overload-overlap]
+    def shapetransform(self) -> tuple[float, float, float, float]: ...
     @overload
     def shapetransform(
         self, t11: float | None = None, t12: float | None = None, t21: float | None = None, t22: float | None = None
@@ -622,7 +622,7 @@ def isvisible() -> bool: ...
 
 # Note: signatures 1 and 2 overlap unsafely when no arguments are provided
 @overload
-def pen() -> _PenState: ...  # type: ignore[overload-overlap]
+def pen() -> _PenState: ...
 @overload
 def pen(
     pen: _PenState | None = None,
@@ -661,7 +661,7 @@ if sys.version_info >= (3, 12):
 
 # Unsafely overlaps when no arguments are provided
 @overload
-def shapesize() -> tuple[float, float, float]: ...  # type: ignore[overload-overlap]
+def shapesize() -> tuple[float, float, float]: ...
 @overload
 def shapesize(stretch_wid: float | None = None, stretch_len: float | None = None, outline: float | None = None) -> None: ...
 @overload
@@ -671,7 +671,7 @@ def shearfactor(shear: float) -> None: ...
 
 # Unsafely overlaps when no arguments are provided
 @overload
-def shapetransform() -> tuple[float, float, float, float]: ...  # type: ignore[overload-overlap]
+def shapetransform() -> tuple[float, float, float, float]: ...
 @overload
 def shapetransform(
     t11: float | None = None, t12: float | None = None, t21: float | None = None, t22: float | None = None

crates/red_knot_python_semantic/vendor/typeshed/stdlib/types.pyi

@@ -305,9 +305,9 @@ class MappingProxyType(Mapping[_KT, _VT_co]):
     def values(self) -> ValuesView[_VT_co]: ...
     def items(self) -> ItemsView[_KT, _VT_co]: ...
     @overload
-    def get(self, key: _KT, /) -> _VT_co | None: ...  # type: ignore[override]
+    def get(self, key: _KT, /) -> _VT_co | None: ...
     @overload
-    def get(self, key: _KT, default: _VT_co | _T2, /) -> _VT_co | _T2: ...  # type: ignore[override]
+    def get(self, key: _KT, default: _VT_co | _T2, /) -> _VT_co | _T2: ...
     if sys.version_info >= (3, 9):
         def __class_getitem__(cls, item: Any, /) -> GenericAlias: ...
         def __reversed__(self) -> Iterator[_KT]: ...
@@ -583,7 +583,7 @@ _P = ParamSpec("_P")
 
 # it's not really an Awaitable, but can be used in an await expression. Real type: Generator & Awaitable
 @overload
-def coroutine(func: Callable[_P, Generator[Any, Any, _R]]) -> Callable[_P, Awaitable[_R]]: ...  # type: ignore[overload-overlap]
+def coroutine(func: Callable[_P, Generator[Any, Any, _R]]) -> Callable[_P, Awaitable[_R]]: ...
 @overload
 def coroutine(func: _Fn) -> _Fn: ...
 

crates/red_knot_python_semantic/vendor/typeshed/stdlib/typing.pyi

@@ -846,7 +846,8 @@ class TextIO(IO[str]):
     @abstractmethod
     def __enter__(self) -> TextIO: ...
 
-ByteString: typing_extensions.TypeAlias = bytes | bytearray | memoryview
+if sys.version_info < (3, 14):
+    ByteString: typing_extensions.TypeAlias = bytes | bytearray | memoryview
 
 # Functions
 

crates/red_knot_python_semantic/vendor/typeshed/stdlib/unittest/mock.pyi

@@ -299,7 +299,7 @@ class _patcher:
     # Ideally we'd be able to add an overload for it so that the return type is _patch[MagicMock],
     # but that's impossible with the current type system.
     @overload
-    def __call__(  # type: ignore[overload-overlap]
+    def __call__(
         self,
         target: str,
         new: _T,

crates/red_knot_python_semantic/vendor/typeshed/stdlib/urllib/parse.pyi

@@ -198,13 +198,13 @@ else:
 
 # Requires an iterable of length 6
 @overload
-def urlunparse(components: Iterable[None]) -> Literal[b""]: ...
+def urlunparse(components: Iterable[None]) -> Literal[b""]: ...  # type: ignore[overload-overlap]
 @overload
 def urlunparse(components: Iterable[AnyStr | None]) -> AnyStr: ...
 
 # Requires an iterable of length 5
 @overload
-def urlunsplit(components: Iterable[None]) -> Literal[b""]: ...
+def urlunsplit(components: Iterable[None]) -> Literal[b""]: ...  # type: ignore[overload-overlap]
 @overload
 def urlunsplit(components: Iterable[AnyStr | None]) -> AnyStr: ...
 def unwrap(url: str) -> str: ...

crates/red_knot_python_semantic/vendor/typeshed/stdlib/urllib/request.pyi

@@ -79,6 +79,7 @@ else:
     def pathname2url(pathname: str) -> str: ...
 
 def getproxies() -> dict[str, str]: ...
+def getproxies_environment() -> dict[str, str]: ...
 def parse_http_list(s: str) -> list[str]: ...
 def parse_keqv_list(l: list[str]) -> dict[str, str]: ...
 

crates/red_knot_python_semantic/vendor/typeshed/stdlib/xml/dom/__init__.pyi

@@ -1,4 +1,4 @@
-from typing import Any
+from typing import Any, Final
 
 from .domreg import getDOMImplementation as getDOMImplementation, registerDOMImplementation as registerDOMImplementation
 
@@ -17,22 +17,22 @@ class Node:
     NOTATION_NODE: int
 
 # ExceptionCode
-INDEX_SIZE_ERR: int
-DOMSTRING_SIZE_ERR: int
-HIERARCHY_REQUEST_ERR: int
-WRONG_DOCUMENT_ERR: int
-INVALID_CHARACTER_ERR: int
-NO_DATA_ALLOWED_ERR: int
-NO_MODIFICATION_ALLOWED_ERR: int
-NOT_FOUND_ERR: int
-NOT_SUPPORTED_ERR: int
-INUSE_ATTRIBUTE_ERR: int
-INVALID_STATE_ERR: int
-SYNTAX_ERR: int
-INVALID_MODIFICATION_ERR: int
-NAMESPACE_ERR: int
-INVALID_ACCESS_ERR: int
-VALIDATION_ERR: int
+INDEX_SIZE_ERR: Final[int]
+DOMSTRING_SIZE_ERR: Final[int]
+HIERARCHY_REQUEST_ERR: Final[int]
+WRONG_DOCUMENT_ERR: Final[int]
+INVALID_CHARACTER_ERR: Final[int]
+NO_DATA_ALLOWED_ERR: Final[int]
+NO_MODIFICATION_ALLOWED_ERR: Final[int]
+NOT_FOUND_ERR: Final[int]
+NOT_SUPPORTED_ERR: Final[int]
+INUSE_ATTRIBUTE_ERR: Final[int]
+INVALID_STATE_ERR: Final[int]
+SYNTAX_ERR: Final[int]
+INVALID_MODIFICATION_ERR: Final[int]
+NAMESPACE_ERR: Final[int]
+INVALID_ACCESS_ERR: Final[int]
+VALIDATION_ERR: Final[int]
 
 class DOMException(Exception):
     code: int
@@ -62,8 +62,8 @@ class UserDataHandler:
     NODE_DELETED: int
     NODE_RENAMED: int
 
-XML_NAMESPACE: str
-XMLNS_NAMESPACE: str
-XHTML_NAMESPACE: str
-EMPTY_NAMESPACE: None
-EMPTY_PREFIX: None
+XML_NAMESPACE: Final[str]
+XMLNS_NAMESPACE: Final[str]
+XHTML_NAMESPACE: Final[str]
+EMPTY_NAMESPACE: Final[None]
+EMPTY_PREFIX: Final[None]

crates/red_knot_python_semantic/vendor/typeshed/stdlib/xml/etree/ElementInclude.pyi

@@ -1,14 +1,15 @@
 import sys
 from _typeshed import FileDescriptorOrPath
 from collections.abc import Callable
+from typing import Final
 from xml.etree.ElementTree import Element
 
-XINCLUDE: str
-XINCLUDE_INCLUDE: str
-XINCLUDE_FALLBACK: str
+XINCLUDE: Final[str]
+XINCLUDE_INCLUDE: Final[str]
+XINCLUDE_FALLBACK: Final[str]
 
 if sys.version_info >= (3, 9):
-    DEFAULT_MAX_INCLUSION_DEPTH: int
+    DEFAULT_MAX_INCLUSION_DEPTH: Final = 6
 
 class FatalIncludeError(SyntaxError): ...
 

crates/red_knot_python_semantic/vendor/typeshed/stdlib/xml/etree/ElementTree.pyi

@@ -2,7 +2,7 @@ import sys
 from _collections_abc import dict_keys
 from _typeshed import FileDescriptorOrPath, ReadableBuffer, SupportsRead, SupportsWrite
 from collections.abc import Callable, Generator, ItemsView, Iterable, Iterator, Mapping, Sequence
-from typing import Any, Literal, SupportsIndex, TypeVar, overload
+from typing import Any, Final, Literal, SupportsIndex, TypeVar, overload
 from typing_extensions import TypeAlias, TypeGuard, deprecated
 
 __all__ = [
@@ -41,7 +41,7 @@ _FileRead: TypeAlias = FileDescriptorOrPath | SupportsRead[bytes] | SupportsRead
 _FileWriteC14N: TypeAlias = FileDescriptorOrPath | SupportsWrite[bytes]
 _FileWrite: TypeAlias = _FileWriteC14N | SupportsWrite[str]
 
-VERSION: str
+VERSION: Final[str]
 
 class ParseError(SyntaxError):
     code: int

crates/red_knot_python_semantic/vendor/typeshed/stdlib/zipfile/__init__.pyi

@@ -94,6 +94,20 @@ class ZipExtFile(io.BufferedIOBase):
 class _Writer(Protocol):
     def write(self, s: str, /) -> object: ...
 
+class _ZipReadable(Protocol):
+    def seek(self, offset: int, whence: int = 0, /) -> int: ...
+    def read(self, n: int = -1, /) -> bytes: ...
+
+class _ZipTellable(Protocol):
+    def tell(self) -> int: ...
+
+class _ZipReadableTellable(_ZipReadable, _ZipTellable, Protocol): ...
+
+class _ZipWritable(Protocol):
+    def flush(self) -> None: ...
+    def close(self) -> None: ...
+    def write(self, b: bytes, /) -> int: ...
+
 class ZipFile:
     filename: str | None
     debug: int
@@ -106,24 +120,50 @@ class ZipFile:
     compresslevel: int | None  # undocumented
     mode: _ZipFileMode  # undocumented
     pwd: bytes | None  # undocumented
+    # metadata_encoding is new in 3.11
     if sys.version_info >= (3, 11):
         @overload
         def __init__(
             self,
             file: StrPath | IO[bytes],
+            mode: _ZipFileMode = "r",
+            compression: int = 0,
+            allowZip64: bool = True,
+            compresslevel: int | None = None,
+            *,
+            strict_timestamps: bool = True,
+            metadata_encoding: str | None = None,
+        ) -> None: ...
+        # metadata_encoding is only allowed for read mode
+        @overload
+        def __init__(
+            self,
+            file: StrPath | _ZipReadable,
             mode: Literal["r"] = "r",
             compression: int = 0,
             allowZip64: bool = True,
             compresslevel: int | None = None,
             *,
             strict_timestamps: bool = True,
-            metadata_encoding: str | None,
+            metadata_encoding: str | None = None,
         ) -> None: ...
         @overload
         def __init__(
             self,
-            file: StrPath | IO[bytes],
-            mode: _ZipFileMode = "r",
+            file: StrPath | _ZipWritable,
+            mode: Literal["w", "x"] = ...,
+            compression: int = 0,
+            allowZip64: bool = True,
+            compresslevel: int | None = None,
+            *,
+            strict_timestamps: bool = True,
+            metadata_encoding: None = None,
+        ) -> None: ...
+        @overload
+        def __init__(
+            self,
+            file: StrPath | _ZipReadableTellable,
+            mode: Literal["a"] = ...,
             compression: int = 0,
             allowZip64: bool = True,
             compresslevel: int | None = None,
@@ -132,6 +172,7 @@ class ZipFile:
             metadata_encoding: None = None,
         ) -> None: ...
     else:
+        @overload
         def __init__(
             self,
             file: StrPath | IO[bytes],
@@ -142,6 +183,39 @@ class ZipFile:
             *,
             strict_timestamps: bool = True,
         ) -> None: ...
+        @overload
+        def __init__(
+            self,
+            file: StrPath | _ZipReadable,
+            mode: Literal["r"] = "r",
+            compression: int = 0,
+            allowZip64: bool = True,
+            compresslevel: int | None = None,
+            *,
+            strict_timestamps: bool = True,
+        ) -> None: ...
+        @overload
+        def __init__(
+            self,
+            file: StrPath | _ZipWritable,
+            mode: Literal["w", "x"] = ...,
+            compression: int = 0,
+            allowZip64: bool = True,
+            compresslevel: int | None = None,
+            *,
+            strict_timestamps: bool = True,
+        ) -> None: ...
+        @overload
+        def __init__(
+            self,
+            file: StrPath | _ZipReadableTellable,
+            mode: Literal["a"] = ...,
+            compression: int = 0,
+            allowZip64: bool = True,
+            compresslevel: int | None = None,
+            *,
+            strict_timestamps: bool = True,
+        ) -> None: ...
 
     def __enter__(self) -> Self: ...
     def __exit__(

crates/red_knot_server/Cargo.toml

@@ -11,6 +11,7 @@ repository = { workspace = true }
 license = { workspace = true }
 
 [dependencies]
+red_knot_python_semantic = { workspace = true }
 red_knot_workspace = { workspace = true }
 ruff_db = { workspace = true }
 ruff_linter = { workspace = true }
@@ -24,7 +25,7 @@ crossbeam = { workspace = true }
 jod-thread = { workspace = true }
 lsp-server = { workspace = true }
 lsp-types = { workspace = true }
-rustc-hash = { workspace = true }
+foldhash = { workspace = true }
 serde = { workspace = true }
 serde_json = { workspace = true }
 shellexpand = { workspace = true }

crates/red_knot_server/src/edit/notebook.rs

@@ -1,6 +1,7 @@
 use anyhow::Ok;
+use foldhash::{HashMap, HashMapExt};
 use lsp_types::NotebookCellKind;
-use rustc_hash::{FxBuildHasher, FxHashMap};
+use ruff_notebook::CellMetadata;
 
 use crate::{PositionEncoding, TextDocument};
 
@@ -16,7 +17,7 @@ pub struct NotebookDocument {
     metadata: ruff_notebook::RawNotebookMetadata,
     version: DocumentVersion,
     // Used to quickly find the index of a cell for a given URL.
-    cell_index: FxHashMap<lsp_types::Url, CellId>,
+    cell_index: HashMap<lsp_types::Url, CellId>,
 }
 
 /// A single cell within a notebook, which has text contents represented as a `TextDocument`.
@@ -34,7 +35,7 @@ impl NotebookDocument {
         metadata: serde_json::Map<String, serde_json::Value>,
         cell_documents: Vec<lsp_types::TextDocumentItem>,
     ) -> crate::Result<Self> {
-        let mut cell_contents: FxHashMap<_, _> = cell_documents
+        let mut cell_contents: HashMap<_, _> = cell_documents
             .into_iter()
             .map(|document| (document.uri, document.text))
             .collect();
@@ -65,7 +66,7 @@ impl NotebookDocument {
                 NotebookCellKind::Code => ruff_notebook::Cell::Code(ruff_notebook::CodeCell {
                     execution_count: None,
                     id: None,
-                    metadata: serde_json::Value::Null,
+                    metadata: CellMetadata::default(),
                     outputs: vec![],
                     source: ruff_notebook::SourceValue::String(
                         cell.document.contents().to_string(),
@@ -75,7 +76,7 @@ impl NotebookDocument {
                     ruff_notebook::Cell::Markdown(ruff_notebook::MarkdownCell {
                         attachments: None,
                         id: None,
-                        metadata: serde_json::Value::Null,
+                        metadata: CellMetadata::default(),
                         source: ruff_notebook::SourceValue::String(
                             cell.document.contents().to_string(),
                         ),
@@ -121,7 +122,7 @@ impl NotebookDocument {
                 // Instead, it only provides that (a) these cell URIs were removed, and (b) these
                 // cell URIs were added.
                 // https://github.com/astral-sh/ruff/issues/12573
-                let mut deleted_cells = FxHashMap::default();
+                let mut deleted_cells = HashMap::default();
 
                 // First, delete the cells and remove them from the index.
                 if delete > 0 {
@@ -215,8 +216,8 @@ impl NotebookDocument {
         self.cells.get_mut(*self.cell_index.get(uri)?)
     }
 
-    fn make_cell_index(cells: &[NotebookCell]) -> FxHashMap<lsp_types::Url, CellId> {
-        let mut index = FxHashMap::with_capacity_and_hasher(cells.len(), FxBuildHasher);
+    fn make_cell_index(cells: &[NotebookCell]) -> HashMap<lsp_types::Url, CellId> {
+        let mut index = HashMap::with_capacity(cells.len());
         for (i, cell) in cells.iter().enumerate() {
             index.insert(cell.url.clone(), i);
         }

crates/red_knot_server/src/server/api/requests/diagnostic.rs

@@ -2,8 +2,9 @@ use std::borrow::Cow;
 
 use lsp_types::request::DocumentDiagnosticRequest;
 use lsp_types::{
-    Diagnostic, DocumentDiagnosticParams, DocumentDiagnosticReport, DocumentDiagnosticReportResult,
-    FullDocumentDiagnosticReport, Range, RelatedFullDocumentDiagnosticReport, Url,
+    Diagnostic, DiagnosticSeverity, DocumentDiagnosticParams, DocumentDiagnosticReport,
+    DocumentDiagnosticReportResult, FullDocumentDiagnosticReport, Position, Range,
+    RelatedFullDocumentDiagnosticReport, Url,
 };
 
 use red_knot_workspace::db::RootDatabase;
@@ -56,16 +57,37 @@ fn compute_diagnostics(snapshot: &DocumentSnapshot, db: &RootDatabase) -> Vec<Di
     diagnostics
         .as_slice()
         .iter()
-        .map(|message| Diagnostic {
-            range: Range::default(),
-            severity: None,
-            tags: None,
-            code: None,
-            code_description: None,
-            source: Some("red-knot".into()),
-            message: message.to_string(),
-            related_information: None,
-            data: None,
-        })
+        .map(|message| to_lsp_diagnostic(message))
         .collect()
 }
+
+fn to_lsp_diagnostic(message: &str) -> Diagnostic {
+    let words = message.split(':').collect::<Vec<_>>();
+
+    let (range, message) = match words.as_slice() {
+        [_filename, line, column, message] => {
+            let line = line.parse::<u32>().unwrap_or_default();
+            let column = column.parse::<u32>().unwrap_or_default();
+            (
+                Range::new(
+                    Position::new(line.saturating_sub(1), column.saturating_sub(1)),
+                    Position::new(line, column),
+                ),
+                message.trim(),
+            )
+        }
+        _ => (Range::default(), message),
+    };
+
+    Diagnostic {
+        range,
+        severity: Some(DiagnosticSeverity::ERROR),
+        tags: None,
+        code: None,
+        code_description: None,
+        source: Some("red-knot".into()),
+        message: message.to_string(),
+        related_information: None,
+        data: None,
+    }
+}

crates/red_knot_server/src/server/client.rs

@@ -1,7 +1,7 @@
 use std::any::TypeId;
 
+use foldhash::HashMap;
 use lsp_server::{Notification, RequestId};
-use rustc_hash::FxHashMap;
 use serde_json::Value;
 
 use super::{schedule::Task, ClientSender};
@@ -23,7 +23,7 @@ pub(crate) struct Responder(ClientSender);
 pub(crate) struct Requester<'s> {
     sender: ClientSender,
     next_request_id: i32,
-    response_handlers: FxHashMap<lsp_server::RequestId, ResponseBuilder<'s>>,
+    response_handlers: HashMap<lsp_server::RequestId, ResponseBuilder<'s>>,
 }
 
 impl<'s> Client<'s> {
@@ -34,7 +34,7 @@ impl<'s> Client<'s> {
             requester: Requester {
                 sender,
                 next_request_id: 1,
-                response_handlers: FxHashMap::default(),
+                response_handlers: HashMap::default(),
             },
         }
     }

crates/red_knot_server/src/session.rs

@@ -8,10 +8,10 @@ use std::sync::Arc;
 use anyhow::anyhow;
 use lsp_types::{ClientCapabilities, Url};
 
+use red_knot_python_semantic::{ProgramSettings, PythonVersion, SearchPathSettings};
 use red_knot_workspace::db::RootDatabase;
 use red_knot_workspace::workspace::WorkspaceMetadata;
 use ruff_db::files::{system_path_to_file, File};
-use ruff_db::program::{ProgramSettings, SearchPathSettings, TargetVersion};
 use ruff_db::system::SystemPath;
 
 use crate::edit::{DocumentKey, NotebookDocument};
@@ -70,7 +70,7 @@ impl Session {
             let metadata = WorkspaceMetadata::from_path(system_path, &system)?;
             // TODO(dhruvmanila): Get the values from the client settings
             let program_settings = ProgramSettings {
-                target_version: TargetVersion::default(),
+                target_version: PythonVersion::default(),
                 search_paths: SearchPathSettings {
                     extra_paths: vec![],
                     src_root: system_path.to_path_buf(),
@@ -78,7 +78,8 @@ impl Session {
                     custom_typeshed: None,
                 },
             };
-            workspaces.insert(path, RootDatabase::new(metadata, program_settings, system));
+            // TODO(micha): Handle the case where the program settings are incorrect more gracefully.
+            workspaces.insert(path, RootDatabase::new(metadata, program_settings, system)?);
         }
 
         Ok(Self {

crates/red_knot_server/src/session/index.rs

@@ -2,8 +2,8 @@ use std::borrow::Cow;
 use std::path::{Path, PathBuf};
 use std::sync::Arc;
 
+use foldhash::HashMap;
 use lsp_types::Url;
-use rustc_hash::FxHashMap;
 
 use crate::{
     edit::{DocumentKey, DocumentVersion, NotebookDocument},
@@ -16,10 +16,10 @@ use super::ClientSettings;
 #[derive(Default, Debug)]
 pub(crate) struct Index {
     /// Maps all document file URLs to the associated document controller
-    documents: FxHashMap<Url, DocumentController>,
+    documents: HashMap<Url, DocumentController>,
 
     /// Maps opaque cell URLs to a notebook URL (document)
-    notebook_cells: FxHashMap<Url, Url>,
+    notebook_cells: HashMap<Url, Url>,
 
     /// Global settings provided by the client.
     global_settings: ClientSettings,
@@ -28,8 +28,8 @@ pub(crate) struct Index {
 impl Index {
     pub(super) fn new(global_settings: ClientSettings) -> Self {
         Self {
-            documents: FxHashMap::default(),
-            notebook_cells: FxHashMap::default(),
+            documents: HashMap::default(),
+            notebook_cells: HashMap::default(),
             global_settings,
         }
     }

crates/red_knot_server/src/session/settings.rs

@@ -1,11 +1,11 @@
 use std::path::PathBuf;
 
+use foldhash::HashMap;
 use lsp_types::Url;
-use rustc_hash::FxHashMap;
 use serde::Deserialize;
 
 /// Maps a workspace URI to its associated client settings. Used during server initialization.
-pub(crate) type WorkspaceSettingsMap = FxHashMap<Url, ClientSettings>;
+pub(crate) type WorkspaceSettingsMap = HashMap<Url, ClientSettings>;
 
 /// This is a direct representation of the settings schema sent by the client.
 #[derive(Debug, Deserialize, Default)]

crates/red_knot_wasm/Cargo.toml

@@ -19,6 +19,7 @@ doctest = false
 default = ["console_error_panic_hook"]
 
 [dependencies]
+red_knot_python_semantic = { workspace = true }
 red_knot_workspace = { workspace = true }
 
 ruff_db = { workspace = true }

crates/red_knot_wasm/src/lib.rs

@@ -3,10 +3,10 @@ use std::any::Any;
 use js_sys::Error;
 use wasm_bindgen::prelude::*;
 
+use red_knot_python_semantic::{ProgramSettings, SearchPathSettings};
 use red_knot_workspace::db::RootDatabase;
 use red_knot_workspace::workspace::WorkspaceMetadata;
 use ruff_db::files::{system_path_to_file, File};
-use ruff_db::program::{ProgramSettings, SearchPathSettings};
 use ruff_db::system::walk_directory::WalkDirectoryBuilder;
 use ruff_db::system::{
     DirectoryEntry, MemoryFileSystem, Metadata, System, SystemPath, SystemPathBuf,
@@ -49,7 +49,8 @@ impl Workspace {
             search_paths: SearchPathSettings::default(),
         };
 
-        let db = RootDatabase::new(workspace, program_settings, system.clone());
+        let db =
+            RootDatabase::new(workspace, program_settings, system.clone()).map_err(into_error)?;
 
         Ok(Self { db, system })
     }
@@ -184,16 +185,16 @@ pub enum TargetVersion {
     Py313,
 }
 
-impl From<TargetVersion> for ruff_db::program::TargetVersion {
+impl From<TargetVersion> for red_knot_python_semantic::PythonVersion {
     fn from(value: TargetVersion) -> Self {
         match value {
-            TargetVersion::Py37 => Self::Py37,
-            TargetVersion::Py38 => Self::Py38,
-            TargetVersion::Py39 => Self::Py39,
-            TargetVersion::Py310 => Self::Py310,
-            TargetVersion::Py311 => Self::Py311,
-            TargetVersion::Py312 => Self::Py312,
-            TargetVersion::Py313 => Self::Py313,
+            TargetVersion::Py37 => Self::PY37,
+            TargetVersion::Py38 => Self::PY38,
+            TargetVersion::Py39 => Self::PY39,
+            TargetVersion::Py310 => Self::PY310,
+            TargetVersion::Py311 => Self::PY311,
+            TargetVersion::Py312 => Self::PY312,
+            TargetVersion::Py313 => Self::PY313,
         }
     }
 }

crates/red_knot_wasm/tests/api.rs

@@ -17,5 +17,5 @@ fn check() {
 
     let result = workspace.check_file(&test).expect("Check to succeed");
 
-    assert_eq!(result, vec!["Unresolved import 'random22'"]);
+    assert_eq!(result, vec!["/test.py:1:8: Unresolved import 'random22'"]);
 }

crates/red_knot_workspace/Cargo.toml

@@ -17,11 +17,12 @@ red_knot_python_semantic = { workspace = true }
 ruff_cache = { workspace = true }
 ruff_db = { workspace = true, features = ["os", "cache"] }
 ruff_python_ast = { workspace = true }
+ruff_text_size = { workspace = true }
 
 anyhow = { workspace = true }
 crossbeam = { workspace = true }
 notify = { workspace = true }
-rustc-hash = { workspace = true }
+foldhash = { workspace = true }
 salsa = { workspace = true }
 thiserror = { workspace = true }
 tracing = { workspace = true }

crates/red_knot_workspace/resources/test/unix-uv-venv/CACHEDIR.TAG

@@ -1 +0,0 @@
-Signature: 8a477f597d28d172789f06886806bc55

crates/red_knot_workspace/resources/test/unix-uv-venv/bin/python

@@ -1 +0,0 @@
-/Users/alexw/.pyenv/versions/3.12.4/bin/python3.12

crates/red_knot_workspace/resources/test/unix-uv-venv/bin/python3

@@ -1 +0,0 @@
-python

crates/red_knot_workspace/resources/test/unix-uv-venv/bin/python3.12

@@ -1 +0,0 @@
-python

crates/red_knot_workspace/resources/test/unix-uv-venv/lib/python3.12/site-packages/_virtualenv.pth

@@ -1 +0,0 @@
-import _virtualenv

crates/red_knot_workspace/resources/test/unix-uv-venv/lib/python3.12/site-packages/_virtualenv.py

@@ -1,103 +0,0 @@
-"""Patches that are applied at runtime to the virtual environment."""
-
-from __future__ import annotations
-
-import os
-import sys
-
-VIRTUALENV_PATCH_FILE = os.path.join(__file__)
-
-
-def patch_dist(dist):
-    """
-    Distutils allows user to configure some arguments via a configuration file:
-    https://docs.python.org/3/install/index.html#distutils-configuration-files.
-
-    Some of this arguments though don't make sense in context of the virtual environment files, let's fix them up.
-    """  # noqa: D205
-    # we cannot allow some install config as that would get packages installed outside of the virtual environment
-    old_parse_config_files = dist.Distribution.parse_config_files
-
-    def parse_config_files(self, *args, **kwargs):
-        result = old_parse_config_files(self, *args, **kwargs)
-        install = self.get_option_dict("install")
-
-        if "prefix" in install:  # the prefix governs where to install the libraries
-            install["prefix"] = VIRTUALENV_PATCH_FILE, os.path.abspath(sys.prefix)
-        for base in ("purelib", "platlib", "headers", "scripts", "data"):
-            key = f"install_{base}"
-            if key in install:  # do not allow global configs to hijack venv paths
-                install.pop(key, None)
-        return result
-
-    dist.Distribution.parse_config_files = parse_config_files
-
-
-# Import hook that patches some modules to ignore configuration values that break package installation in case
-# of virtual environments.
-_DISTUTILS_PATCH = "distutils.dist", "setuptools.dist"
-# https://docs.python.org/3/library/importlib.html#setting-up-an-importer
-
-
-class _Finder:
-    """A meta path finder that allows patching the imported distutils modules."""
-
-    fullname = None
-
-    # lock[0] is threading.Lock(), but initialized lazily to avoid importing threading very early at startup,
-    # because there are gevent-based applications that need to be first to import threading by themselves.
-    # See https://github.com/pypa/virtualenv/issues/1895 for details.
-    lock = []  # noqa: RUF012
-
-    def find_spec(self, fullname, path, target=None):  # noqa: ARG002
-        if fullname in _DISTUTILS_PATCH and self.fullname is None:
-            # initialize lock[0] lazily
-            if len(self.lock) == 0:
-                import threading
-
-                lock = threading.Lock()
-                # there is possibility that two threads T1 and T2 are simultaneously running into find_spec,
-                # observing .lock as empty, and further going into hereby initialization. However due to the GIL,
-                # list.append() operation is atomic and this way only one of the threads will "win" to put the lock
-                # - that every thread will use - into .lock[0].
-                # https://docs.python.org/3/faq/library.html#what-kinds-of-global-value-mutation-are-thread-safe
-                self.lock.append(lock)
-
-            from functools import partial
-            from importlib.util import find_spec
-
-            with self.lock[0]:
-                self.fullname = fullname
-                try:
-                    spec = find_spec(fullname, path)
-                    if spec is not None:
-                        # https://www.python.org/dev/peps/pep-0451/#how-loading-will-work
-                        is_new_api = hasattr(spec.loader, "exec_module")
-                        func_name = "exec_module" if is_new_api else "load_module"
-                        old = getattr(spec.loader, func_name)
-                        func = self.exec_module if is_new_api else self.load_module
-                        if old is not func:
-                            try:  # noqa: SIM105
-                                setattr(spec.loader, func_name, partial(func, old))
-                            except AttributeError:
-                                pass  # C-Extension loaders are r/o such as zipimporter with <3.7
-                        return spec
-                finally:
-                    self.fullname = None
-        return None
-
-    @staticmethod
-    def exec_module(old, module):
-        old(module)
-        if module.__name__ in _DISTUTILS_PATCH:
-            patch_dist(module)
-
-    @staticmethod
-    def load_module(old, name):
-        module = old(name)
-        if module.__name__ in _DISTUTILS_PATCH:
-            patch_dist(module)
-        return module
-
-
-sys.meta_path.insert(0, _Finder())

crates/red_knot_workspace/resources/test/unix-uv-venv/pyvenv.cfg

@@ -1,6 +0,0 @@
-home = /Users/alexw/.pyenv/versions/3.12.4/bin
-implementation = CPython
-uv = 0.2.32
-version_info = 3.12.4
-include-system-site-packages = false
-relocatable = false

crates/red_knot_workspace/src/db.rs

@@ -1,9 +1,10 @@
 use std::panic::RefUnwindSafe;
 use std::sync::Arc;
 
-use red_knot_python_semantic::{vendored_typeshed_stubs, Db as SemanticDb};
+use red_knot_python_semantic::{
+    vendored_typeshed_stubs, Db as SemanticDb, Program, ProgramSettings,
+};
 use ruff_db::files::{File, Files};
-use ruff_db::program::{Program, ProgramSettings};
 use ruff_db::system::System;
 use ruff_db::vendored::VendoredFileSystem;
 use ruff_db::{Db as SourceDb, Upcast};
@@ -27,7 +28,11 @@ pub struct RootDatabase {
 }
 
 impl RootDatabase {
-    pub fn new<S>(workspace: WorkspaceMetadata, settings: ProgramSettings, system: S) -> Self
+    pub fn new<S>(
+        workspace: WorkspaceMetadata,
+        settings: ProgramSettings,
+        system: S,
+    ) -> anyhow::Result<Self>
     where
         S: System + 'static + Send + Sync + RefUnwindSafe,
     {
@@ -40,10 +45,10 @@ impl RootDatabase {
 
         let workspace = Workspace::from_metadata(&db, workspace);
         // Initialize the `Program` singleton
-        Program::from_settings(&db, settings);
+        Program::from_settings(&db, settings)?;
 
         db.workspace = Some(workspace);
-        db
+        Ok(db)
     }
 
     pub fn workspace(&self) -> Workspace {
@@ -129,7 +134,18 @@ impl SourceDb for RootDatabase {
 
 #[salsa::db]
 impl salsa::Database for RootDatabase {
-    fn salsa_event(&self, _event: &dyn Fn() -> Event) {}
+    fn salsa_event(&self, event: &dyn Fn() -> Event) {
+        if !tracing::enabled!(tracing::Level::TRACE) {
+            return;
+        }
+
+        let event = event();
+        if matches!(event.kind, salsa::EventKind::WillCheckCancellation { .. }) {
+            return;
+        }
+
+        tracing::trace!("Salsa event: {event:?}");
+    }
 }
 
 #[salsa::db]
@@ -138,6 +154,7 @@ impl Db for RootDatabase {}
 #[cfg(test)]
 pub(crate) mod tests {
     use salsa::Event;
+    use std::sync::Arc;
 
     use red_knot_python_semantic::{vendored_typeshed_stubs, Db as SemanticDb};
     use ruff_db::files::Files;
@@ -150,6 +167,7 @@ pub(crate) mod tests {
     #[salsa::db]
     pub(crate) struct TestDb {
         storage: salsa::Storage<Self>,
+        events: std::sync::Arc<std::sync::Mutex<Vec<salsa::Event>>>,
         files: Files,
         system: TestSystem,
         vendored: VendoredFileSystem,
@@ -162,10 +180,24 @@ pub(crate) mod tests {
                 system: TestSystem::default(),
                 vendored: vendored_typeshed_stubs().clone(),
                 files: Files::default(),
+                events: Arc::default(),
             }
         }
     }
 
+    impl TestDb {
+        /// Takes the salsa events.
+        ///
+        /// ## Panics
+        /// If there are any pending salsa snapshots.
+        pub(crate) fn take_salsa_events(&mut self) -> Vec<salsa::Event> {
+            let inner = Arc::get_mut(&mut self.events).expect("no pending salsa snapshots");
+
+            let events = inner.get_mut().unwrap();
+            std::mem::take(&mut *events)
+        }
+    }
+
     impl DbWithTestSystem for TestDb {
         fn test_system(&self) -> &TestSystem {
             &self.system
@@ -216,6 +248,9 @@ pub(crate) mod tests {
 
     #[salsa::db]
     impl salsa::Database for TestDb {
-        fn salsa_event(&self, _event: &dyn Fn() -> Event) {}
+        fn salsa_event(&self, event: &dyn Fn() -> Event) {
+            let mut events = self.events.lock().unwrap();
+            events.push(event());
+        }
     }
 }

crates/red_knot_workspace/src/db/changes.rs

@@ -1,4 +1,4 @@
-use rustc_hash::FxHashSet;
+use foldhash::HashSet;
 
 use ruff_db::files::{system_path_to_file, File, Files};
 use ruff_db::system::walk_directory::WalkState;
@@ -18,13 +18,13 @@ impl RootDatabase {
 
         let mut workspace_change = false;
         // Packages that need reloading
-        let mut changed_packages = FxHashSet::default();
+        let mut changed_packages = HashSet::default();
         // Paths that were added
-        let mut added_paths = FxHashSet::default();
+        let mut added_paths = HashSet::default();
 
         // Deduplicate the `sync` calls. Many file watchers emit multiple events for the same path.
-        let mut synced_files = FxHashSet::default();
-        let mut synced_recursively = FxHashSet::default();
+        let mut synced_files = HashSet::default();
+        let mut synced_recursively = HashSet::default();
 
         let mut sync_path = |db: &mut RootDatabase, path: &SystemPath| {
             if synced_files.insert(path.to_path_buf()) {

crates/red_knot_workspace/src/lint.rs

@@ -8,9 +8,10 @@ use red_knot_python_semantic::types::Type;
 use red_knot_python_semantic::{HasTy, ModuleName, SemanticModel};
 use ruff_db::files::File;
 use ruff_db::parsed::{parsed_module, ParsedModule};
-use ruff_db::source::{source_text, SourceText};
+use ruff_db::source::{line_index, source_text, SourceText};
 use ruff_python_ast as ast;
 use ruff_python_ast::visitor::{walk_expr, walk_stmt, Visitor};
+use ruff_text_size::{Ranged, TextSize};
 
 use crate::db::Db;
 
@@ -49,7 +50,18 @@ pub(crate) fn lint_syntax(db: &dyn Db, file_id: File) -> Diagnostics {
         visitor.visit_body(&ast.body);
         diagnostics = visitor.diagnostics;
     } else {
-        diagnostics.extend(parsed.errors().iter().map(ToString::to_string));
+        let path = file_id.path(db);
+        let line_index = line_index(db.upcast(), file_id);
+        diagnostics.extend(parsed.errors().iter().map(|err| {
+            let source_location = line_index.source_location(err.location.start(), source.as_str());
+            format!(
+                "{}:{}:{}: {}",
+                path.as_str(),
+                source_location.row,
+                source_location.column,
+                err,
+            )
+        }));
     }
 
     Diagnostics::from(diagnostics)
@@ -97,14 +109,36 @@ pub fn lint_semantic(db: &dyn Db, file_id: File) -> Diagnostics {
     Diagnostics::from(context.diagnostics.take())
 }
 
+fn format_diagnostic(context: &SemanticLintContext, message: &str, start: TextSize) -> String {
+    let source_location = context
+        .semantic
+        .line_index()
+        .source_location(start, context.source_text());
+    format!(
+        "{}:{}:{}: {}",
+        context.semantic.file_path().as_str(),
+        source_location.row,
+        source_location.column,
+        message,
+    )
+}
+
 fn lint_unresolved_imports(context: &SemanticLintContext, import: AnyImportRef) {
+    // TODO: this treats any symbol with `Type::Unknown` as an unresolved import,
+    // which isn't really correct: if it exists but has `Type::Unknown` in the
+    // module we're importing it from, we shouldn't really emit a diagnostic here,
+    // but currently do.
     match import {
         AnyImportRef::Import(import) => {
             for alias in &import.names {
                 let ty = alias.ty(&context.semantic);
 
-                if ty.is_unbound() {
-                    context.push_diagnostic(format!("Unresolved import '{}'", &alias.name));
+                if ty.is_unknown() {
+                    context.push_diagnostic(format_diagnostic(
+                        context,
+                        &format!("Unresolved import '{}'", &alias.name),
+                        alias.start(),
+                    ));
                 }
             }
         }
@@ -112,8 +146,12 @@ fn lint_unresolved_imports(context: &SemanticLintContext, import: AnyImportRef)
             for alias in &import.names {
                 let ty = alias.ty(&context.semantic);
 
-                if ty.is_unbound() {
-                    context.push_diagnostic(format!("Unresolved import '{}'", &alias.name));
+                if ty.is_unknown() {
+                    context.push_diagnostic(format_diagnostic(
+                        context,
+                        &format!("Unresolved import '{}'", &alias.name),
+                        alias.start(),
+                    ));
                 }
             }
         }
@@ -127,12 +165,17 @@ fn lint_maybe_undefined(context: &SemanticLintContext, name: &ast::ExprName) {
     let semantic = &context.semantic;
     match name.ty(semantic) {
         Type::Unbound => {
-            context.push_diagnostic(format!("Name '{}' used when not defined.", &name.id));
+            context.push_diagnostic(format_diagnostic(
+                context,
+                &format!("Name '{}' used when not defined.", &name.id),
+                name.start(),
+            ));
         }
         Type::Union(union) if union.contains(semantic.db(), Type::Unbound) => {
-            context.push_diagnostic(format!(
-                "Name '{}' used when possibly not defined.",
-                &name.id
+            context.push_diagnostic(format_diagnostic(
+                context,
+                &format!("Name '{}' used when possibly not defined.", &name.id),
+                name.start(),
             ));
         }
         _ => {}
@@ -303,15 +346,25 @@ enum AnyImportRef<'a> {
     ImportFrom(&'a ast::StmtImportFrom),
 }
 
+impl Ranged for AnyImportRef<'_> {
+    fn range(&self) -> ruff_text_size::TextRange {
+        match self {
+            AnyImportRef::Import(import) => import.range(),
+            AnyImportRef::ImportFrom(import) => import.range(),
+        }
+    }
+}
+
 #[cfg(test)]
 mod tests {
+    use red_knot_python_semantic::{Program, ProgramSettings, PythonVersion, SearchPathSettings};
     use ruff_db::files::system_path_to_file;
-    use ruff_db::program::{Program, SearchPathSettings, TargetVersion};
     use ruff_db::system::{DbWithTestSystem, SystemPathBuf};
 
-    use super::{lint_semantic, Diagnostics};
     use crate::db::tests::TestDb;
 
+    use super::{lint_semantic, Diagnostics};
+
     fn setup_db() -> TestDb {
         setup_db_with_root(SystemPathBuf::from("/src"))
     }
@@ -319,16 +372,23 @@ mod tests {
     fn setup_db_with_root(src_root: SystemPathBuf) -> TestDb {
         let db = TestDb::new();
 
-        Program::new(
+        db.memory_file_system()
+            .create_directory_all(&src_root)
+            .unwrap();
+
+        Program::from_settings(
             &db,
-            TargetVersion::Py38,
-            SearchPathSettings {
-                extra_paths: Vec::new(),
-                src_root,
-                site_packages: vec![],
-                custom_typeshed: None,
+            ProgramSettings {
+                target_version: PythonVersion::default(),
+                search_paths: SearchPathSettings {
+                    extra_paths: Vec::new(),
+                    src_root,
+                    site_packages: vec![],
+                    custom_typeshed: None,
+                },
             },
-        );
+        )
+        .expect("Valid program settings");
 
         db
     }
@@ -355,10 +415,17 @@ mod tests {
 
         assert_eq!(
             *messages,
-            vec![
-                "Name 'flag' used when not defined.",
-                "Name 'y' used when possibly not defined."
-            ]
+            if cfg!(windows) {
+                vec![
+                    "\\src\\a.py:3:4: Name 'flag' used when not defined.",
+                    "\\src\\a.py:5:1: Name 'y' used when possibly not defined.",
+                ]
+            } else {
+                vec![
+                    "/src/a.py:3:4: Name 'flag' used when not defined.",
+                    "/src/a.py:5:1: Name 'y' used when possibly not defined.",
+                ]
+            }
         );
     }
 }

crates/red_knot_workspace/src/site_packages.rs

@@ -8,43 +8,270 @@
 //! reasonably ask us to type-check code assuming that the code runs
 //! on Linux.)
 
+use std::fmt;
 use std::io;
+use std::num::NonZeroUsize;
+use std::ops::Deref;
 
+use red_knot_python_semantic::PythonVersion;
 use ruff_db::system::{System, SystemPath, SystemPathBuf};
 
+type SitePackagesDiscoveryResult<T> = Result<T, SitePackagesDiscoveryError>;
+
+/// Abstraction for a Python virtual environment.
+///
+/// Most of this information is derived from the virtual environment's `pyvenv.cfg` file.
+/// The format of this file is not defined anywhere, and exactly which keys are present
+/// depends on the tool that was used to create the virtual environment.
+#[derive(Debug)]
+pub struct VirtualEnvironment {
+    venv_path: SysPrefixPath,
+    base_executable_home_path: PythonHomePath,
+    include_system_site_packages: bool,
+
+    /// The version of the Python executable that was used to create this virtual environment.
+    ///
+    /// The Python version is encoded under different keys and in different formats
+    /// by different virtual-environment creation tools,
+    /// and the key is never read by the standard-library `site.py` module,
+    /// so it's possible that we might not be able to find this information
+    /// in an acceptable format under any of the keys we expect.
+    /// This field will be `None` if so.
+    version: Option<PythonVersion>,
+}
+
+impl VirtualEnvironment {
+    pub fn new(
+        path: impl AsRef<SystemPath>,
+        system: &dyn System,
+    ) -> SitePackagesDiscoveryResult<Self> {
+        Self::new_impl(path.as_ref(), system)
+    }
+
+    fn new_impl(path: &SystemPath, system: &dyn System) -> SitePackagesDiscoveryResult<Self> {
+        fn pyvenv_cfg_line_number(index: usize) -> NonZeroUsize {
+            index.checked_add(1).and_then(NonZeroUsize::new).unwrap()
+        }
+
+        let venv_path = SysPrefixPath::new(path, system)?;
+        let pyvenv_cfg_path = venv_path.join("pyvenv.cfg");
+        tracing::debug!("Attempting to parse virtual environment metadata at {pyvenv_cfg_path}");
+
+        let pyvenv_cfg = system
+            .read_to_string(&pyvenv_cfg_path)
+            .map_err(SitePackagesDiscoveryError::NoPyvenvCfgFile)?;
+
+        let mut include_system_site_packages = false;
+        let mut base_executable_home_path = None;
+        let mut version_info_string = None;
+
+        // A `pyvenv.cfg` file *looks* like a `.ini` file, but actually isn't valid `.ini` syntax!
+        // The Python standard-library's `site` module parses these files by splitting each line on
+        // '=' characters, so that's what we should do as well.
+        //
+        // See also: https://snarky.ca/how-virtual-environments-work/
+        for (index, line) in pyvenv_cfg.lines().enumerate() {
+            if let Some((key, value)) = line.split_once('=') {
+                let key = key.trim();
+                if key.is_empty() {
+                    return Err(SitePackagesDiscoveryError::PyvenvCfgParseError(
+                        pyvenv_cfg_path,
+                        PyvenvCfgParseErrorKind::MalformedKeyValuePair {
+                            line_number: pyvenv_cfg_line_number(index),
+                        },
+                    ));
+                }
+
+                let value = value.trim();
+                if value.is_empty() {
+                    return Err(SitePackagesDiscoveryError::PyvenvCfgParseError(
+                        pyvenv_cfg_path,
+                        PyvenvCfgParseErrorKind::MalformedKeyValuePair {
+                            line_number: pyvenv_cfg_line_number(index),
+                        },
+                    ));
+                }
+
+                if value.contains('=') {
+                    return Err(SitePackagesDiscoveryError::PyvenvCfgParseError(
+                        pyvenv_cfg_path,
+                        PyvenvCfgParseErrorKind::TooManyEquals {
+                            line_number: pyvenv_cfg_line_number(index),
+                        },
+                    ));
+                }
+
+                match key {
+                    "include-system-site-packages" => {
+                        include_system_site_packages = value.eq_ignore_ascii_case("true");
+                    }
+                    "home" => base_executable_home_path = Some(value),
+                    // `virtualenv` and `uv` call this key `version_info`,
+                    // but the stdlib venv module calls it `version`
+                    "version" | "version_info" => version_info_string = Some(value),
+                    _ => continue,
+                }
+            }
+        }
+
+        // The `home` key is read by the standard library's `site.py` module,
+        // so if it's missing from the `pyvenv.cfg` file
+        // (or the provided value is invalid),
+        // it's reasonable to consider the virtual environment irredeemably broken.
+        let Some(base_executable_home_path) = base_executable_home_path else {
+            return Err(SitePackagesDiscoveryError::PyvenvCfgParseError(
+                pyvenv_cfg_path,
+                PyvenvCfgParseErrorKind::NoHomeKey,
+            ));
+        };
+        let base_executable_home_path = PythonHomePath::new(base_executable_home_path, system)
+            .map_err(|io_err| {
+                SitePackagesDiscoveryError::PyvenvCfgParseError(
+                    pyvenv_cfg_path,
+                    PyvenvCfgParseErrorKind::InvalidHomeValue(io_err),
+                )
+            })?;
+
+        // but the `version`/`version_info` key is not read by the standard library,
+        // and is provided under different keys depending on which virtual-environment creation tool
+        // created the `pyvenv.cfg` file. Lenient parsing is appropriate here:
+        // the file isn't really *invalid* if it doesn't have this key,
+        // or if the value doesn't parse according to our expectations.
+        let version = version_info_string.and_then(|version_string| {
+            let mut version_info_parts = version_string.split('.');
+            let (major, minor) = (version_info_parts.next()?, version_info_parts.next()?);
+            PythonVersion::try_from((major, minor)).ok()
+        });
+
+        let metadata = Self {
+            venv_path,
+            base_executable_home_path,
+            include_system_site_packages,
+            version,
+        };
+
+        tracing::trace!("Resolved metadata for virtual environment: {metadata:?}");
+        Ok(metadata)
+    }
+
+    /// Return a list of `site-packages` directories that are available from this virtual environment
+    ///
+    /// See the documentation for `site_packages_dir_from_sys_prefix` for more details.
+    pub fn site_packages_directories(
+        &self,
+        system: &dyn System,
+    ) -> SitePackagesDiscoveryResult<Vec<SystemPathBuf>> {
+        let VirtualEnvironment {
+            venv_path,
+            base_executable_home_path,
+            include_system_site_packages,
+            version,
+        } = self;
+
+        let mut site_packages_directories = vec![site_packages_directory_from_sys_prefix(
+            venv_path, *version, system,
+        )?];
+
+        if *include_system_site_packages {
+            let system_sys_prefix =
+                SysPrefixPath::from_executable_home_path(base_executable_home_path);
+
+            // If we fail to resolve the `sys.prefix` path from the base executable home path,
+            // or if we fail to resolve the `site-packages` from the `sys.prefix` path,
+            // we should probably print a warning but *not* abort type checking
+            if let Some(sys_prefix_path) = system_sys_prefix {
+                match site_packages_directory_from_sys_prefix(&sys_prefix_path, *version, system) {
+                    Ok(site_packages_directory) => {
+                        site_packages_directories.push(site_packages_directory);
+                    }
+                    Err(error) => tracing::warn!(
+                        "{error}. System site-packages will not be used for module resolution."
+                    ),
+                }
+            } else {
+                tracing::warn!(
+                    "Failed to resolve `sys.prefix` of the system Python installation \
+from the `home` value in the `pyvenv.cfg` file at {}. \
+System site-packages will not be used for module resolution.",
+                    venv_path.join("pyvenv.cfg")
+                );
+            }
+        }
+
+        tracing::debug!("Resolved site-packages directories for this virtual environment are: {site_packages_directories:?}");
+        Ok(site_packages_directories)
+    }
+}
+
+#[derive(Debug, thiserror::Error)]
+pub enum SitePackagesDiscoveryError {
+    #[error("Invalid --venv-path argument: {0} could not be canonicalized")]
+    VenvDirCanonicalizationError(SystemPathBuf, #[source] io::Error),
+    #[error("Invalid --venv-path argument: {0} does not point to a directory on disk")]
+    VenvDirIsNotADirectory(SystemPathBuf),
+    #[error("--venv-path points to a broken venv with no pyvenv.cfg file")]
+    NoPyvenvCfgFile(#[source] io::Error),
+    #[error("Failed to parse the pyvenv.cfg file at {0} because {1}")]
+    PyvenvCfgParseError(SystemPathBuf, PyvenvCfgParseErrorKind),
+    #[error("Failed to search the `lib` directory of the Python installation at {1} for `site-packages`")]
+    CouldNotReadLibDirectory(#[source] io::Error, SysPrefixPath),
+    #[error("Could not find the `site-packages` directory for the Python installation at {0}")]
+    NoSitePackagesDirFound(SysPrefixPath),
+}
+
+/// The various ways in which parsing a `pyvenv.cfg` file could fail
+#[derive(Debug)]
+pub enum PyvenvCfgParseErrorKind {
+    TooManyEquals { line_number: NonZeroUsize },
+    MalformedKeyValuePair { line_number: NonZeroUsize },
+    NoHomeKey,
+    InvalidHomeValue(io::Error),
+}
+
+impl fmt::Display for PyvenvCfgParseErrorKind {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            Self::TooManyEquals { line_number } => {
+                write!(f, "line {line_number} has too many '=' characters")
+            }
+            Self::MalformedKeyValuePair { line_number } => write!(
+                f,
+                "line {line_number} has a malformed `<key> = <value>` pair"
+            ),
+            Self::NoHomeKey => f.write_str("the file does not have a `home` key"),
+            Self::InvalidHomeValue(io_err) => {
+                write!(
+                    f,
+                    "the following error was encountered \
+when trying to resolve the `home` value to a directory on disk: {io_err}"
+                )
+            }
+        }
+    }
+}
+
 /// Attempt to retrieve the `site-packages` directory
 /// associated with a given Python installation.
 ///
-/// `sys_prefix_path` is equivalent to the value of [`sys.prefix`]
-/// at runtime in Python. For the case of a virtual environment, where a
-/// Python binary is at `/.venv/bin/python`, `sys.prefix` is the path to
-/// the virtual environment the Python binary lies inside, i.e. `/.venv`,
-/// and `site-packages` will be at `.venv/lib/python3.X/site-packages`.
-/// System Python installations generally work the same way: if a system
-/// Python installation lies at `/opt/homebrew/bin/python`, `sys.prefix`
-/// will be `/opt/homebrew`, and `site-packages` will be at
-/// `/opt/homebrew/lib/python3.X/site-packages`.
-///
-/// This routine does not verify that `sys_prefix_path` points
-/// to an existing directory on disk; it is assumed that this has already
-/// been checked.
-///
-/// [`sys.prefix`]: https://docs.python.org/3/library/sys.html#sys.prefix
-fn site_packages_dir_from_sys_prefix(
-    sys_prefix_path: &SystemPath,
+/// The location of the `site-packages` directory can vary according to the
+/// Python version that this installation represents. The Python version may
+/// or may not be known at this point, which is why the `python_version`
+/// parameter is an `Option`.
+fn site_packages_directory_from_sys_prefix(
+    sys_prefix_path: &SysPrefixPath,
+    python_version: Option<PythonVersion>,
     system: &dyn System,
-) -> Result<SystemPathBuf, SitePackagesDiscoveryError> {
-    tracing::debug!("Searching for site-packages directory in '{sys_prefix_path}'");
+) -> SitePackagesDiscoveryResult<SystemPathBuf> {
+    tracing::debug!("Searching for site-packages directory in {sys_prefix_path}");
 
     if cfg!(target_os = "windows") {
-        let site_packages = sys_prefix_path.join("Lib/site-packages");
-
-        return if system.is_directory(&site_packages) {
-            tracing::debug!("Resolved site-packages directory to '{site_packages}'");
-            Ok(site_packages)
-        } else {
-            Err(SitePackagesDiscoveryError::NoSitePackagesDirFound)
-        };
+        let site_packages = sys_prefix_path.join(r"Lib\site-packages");
+        return system
+            .is_directory(&site_packages)
+            .then_some(site_packages)
+            .ok_or(SitePackagesDiscoveryError::NoSitePackagesDirFound(
+                sys_prefix_path.to_owned(),
+            ));
     }
 
     // In the Python standard library's `site.py` module (used for finding `site-packages`
@@ -69,7 +296,38 @@ fn site_packages_dir_from_sys_prefix(
     //
     // [the non-Windows branch]: https://github.com/python/cpython/blob/a8be8fc6c4682089be45a87bd5ee1f686040116c/Lib/site.py#L401-L410
     // [the `sys`-module documentation]: https://docs.python.org/3/library/sys.html#sys.platlibdir
-    for entry_result in system.read_directory(&sys_prefix_path.join("lib"))? {
+
+    // If we were able to figure out what Python version this installation is,
+    // we should be able to avoid iterating through all items in the `lib/` directory:
+    if let Some(version) = python_version {
+        let expected_path = sys_prefix_path.join(format!("lib/python{version}/site-packages"));
+        if system.is_directory(&expected_path) {
+            return Ok(expected_path);
+        }
+        if version.free_threaded_build_available() {
+            // Nearly the same as `expected_path`, but with an additional `t` after {version}:
+            let alternative_path =
+                sys_prefix_path.join(format!("lib/python{version}t/site-packages"));
+            if system.is_directory(&alternative_path) {
+                return Ok(alternative_path);
+            }
+        }
+    }
+
+    // Either we couldn't figure out the version before calling this function
+    // (e.g., from a `pyvenv.cfg` file if this was a venv),
+    // or we couldn't find a `site-packages` folder at the expected location given
+    // the parsed version
+    //
+    // Note: the `python3.x` part of the `site-packages` path can't be computed from
+    // the `--target-version` the user has passed, as they might be running Python 3.12 locally
+    // even if they've requested that we type check their code "as if" they're running 3.8.
+    for entry_result in system
+        .read_directory(&sys_prefix_path.join("lib"))
+        .map_err(|io_err| {
+            SitePackagesDiscoveryError::CouldNotReadLibDirectory(io_err, sys_prefix_path.to_owned())
+        })?
+    {
         let Ok(entry) = entry_result else {
             continue;
         };
@@ -80,16 +338,6 @@ fn site_packages_dir_from_sys_prefix(
 
         let mut path = entry.into_path();
 
-        // The `python3.x` part of the `site-packages` path can't be computed from
-        // the `--target-version` the user has passed, as they might be running Python 3.12 locally
-        // even if they've requested that we type check their code "as if" they're running 3.8.
-        //
-        // The `python3.x` part of the `site-packages` path *could* be computed
-        // by parsing the virtual environment's `pyvenv.cfg` file.
-        // Right now that seems like overkill, but in the future we may need to parse
-        // the `pyvenv.cfg` file anyway, in which case we could switch to that method
-        // rather than iterating through the whole directory until we find
-        // an entry where the last component of the path starts with `python3.`
         let name = path
             .file_name()
             .expect("File name to be non-null because path is guaranteed to be a child of `lib`");
@@ -100,55 +348,494 @@ fn site_packages_dir_from_sys_prefix(
 
         path.push("site-packages");
         if system.is_directory(&path) {
-            tracing::debug!("Resolved site-packages directory to '{path}'");
             return Ok(path);
         }
     }
-    Err(SitePackagesDiscoveryError::NoSitePackagesDirFound)
+    Err(SitePackagesDiscoveryError::NoSitePackagesDirFound(
+        sys_prefix_path.to_owned(),
+    ))
 }
 
-#[derive(Debug, thiserror::Error)]
-pub enum SitePackagesDiscoveryError {
-    #[error("Failed to search the virtual environment directory for `site-packages`")]
-    CouldNotReadLibDirectory(#[from] io::Error),
-    #[error("Could not find the `site-packages` directory in the virtual environment")]
-    NoSitePackagesDirFound,
+/// A path that represents the value of [`sys.prefix`] at runtime in Python
+/// for a given Python executable.
+///
+/// For the case of a virtual environment, where a
+/// Python binary is at `/.venv/bin/python`, `sys.prefix` is the path to
+/// the virtual environment the Python binary lies inside, i.e. `/.venv`,
+/// and `site-packages` will be at `.venv/lib/python3.X/site-packages`.
+/// System Python installations generally work the same way: if a system
+/// Python installation lies at `/opt/homebrew/bin/python`, `sys.prefix`
+/// will be `/opt/homebrew`, and `site-packages` will be at
+/// `/opt/homebrew/lib/python3.X/site-packages`.
+///
+/// [`sys.prefix`]: https://docs.python.org/3/library/sys.html#sys.prefix
+#[derive(Debug, PartialEq, Eq, Clone)]
+pub struct SysPrefixPath(SystemPathBuf);
+
+impl SysPrefixPath {
+    fn new(
+        unvalidated_path: impl AsRef<SystemPath>,
+        system: &dyn System,
+    ) -> SitePackagesDiscoveryResult<Self> {
+        Self::new_impl(unvalidated_path.as_ref(), system)
+    }
+
+    fn new_impl(
+        unvalidated_path: &SystemPath,
+        system: &dyn System,
+    ) -> SitePackagesDiscoveryResult<Self> {
+        // It's important to resolve symlinks here rather than simply making the path absolute,
+        // since system Python installations often only put symlinks in the "expected"
+        // locations for `home` and `site-packages`
+        let canonicalized = system
+            .canonicalize_path(unvalidated_path)
+            .map_err(|io_err| {
+                SitePackagesDiscoveryError::VenvDirCanonicalizationError(
+                    unvalidated_path.to_path_buf(),
+                    io_err,
+                )
+            })?;
+        system
+            .is_directory(&canonicalized)
+            .then_some(Self(canonicalized))
+            .ok_or_else(|| {
+                SitePackagesDiscoveryError::VenvDirIsNotADirectory(unvalidated_path.to_path_buf())
+            })
+    }
+
+    fn from_executable_home_path(path: &PythonHomePath) -> Option<Self> {
+        // No need to check whether `path.parent()` is a directory:
+        // the parent of a canonicalised path that is known to exist
+        // is guaranteed to be a directory.
+        if cfg!(target_os = "windows") {
+            Some(Self(path.to_path_buf()))
+        } else {
+            path.parent().map(|path| Self(path.to_path_buf()))
+        }
+    }
 }
 
-/// Given a validated, canonicalized path to a virtual environment,
-/// return a list of `site-packages` directories that are available from that environment.
+impl Deref for SysPrefixPath {
+    type Target = SystemPath;
+
+    fn deref(&self) -> &Self::Target {
+        &self.0
+    }
+}
+
+impl fmt::Display for SysPrefixPath {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "`sys.prefix` path {}", self.0)
+    }
+}
+
+/// The value given by the `home` key in `pyvenv.cfg` files.
 ///
-/// See the documentation for `site_packages_dir_from_sys_prefix` for more details.
+/// This is equivalent to `{sys_prefix_path}/bin`, and points
+/// to a directory in which a Python executable can be found.
+/// Confusingly, it is *not* the same as the [`PYTHONHOME`]
+/// environment variable that Python provides! However, it's
+/// consistent among all mainstream creators of Python virtual
+/// environments (the stdlib Python `venv` module, the third-party
+/// `virtualenv` library, and `uv`), was specified by
+/// [the original PEP adding the `venv` module],
+/// and it's one of the few fields that's read by the Python
+/// standard library's `site.py` module.
 ///
-/// TODO: Currently we only ever return 1 path from this function:
-/// the `site-packages` directory that is actually inside the virtual environment.
-/// Some `site-packages` directories are able to also access system `site-packages` and
-/// user `site-packages`, however.
-pub fn site_packages_dirs_of_venv(
-    venv_path: &SystemPath,
-    system: &dyn System,
-) -> Result<Vec<SystemPathBuf>, SitePackagesDiscoveryError> {
-    Ok(vec![site_packages_dir_from_sys_prefix(venv_path, system)?])
+/// Although it doesn't appear to be specified anywhere,
+/// all existing virtual environment tools always use an absolute path
+/// for the `home` value, and the Python standard library also assumes
+/// that the `home` value will be an absolute path.
+///
+/// Other values, such as the path to the Python executable or the
+/// base-executable `sys.prefix` value, are either only provided in
+/// `pyvenv.cfg` files by some virtual-environment creators,
+/// or are included under different keys depending on which
+/// virtual-environment creation tool you've used.
+///
+/// [`PYTHONHOME`]: https://docs.python.org/3/using/cmdline.html#envvar-PYTHONHOME
+/// [the original PEP adding the `venv` module]: https://peps.python.org/pep-0405/
+#[derive(Debug, PartialEq, Eq)]
+struct PythonHomePath(SystemPathBuf);
+
+impl PythonHomePath {
+    fn new(path: impl AsRef<SystemPath>, system: &dyn System) -> io::Result<Self> {
+        let path = path.as_ref();
+        // It's important to resolve symlinks here rather than simply making the path absolute,
+        // since system Python installations often only put symlinks in the "expected"
+        // locations for `home` and `site-packages`
+        let canonicalized = system.canonicalize_path(path)?;
+        system
+            .is_directory(&canonicalized)
+            .then_some(Self(canonicalized))
+            .ok_or_else(|| io::Error::new(io::ErrorKind::Other, "not a directory"))
+    }
+}
+
+impl Deref for PythonHomePath {
+    type Target = SystemPath;
+
+    fn deref(&self) -> &Self::Target {
+        &self.0
+    }
+}
+
+impl fmt::Display for PythonHomePath {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "`home` location {}", self.0)
+    }
+}
+
+impl PartialEq<SystemPath> for PythonHomePath {
+    fn eq(&self, other: &SystemPath) -> bool {
+        &*self.0 == other
+    }
+}
+
+impl PartialEq<SystemPathBuf> for PythonHomePath {
+    fn eq(&self, other: &SystemPathBuf) -> bool {
+        self == &**other
+    }
 }
 
 #[cfg(test)]
 mod tests {
-    use ruff_db::system::{OsSystem, System, SystemPath};
+    use ruff_db::system::TestSystem;
 
-    use crate::site_packages::site_packages_dirs_of_venv;
+    use super::*;
+
+    struct VirtualEnvironmentTester {
+        system: TestSystem,
+        minor_version: u8,
+        free_threaded: bool,
+        system_site_packages: bool,
+        pyvenv_cfg_version_field: Option<&'static str>,
+    }
+
+    impl VirtualEnvironmentTester {
+        /// Builds a mock virtual environment, and returns the path to the venv
+        fn build_mock_venv(&self) -> SystemPathBuf {
+            let VirtualEnvironmentTester {
+                system,
+                minor_version,
+                system_site_packages,
+                free_threaded,
+                pyvenv_cfg_version_field,
+            } = self;
+            let memory_fs = system.memory_file_system();
+            let unix_site_packages = if *free_threaded {
+                format!("lib/python3.{minor_version}t/site-packages")
+            } else {
+                format!("lib/python3.{minor_version}/site-packages")
+            };
+
+            let system_install_sys_prefix =
+                SystemPathBuf::from(&*format!("/Python3.{minor_version}"));
+            let (system_home_path, system_exe_path, system_site_packages_path) =
+                if cfg!(target_os = "windows") {
+                    let system_home_path = system_install_sys_prefix.clone();
+                    let system_exe_path = system_home_path.join("python.exe");
+                    let system_site_packages_path =
+                        system_install_sys_prefix.join(r"Lib\site-packages");
+                    (system_home_path, system_exe_path, system_site_packages_path)
+                } else {
+                    let system_home_path = system_install_sys_prefix.join("bin");
+                    let system_exe_path = system_home_path.join("python");
+                    let system_site_packages_path =
+                        system_install_sys_prefix.join(&unix_site_packages);
+                    (system_home_path, system_exe_path, system_site_packages_path)
+                };
+            memory_fs.write_file(system_exe_path, "").unwrap();
+            memory_fs
+                .create_directory_all(&system_site_packages_path)
+                .unwrap();
+
+            let venv_sys_prefix = SystemPathBuf::from("/.venv");
+            let (venv_exe, site_packages_path) = if cfg!(target_os = "windows") {
+                (
+                    venv_sys_prefix.join(r"Scripts\python.exe"),
+                    venv_sys_prefix.join(r"Lib\site-packages"),
+                )
+            } else {
+                (
+                    venv_sys_prefix.join("bin/python"),
+                    venv_sys_prefix.join(&unix_site_packages),
+                )
+            };
+            memory_fs.write_file(&venv_exe, "").unwrap();
+            memory_fs.create_directory_all(&site_packages_path).unwrap();
+
+            let pyvenv_cfg_path = venv_sys_prefix.join("pyvenv.cfg");
+            let mut pyvenv_cfg_contents = format!("home = {system_home_path}\n");
+            if let Some(version_field) = pyvenv_cfg_version_field {
+                pyvenv_cfg_contents.push_str(version_field);
+                pyvenv_cfg_contents.push('\n');
+            }
+            // Deliberately using weird casing here to test that our pyvenv.cfg parsing is case-insensitive:
+            if *system_site_packages {
+                pyvenv_cfg_contents.push_str("include-system-site-packages = TRuE\n");
+            }
+            memory_fs
+                .write_file(pyvenv_cfg_path, &pyvenv_cfg_contents)
+                .unwrap();
+
+            venv_sys_prefix
+        }
+
+        fn test(self) {
+            let venv_path = self.build_mock_venv();
+            let venv = VirtualEnvironment::new(venv_path.clone(), &self.system).unwrap();
+
+            assert_eq!(
+                venv.venv_path,
+                SysPrefixPath(self.system.canonicalize_path(&venv_path).unwrap())
+            );
+            assert_eq!(venv.include_system_site_packages, self.system_site_packages);
+
+            if self.pyvenv_cfg_version_field.is_some() {
+                assert_eq!(
+                    venv.version,
+                    Some(PythonVersion {
+                        major: 3,
+                        minor: self.minor_version
+                    })
+                );
+            } else {
+                assert_eq!(venv.version, None);
+            }
+
+            let expected_home = if cfg!(target_os = "windows") {
+                SystemPathBuf::from(&*format!(r"\Python3.{}", self.minor_version))
+            } else {
+                SystemPathBuf::from(&*format!("/Python3.{}/bin", self.minor_version))
+            };
+            assert_eq!(venv.base_executable_home_path, expected_home);
+
+            let site_packages_directories = venv.site_packages_directories(&self.system).unwrap();
+            let expected_venv_site_packages = if cfg!(target_os = "windows") {
+                SystemPathBuf::from(r"\.venv\Lib\site-packages")
+            } else if self.free_threaded {
+                SystemPathBuf::from(&*format!(
+                    "/.venv/lib/python3.{}t/site-packages",
+                    self.minor_version
+                ))
+            } else {
+                SystemPathBuf::from(&*format!(
+                    "/.venv/lib/python3.{}/site-packages",
+                    self.minor_version
+                ))
+            };
+
+            let expected_system_site_packages = if cfg!(target_os = "windows") {
+                SystemPathBuf::from(&*format!(
+                    r"\Python3.{}\Lib\site-packages",
+                    self.minor_version
+                ))
+            } else if self.free_threaded {
+                SystemPathBuf::from(&*format!(
+                    "/Python3.{minor_version}/lib/python3.{minor_version}t/site-packages",
+                    minor_version = self.minor_version
+                ))
+            } else {
+                SystemPathBuf::from(&*format!(
+                    "/Python3.{minor_version}/lib/python3.{minor_version}/site-packages",
+                    minor_version = self.minor_version
+                ))
+            };
+
+            if self.system_site_packages {
+                assert_eq!(
+                    &site_packages_directories,
+                    &[expected_venv_site_packages, expected_system_site_packages]
+                );
+            } else {
+                assert_eq!(&site_packages_directories, &[expected_venv_site_packages]);
+            }
+        }
+    }
 
     #[test]
-    // Windows venvs have different layouts, and we only have a Unix venv committed for now.
-    // This test is skipped on Windows until we commit a Windows venv.
-    #[cfg_attr(target_os = "windows", ignore = "Windows has a different venv layout")]
-    fn can_find_site_packages_dir_in_committed_venv() {
-        let path_to_venv = SystemPath::new("resources/test/unix-uv-venv");
-        let system = OsSystem::default();
+    fn can_find_site_packages_directory_no_version_field_in_pyvenv_cfg() {
+        let tester = VirtualEnvironmentTester {
+            system: TestSystem::default(),
+            minor_version: 12,
+            free_threaded: false,
+            system_site_packages: false,
+            pyvenv_cfg_version_field: None,
+        };
+        tester.test();
+    }
 
-        // if this doesn't hold true, the premise of the test is incorrect.
-        assert!(system.is_directory(path_to_venv));
+    #[test]
+    fn can_find_site_packages_directory_venv_style_version_field_in_pyvenv_cfg() {
+        let tester = VirtualEnvironmentTester {
+            system: TestSystem::default(),
+            minor_version: 12,
+            free_threaded: false,
+            system_site_packages: false,
+            pyvenv_cfg_version_field: Some("version = 3.12"),
+        };
+        tester.test();
+    }
 
-        let site_packages_dirs = site_packages_dirs_of_venv(path_to_venv, &system).unwrap();
-        assert_eq!(site_packages_dirs.len(), 1);
+    #[test]
+    fn can_find_site_packages_directory_uv_style_version_field_in_pyvenv_cfg() {
+        let tester = VirtualEnvironmentTester {
+            system: TestSystem::default(),
+            minor_version: 12,
+            free_threaded: false,
+            system_site_packages: false,
+            pyvenv_cfg_version_field: Some("version_info = 3.12"),
+        };
+        tester.test();
+    }
+
+    #[test]
+    fn can_find_site_packages_directory_virtualenv_style_version_field_in_pyvenv_cfg() {
+        let tester = VirtualEnvironmentTester {
+            system: TestSystem::default(),
+            minor_version: 12,
+            free_threaded: false,
+            system_site_packages: false,
+            pyvenv_cfg_version_field: Some("version_info = 3.12.0rc2"),
+        };
+        tester.test();
+    }
+
+    #[test]
+    fn can_find_site_packages_directory_freethreaded_build() {
+        let tester = VirtualEnvironmentTester {
+            system: TestSystem::default(),
+            minor_version: 13,
+            free_threaded: true,
+            system_site_packages: false,
+            pyvenv_cfg_version_field: Some("version_info = 3.13"),
+        };
+        tester.test();
+    }
+
+    #[test]
+    fn finds_system_site_packages() {
+        let tester = VirtualEnvironmentTester {
+            system: TestSystem::default(),
+            minor_version: 13,
+            free_threaded: true,
+            system_site_packages: true,
+            pyvenv_cfg_version_field: Some("version_info = 3.13"),
+        };
+        tester.test();
+    }
+
+    #[test]
+    fn reject_venv_that_does_not_exist() {
+        let system = TestSystem::default();
+        assert!(matches!(
+            VirtualEnvironment::new("/.venv", &system),
+            Err(SitePackagesDiscoveryError::VenvDirIsNotADirectory(_))
+        ));
+    }
+
+    #[test]
+    fn reject_venv_with_no_pyvenv_cfg_file() {
+        let system = TestSystem::default();
+        system
+            .memory_file_system()
+            .create_directory_all("/.venv")
+            .unwrap();
+        assert!(matches!(
+            VirtualEnvironment::new("/.venv", &system),
+            Err(SitePackagesDiscoveryError::NoPyvenvCfgFile(_))
+        ));
+    }
+
+    #[test]
+    fn parsing_pyvenv_cfg_with_too_many_equals() {
+        let system = TestSystem::default();
+        let memory_fs = system.memory_file_system();
+        let pyvenv_cfg_path = SystemPathBuf::from("/.venv/pyvenv.cfg");
+        memory_fs
+            .write_file(&pyvenv_cfg_path, "home = bar = /.venv/bin")
+            .unwrap();
+        let venv_result = VirtualEnvironment::new("/.venv", &system);
+        assert!(matches!(
+            venv_result,
+            Err(SitePackagesDiscoveryError::PyvenvCfgParseError(
+                path,
+                PyvenvCfgParseErrorKind::TooManyEquals { line_number }
+            ))
+            if path == pyvenv_cfg_path && Some(line_number) == NonZeroUsize::new(1)
+        ));
+    }
+
+    #[test]
+    fn parsing_pyvenv_cfg_with_key_but_no_value_fails() {
+        let system = TestSystem::default();
+        let memory_fs = system.memory_file_system();
+        let pyvenv_cfg_path = SystemPathBuf::from("/.venv/pyvenv.cfg");
+        memory_fs.write_file(&pyvenv_cfg_path, "home =").unwrap();
+        let venv_result = VirtualEnvironment::new("/.venv", &system);
+        assert!(matches!(
+            venv_result,
+            Err(SitePackagesDiscoveryError::PyvenvCfgParseError(
+                path,
+                PyvenvCfgParseErrorKind::MalformedKeyValuePair { line_number }
+            ))
+            if path == pyvenv_cfg_path && Some(line_number) == NonZeroUsize::new(1)
+        ));
+    }
+
+    #[test]
+    fn parsing_pyvenv_cfg_with_value_but_no_key_fails() {
+        let system = TestSystem::default();
+        let memory_fs = system.memory_file_system();
+        let pyvenv_cfg_path = SystemPathBuf::from("/.venv/pyvenv.cfg");
+        memory_fs
+            .write_file(&pyvenv_cfg_path, "= whatever")
+            .unwrap();
+        let venv_result = VirtualEnvironment::new("/.venv", &system);
+        assert!(matches!(
+            venv_result,
+            Err(SitePackagesDiscoveryError::PyvenvCfgParseError(
+                path,
+                PyvenvCfgParseErrorKind::MalformedKeyValuePair { line_number }
+            ))
+            if path == pyvenv_cfg_path && Some(line_number) == NonZeroUsize::new(1)
+        ));
+    }
+
+    #[test]
+    fn parsing_pyvenv_cfg_with_no_home_key_fails() {
+        let system = TestSystem::default();
+        let memory_fs = system.memory_file_system();
+        let pyvenv_cfg_path = SystemPathBuf::from("/.venv/pyvenv.cfg");
+        memory_fs.write_file(&pyvenv_cfg_path, "").unwrap();
+        let venv_result = VirtualEnvironment::new("/.venv", &system);
+        assert!(matches!(
+            venv_result,
+            Err(SitePackagesDiscoveryError::PyvenvCfgParseError(
+                path,
+                PyvenvCfgParseErrorKind::NoHomeKey
+            ))
+            if path == pyvenv_cfg_path
+        ));
+    }
+
+    #[test]
+    fn parsing_pyvenv_cfg_with_invalid_home_key_fails() {
+        let system = TestSystem::default();
+        let memory_fs = system.memory_file_system();
+        let pyvenv_cfg_path = SystemPathBuf::from("/.venv/pyvenv.cfg");
+        memory_fs
+            .write_file(&pyvenv_cfg_path, "home = foo")
+            .unwrap();
+        let venv_result = VirtualEnvironment::new("/.venv", &system);
+        assert!(matches!(
+            venv_result,
+            Err(SitePackagesDiscoveryError::PyvenvCfgParseError(
+                path,
+                PyvenvCfgParseErrorKind::InvalidHomeValue(_)
+            ))
+            if path == pyvenv_cfg_path
+        ));
     }
 }

crates/red_knot_workspace/src/workspace.rs

@@ -1,9 +1,10 @@
-use salsa::{Durability, Setter as _};
 use std::{collections::BTreeMap, sync::Arc};
 
-use rustc_hash::{FxBuildHasher, FxHashSet};
+use foldhash::{HashMapExt, HashSet, HashSetExt};
+use salsa::{Durability, Setter as _};
 
 pub use metadata::{PackageMetadata, WorkspaceMetadata};
+use ruff_db::source::{source_text, SourceDiagnostic};
 use ruff_db::{
     files::{system_path_to_file, File},
     system::{walk_directory::WalkState, SystemPath, SystemPathBuf},
@@ -73,7 +74,7 @@ pub struct Workspace {
     /// open files rather than all files in the workspace.
     #[return_ref]
     #[default]
-    open_fileset: Option<Arc<FxHashSet<File>>>,
+    open_fileset: Option<Arc<HashSet<File>>>,
 
     /// The (first-party) packages in this workspace.
     #[return_ref]
@@ -218,7 +219,7 @@ impl Workspace {
     }
 
     /// Returns the open files in the workspace or `None` if the entire workspace should be checked.
-    pub fn open_files(self, db: &dyn Db) -> Option<&FxHashSet<File>> {
+    pub fn open_files(self, db: &dyn Db) -> Option<&HashSet<File>> {
         self.open_fileset(db).as_deref()
     }
 
@@ -226,7 +227,7 @@ impl Workspace {
     ///
     /// This changes the behavior of `check` to only check the open files rather than all files in the workspace.
     #[tracing::instrument(level = "debug", skip(self, db))]
-    pub fn set_open_files(self, db: &mut dyn Db, open_files: FxHashSet<File>) {
+    pub fn set_open_files(self, db: &mut dyn Db, open_files: HashSet<File>) {
         tracing::debug!("Set open workspace files (count: {})", open_files.len());
 
         self.set_open_fileset(db).to(Some(Arc::new(open_files)));
@@ -235,7 +236,7 @@ impl Workspace {
     /// This takes the open files from the workspace and returns them.
     ///
     /// This changes the behavior of `check` to check all files in the workspace instead of just the open files.
-    pub fn take_open_files(self, db: &mut dyn Db) -> FxHashSet<File> {
+    pub fn take_open_files(self, db: &mut dyn Db) -> HashSet<File> {
         tracing::debug!("Take open workspace files");
 
         // Salsa will cancel any pending queries and remove its own reference to `open_files`
@@ -245,7 +246,7 @@ impl Workspace {
         if let Some(open_files) = open_files {
             Arc::try_unwrap(open_files).unwrap()
         } else {
-            FxHashSet::default()
+            HashSet::default()
         }
     }
 }
@@ -345,18 +346,33 @@ impl Package {
     }
 }
 
+#[salsa::tracked]
 pub(super) fn check_file(db: &dyn Db, file: File) -> Diagnostics {
     let path = file.path(db);
     let _span = tracing::debug_span!("check_file", file=%path).entered();
     tracing::debug!("Checking file {path}");
 
     let mut diagnostics = Vec::new();
+
+    let source_diagnostics = source_text::accumulated::<SourceDiagnostic>(db.upcast(), file);
+    // TODO(micha): Consider using a single accumulator for all diagnostics
+    diagnostics.extend(
+        source_diagnostics
+            .iter()
+            .map(std::string::ToString::to_string),
+    );
+
+    // Abort checking if there are IO errors.
+    if source_text(db.upcast(), file).has_read_error() {
+        return Diagnostics::from(diagnostics);
+    }
+
     diagnostics.extend_from_slice(lint_syntax(db, file));
     diagnostics.extend_from_slice(lint_semantic(db, file));
     Diagnostics::from(diagnostics)
 }
 
-fn discover_package_files(db: &dyn Db, path: &SystemPath) -> FxHashSet<File> {
+fn discover_package_files(db: &dyn Db, path: &SystemPath) -> HashSet<File> {
     let paths = std::sync::Mutex::new(Vec::new());
 
     db.system().walk_directory(path).run(|| {
@@ -386,7 +402,7 @@ fn discover_package_files(db: &dyn Db, path: &SystemPath) -> FxHashSet<File> {
     });
 
     let paths = paths.into_inner().unwrap();
-    let mut files = FxHashSet::with_capacity_and_hasher(paths.len(), FxBuildHasher);
+    let mut files = HashSet::with_capacity(paths.len());
 
     for path in paths {
         // If this returns `None`, then the file was deleted between the `walk_directory` call and now.
@@ -398,3 +414,48 @@ fn discover_package_files(db: &dyn Db, path: &SystemPath) -> FxHashSet<File> {
 
     files
 }
+
+#[cfg(test)]
+mod tests {
+    use ruff_db::files::system_path_to_file;
+    use ruff_db::source::source_text;
+    use ruff_db::system::{DbWithTestSystem, SystemPath};
+    use ruff_db::testing::assert_function_query_was_not_run;
+
+    use crate::db::tests::TestDb;
+    use crate::lint::{lint_syntax, Diagnostics};
+    use crate::workspace::check_file;
+
+    #[test]
+    fn check_file_skips_linting_when_file_cant_be_read() -> ruff_db::system::Result<()> {
+        let mut db = TestDb::new();
+        let path = SystemPath::new("test.py");
+
+        db.write_file(path, "x = 10")?;
+        let file = system_path_to_file(&db, path).unwrap();
+
+        // Now the file gets deleted before we had a chance to read its source text.
+        db.memory_file_system().remove_file(path)?;
+        file.sync(&mut db);
+
+        assert_eq!(source_text(&db, file).as_str(), "");
+        assert_eq!(
+            check_file(&db, file),
+            Diagnostics::List(vec![
+                "Failed to read file: No such file or directory".to_string()
+            ])
+        );
+
+        let events = db.take_salsa_events();
+        assert_function_query_was_not_run(&db, lint_syntax, file, &events);
+
+        // The user now creates a new file with an empty text. The source text
+        // content returned by `source_text` remains unchanged, but the diagnostics should get updated.
+        db.write_file(path, "").unwrap();
+
+        assert_eq!(source_text(&db, file).as_str(), "");
+        assert_eq!(check_file(&db, file), Diagnostics::Empty);
+
+        Ok(())
+    }
+}

crates/red_knot_workspace/src/workspace/files.rs

@@ -2,7 +2,7 @@ use std::iter::FusedIterator;
 use std::ops::Deref;
 use std::sync::Arc;
 
-use rustc_hash::FxHashSet;
+use foldhash::HashSet;
 use salsa::Setter;
 
 use ruff_db::files::File;
@@ -105,7 +105,7 @@ pub struct LazyFiles<'a> {
 
 impl<'a> LazyFiles<'a> {
     /// Sets the indexed files of a package to `files`.
-    pub fn set(mut self, files: FxHashSet<File>) -> IndexedFiles {
+    pub fn set(mut self, files: HashSet<File>) -> IndexedFiles {
         let files = IndexedFiles::new(files);
         *self.files = State::Indexed(files.clone());
         files
@@ -127,11 +127,11 @@ impl<'a> LazyFiles<'a> {
 #[derive(Debug, Clone)]
 pub struct IndexedFiles {
     revision: u64,
-    files: Arc<std::sync::Mutex<FxHashSet<File>>>,
+    files: Arc<std::sync::Mutex<HashSet<File>>>,
 }
 
 impl IndexedFiles {
-    fn new(files: FxHashSet<File>) -> Self {
+    fn new(files: HashSet<File>) -> Self {
         Self {
             files: Arc::new(std::sync::Mutex::new(files)),
             revision: 0,
@@ -155,11 +155,11 @@ impl PartialEq for IndexedFiles {
 impl Eq for IndexedFiles {}
 
 pub struct IndexedFilesGuard<'a> {
-    guard: std::sync::MutexGuard<'a, FxHashSet<File>>,
+    guard: std::sync::MutexGuard<'a, HashSet<File>>,
 }
 
 impl Deref for IndexedFilesGuard<'_> {
-    type Target = FxHashSet<File>;
+    type Target = HashSet<File>;
 
     fn deref(&self) -> &Self::Target {
         &self.guard