Try to reduce false positives more

Co-authored-by: Micha Reiser <micha@reiser.io>

CHANGELOG.md

@@ -1,5 +1,39 @@
 # Changelog
 
+## 0.6.2
+
+### Preview features
+
+- \[`flake8-simplify`\] Extend `open-file-with-context-handler` to work with other standard-library IO modules (`SIM115`) ([#12959](https://github.com/astral-sh/ruff/pull/12959))
+- \[`ruff`\] Avoid `unused-async` for functions with FastAPI route decorator (`RUF029`) ([#12938](https://github.com/astral-sh/ruff/pull/12938))
+- \[`ruff`\] Ignore `fstring-missing-syntax` (`RUF027`) for `fastAPI` paths ([#12939](https://github.com/astral-sh/ruff/pull/12939))
+- \[`ruff`\] Implement check for Decimal called with a float literal (RUF032) ([#12909](https://github.com/astral-sh/ruff/pull/12909))
+
+### Rule changes
+
+- \[`flake8-bugbear`\] Update diagnostic message when expression is at the end of function (`B015`) ([#12944](https://github.com/astral-sh/ruff/pull/12944))
+- \[`flake8-pyi`\] Skip type annotations in `string-or-bytes-too-long` (`PYI053`) ([#13002](https://github.com/astral-sh/ruff/pull/13002))
+- \[`flake8-type-checking`\] Always recognise relative imports as first-party ([#12994](https://github.com/astral-sh/ruff/pull/12994))
+- \[`flake8-unused-arguments`\] Ignore unused arguments on stub functions (`ARG001`) ([#12966](https://github.com/astral-sh/ruff/pull/12966))
+- \[`pylint`\] Ignore augmented assignment for `self-cls-assignment` (`PLW0642`) ([#12957](https://github.com/astral-sh/ruff/pull/12957))
+
+### Server
+
+- Show full context in error log messages ([#13029](https://github.com/astral-sh/ruff/pull/13029))
+
+### Bug fixes
+
+- \[`pep8-naming`\] Don't flag `from` imports following conventional import names (`N817`) ([#12946](https://github.com/astral-sh/ruff/pull/12946))
+- \[`pylint`\] - Allow `__new__` methods to have `cls` as their first argument even if decorated with `@staticmethod` for `bad-staticmethod-argument` (`PLW0211`) ([#12958](https://github.com/astral-sh/ruff/pull/12958))
+
+### Documentation
+
+- Add `hyperfine` installation instructions; update `hyperfine` code samples ([#13034](https://github.com/astral-sh/ruff/pull/13034))
+- Expand note to use Ruff with other language server in Kate ([#12806](https://github.com/astral-sh/ruff/pull/12806))
+- Update example for `PT001` as per the new default behavior ([#13019](https://github.com/astral-sh/ruff/pull/13019))
+- \[`perflint`\] Improve docs for `try-except-in-loop` (`PERF203`) ([#12947](https://github.com/astral-sh/ruff/pull/12947))
+- \[`pydocstyle`\] Add reference to `lint.pydocstyle.ignore-decorators` setting to rule docs ([#12996](https://github.com/astral-sh/ruff/pull/12996))
+
 ## 0.6.1
 
 This is a hotfix release to address an issue with `ruff-pre-commit`. In v0.6,

CONTRIBUTING.md

@@ -397,12 +397,18 @@ which makes it a good target for benchmarking.
 git clone --branch 3.10 https://github.com/python/cpython.git crates/ruff_linter/resources/test/cpython
 ```
 
+Install `hyperfine`:
+
+```shell
+cargo install hyperfine
+```
+
 To benchmark the release build:
 
 ```shell
 cargo build --release && hyperfine --warmup 10 \
-  "./target/release/ruff ./crates/ruff_linter/resources/test/cpython/ --no-cache -e" \
-  "./target/release/ruff ./crates/ruff_linter/resources/test/cpython/ -e"
+  "./target/release/ruff check ./crates/ruff_linter/resources/test/cpython/ --no-cache -e" \
+  "./target/release/ruff check ./crates/ruff_linter/resources/test/cpython/ -e"
 
 Benchmark 1: ./target/release/ruff ./crates/ruff_linter/resources/test/cpython/ --no-cache
   Time (mean ± σ):     293.8 ms ±   3.2 ms    [User: 2384.6 ms, System: 90.3 ms]
@@ -421,7 +427,7 @@ To benchmark against the ecosystem's existing tools:
 
 ```shell
 hyperfine --ignore-failure --warmup 5 \
-  "./target/release/ruff ./crates/ruff_linter/resources/test/cpython/ --no-cache" \
+  "./target/release/ruff check ./crates/ruff_linter/resources/test/cpython/ --no-cache" \
   "pyflakes crates/ruff_linter/resources/test/cpython" \
   "autoflake --recursive --expand-star-imports --remove-all-unused-imports --remove-unused-variables --remove-duplicate-keys resources/test/cpython" \
   "pycodestyle crates/ruff_linter/resources/test/cpython" \
@@ -467,7 +473,7 @@ To benchmark a subset of rules, e.g. `LineTooLong` and `DocLineTooLong`:
 
 ```shell
 cargo build --release && hyperfine --warmup 10 \
-  "./target/release/ruff ./crates/ruff_linter/resources/test/cpython/ --no-cache -e --select W505,E501"
+  "./target/release/ruff check ./crates/ruff_linter/resources/test/cpython/ --no-cache -e --select W505,E501"
 ```
 
 You can run `poetry install` from `./scripts/benchmarks` to create a working environment for the

Cargo.lock

@@ -1926,6 +1926,7 @@ dependencies = [
  "ruff_db",
  "ruff_index",
  "ruff_python_ast",
+ "ruff_python_literal",
  "ruff_python_parser",
  "ruff_python_stdlib",
  "ruff_source_file",
@@ -1935,6 +1936,7 @@ dependencies = [
  "smallvec",
  "static_assertions",
  "tempfile",
+ "thiserror",
  "tracing",
  "walkdir",
  "zip",
@@ -1950,7 +1952,6 @@ dependencies = [
  "libc",
  "lsp-server",
  "lsp-types",
- "red_knot_python_semantic",
  "red_knot_workspace",
  "ruff_db",
  "ruff_notebook",
@@ -1995,7 +1996,6 @@ dependencies = [
  "ruff_text_size",
  "rustc-hash 2.0.0",
  "salsa",
- "thiserror",
  "tracing",
 ]
 
@@ -2089,7 +2089,7 @@ dependencies = [
 
 [[package]]
 name = "ruff"
-version = "0.6.1"
+version = "0.6.2"
 dependencies = [
  "anyhow",
  "argfile",
@@ -2281,7 +2281,7 @@ dependencies = [
 
 [[package]]
 name = "ruff_linter"
-version = "0.6.1"
+version = "0.6.2"
 dependencies = [
  "aho-corasick",
  "annotate-snippets 0.9.2",
@@ -2601,7 +2601,7 @@ dependencies = [
 
 [[package]]
 name = "ruff_wasm"
-version = "0.6.1"
+version = "0.6.2"
 dependencies = [
  "console_error_panic_hook",
  "console_log",

README.md

@@ -136,8 +136,8 @@ curl -LsSf https://astral.sh/ruff/install.sh | sh
 powershell -c "irm https://astral.sh/ruff/install.ps1 | iex"
 
 # For a specific version.
-curl -LsSf https://astral.sh/ruff/0.6.1/install.sh | sh
-powershell -c "irm https://astral.sh/ruff/0.6.1/install.ps1 | iex"
+curl -LsSf https://astral.sh/ruff/0.6.2/install.sh | sh
+powershell -c "irm https://astral.sh/ruff/0.6.2/install.ps1 | iex"
 ```
 
 You can also install Ruff via [Homebrew](https://formulae.brew.sh/formula/ruff), [Conda](https://anaconda.org/conda-forge/ruff),
@@ -170,7 +170,7 @@ Ruff can also be used as a [pre-commit](https://pre-commit.com/) hook via [`ruff
 ```yaml
 - repo: https://github.com/astral-sh/ruff-pre-commit
   # Ruff version.
-  rev: v0.6.1
+  rev: v0.6.2
   hooks:
     # Run the linter.
     - id: ruff

crates/red_knot/src/main.rs

@@ -7,12 +7,12 @@ use colored::Colorize;
 use crossbeam::channel as crossbeam_channel;
 use salsa::plumbing::ZalsaDatabase;
 
-use red_knot_python_semantic::{ProgramSettings, SearchPathSettings};
+use red_knot_python_semantic::SitePackages;
 use red_knot_server::run_server;
 use red_knot_workspace::db::RootDatabase;
-use red_knot_workspace::site_packages::VirtualEnvironment;
 use red_knot_workspace::watch;
 use red_knot_workspace::watch::WorkspaceWatcher;
+use red_knot_workspace::workspace::settings::Configuration;
 use red_knot_workspace::workspace::WorkspaceMetadata;
 use ruff_db::system::{OsSystem, System, SystemPath, SystemPathBuf};
 use target_version::TargetVersion;
@@ -65,15 +65,14 @@ to resolve type information for the project's third-party dependencies.",
         value_name = "PATH",
         help = "Additional path to use as a module-resolution source (can be passed multiple times)"
     )]
-    extra_search_path: Vec<SystemPathBuf>,
+    extra_search_path: Option<Vec<SystemPathBuf>>,
 
     #[arg(
         long,
         help = "Python version to assume when resolving types",
-        default_value_t = TargetVersion::default(),
-        value_name="VERSION")
-    ]
-    target_version: TargetVersion,
+        value_name = "VERSION"
+    )]
+    target_version: Option<TargetVersion>,
 
     #[clap(flatten)]
     verbosity: Verbosity,
@@ -86,6 +85,36 @@ to resolve type information for the project's third-party dependencies.",
     watch: bool,
 }
 
+impl Args {
+    fn to_configuration(&self, cli_cwd: &SystemPath) -> Configuration {
+        let mut configuration = Configuration::default();
+
+        if let Some(target_version) = self.target_version {
+            configuration.target_version = Some(target_version.into());
+        }
+
+        if let Some(venv_path) = &self.venv_path {
+            configuration.search_paths.site_packages = Some(SitePackages::Derived {
+                venv_path: SystemPath::absolute(venv_path, cli_cwd),
+            });
+        }
+
+        if let Some(custom_typeshed_dir) = &self.custom_typeshed_dir {
+            configuration.search_paths.custom_typeshed =
+                Some(SystemPath::absolute(custom_typeshed_dir, cli_cwd));
+        }
+
+        if let Some(extra_search_paths) = &self.extra_search_path {
+            configuration.search_paths.extra_paths = extra_search_paths
+                .iter()
+                .map(|path| Some(SystemPath::absolute(path, cli_cwd)))
+                .collect();
+        }
+
+        configuration
+    }
+}
+
 #[derive(Debug, clap::Subcommand)]
 pub enum Command {
     /// Start the language server
@@ -115,22 +144,13 @@ pub fn main() -> ExitStatus {
 }
 
 fn run() -> anyhow::Result<ExitStatus> {
-    let Args {
-        command,
-        current_directory,
-        custom_typeshed_dir,
-        extra_search_path: extra_paths,
-        venv_path,
-        target_version,
-        verbosity,
-        watch,
-    } = Args::parse_from(std::env::args().collect::<Vec<_>>());
+    let args = Args::parse_from(std::env::args().collect::<Vec<_>>());
 
-    if matches!(command, Some(Command::Server)) {
+    if matches!(args.command, Some(Command::Server)) {
         return run_server().map(|()| ExitStatus::Success);
     }
 
-    let verbosity = verbosity.level();
+    let verbosity = args.verbosity.level();
     countme::enable(verbosity.is_trace());
     let _guard = setup_tracing(verbosity)?;
 
@@ -146,10 +166,12 @@ fn run() -> anyhow::Result<ExitStatus> {
             })?
     };
 
-    let cwd = current_directory
+    let cwd = args
+        .current_directory
+        .as_ref()
         .map(|cwd| {
             if cwd.as_std_path().is_dir() {
-                Ok(SystemPath::absolute(&cwd, &cli_base_path))
+                Ok(SystemPath::absolute(cwd, &cli_base_path))
             } else {
                 Err(anyhow!(
                     "Provided current-directory path '{cwd}' is not a directory."
@@ -160,33 +182,18 @@ fn run() -> anyhow::Result<ExitStatus> {
         .unwrap_or_else(|| cli_base_path.clone());
 
     let system = OsSystem::new(cwd.clone());
-    let workspace_metadata = WorkspaceMetadata::from_path(system.current_directory(), &system)?;
-
-    // TODO: Verify the remaining search path settings eagerly.
-    let site_packages = venv_path
-        .map(|path| {
-            VirtualEnvironment::new(path, &OsSystem::new(cli_base_path))
-                .and_then(|venv| venv.site_packages_directories(&system))
-        })
-        .transpose()?
-        .unwrap_or_default();
-
-    // TODO: Respect the settings from the workspace metadata. when resolving the program settings.
-    let program_settings = ProgramSettings {
-        target_version: target_version.into(),
-        search_paths: SearchPathSettings {
-            extra_paths,
-            src_root: workspace_metadata.root().to_path_buf(),
-            custom_typeshed: custom_typeshed_dir,
-            site_packages,
-        },
-    };
+    let cli_configuration = args.to_configuration(&cwd);
+    let workspace_metadata = WorkspaceMetadata::from_path(
+        system.current_directory(),
+        &system,
+        Some(cli_configuration.clone()),
+    )?;
 
     // TODO: Use the `program_settings` to compute the key for the database's persistent
     //   cache and load the cache if it exists.
-    let mut db = RootDatabase::new(workspace_metadata, program_settings, system)?;
+    let mut db = RootDatabase::new(workspace_metadata, system)?;
 
-    let (main_loop, main_loop_cancellation_token) = MainLoop::new();
+    let (main_loop, main_loop_cancellation_token) = MainLoop::new(cli_configuration);
 
     // Listen to Ctrl+C and abort the watch mode.
     let main_loop_cancellation_token = Mutex::new(Some(main_loop_cancellation_token));
@@ -198,7 +205,7 @@ fn run() -> anyhow::Result<ExitStatus> {
         }
     })?;
 
-    let exit_status = if watch {
+    let exit_status = if args.watch {
         main_loop.watch(&mut db)?
     } else {
         main_loop.run(&mut db)
@@ -238,10 +245,12 @@ struct MainLoop {
 
     /// The file system watcher, if running in watch mode.
     watcher: Option<WorkspaceWatcher>,
+
+    cli_configuration: Configuration,
 }
 
 impl MainLoop {
-    fn new() -> (Self, MainLoopCancellationToken) {
+    fn new(cli_configuration: Configuration) -> (Self, MainLoopCancellationToken) {
         let (sender, receiver) = crossbeam_channel::bounded(10);
 
         (
@@ -249,6 +258,7 @@ impl MainLoop {
                 sender: sender.clone(),
                 receiver,
                 watcher: None,
+                cli_configuration,
             },
             MainLoopCancellationToken { sender },
         )
@@ -331,7 +341,7 @@ impl MainLoop {
                 MainLoopMessage::ApplyChanges(changes) => {
                     revision += 1;
                     // Automatically cancels any pending queries and waits for them to complete.
-                    db.apply_changes(changes);
+                    db.apply_changes(changes, Some(&self.cli_configuration));
                     if let Some(watcher) = self.watcher.as_mut() {
                         watcher.update(db);
                     }

crates/red_knot/tests/file_watching.rs

@@ -5,12 +5,11 @@ use std::time::Duration;
 
 use anyhow::{anyhow, Context};
 
-use red_knot_python_semantic::{
-    resolve_module, ModuleName, Program, ProgramSettings, PythonVersion, SearchPathSettings,
-};
+use red_knot_python_semantic::{resolve_module, ModuleName, Program, PythonVersion, SitePackages};
 use red_knot_workspace::db::RootDatabase;
 use red_knot_workspace::watch;
 use red_knot_workspace::watch::{directory_watcher, WorkspaceWatcher};
+use red_knot_workspace::workspace::settings::{Configuration, SearchPathConfiguration};
 use red_knot_workspace::workspace::WorkspaceMetadata;
 use ruff_db::files::{system_path_to_file, File, FileError};
 use ruff_db::source::source_text;
@@ -25,7 +24,7 @@ struct TestCase {
     /// We need to hold on to it in the test case or the temp files get deleted.
     _temp_dir: tempfile::TempDir,
     root_dir: SystemPathBuf,
-    search_path_settings: SearchPathSettings,
+    configuration: Configuration,
 }
 
 impl TestCase {
@@ -41,10 +40,6 @@ impl TestCase {
         &self.db
     }
 
-    fn db_mut(&mut self) -> &mut RootDatabase {
-        &mut self.db
-    }
-
     fn stop_watch(&mut self) -> Vec<watch::ChangeEvent> {
         self.try_stop_watch(Duration::from_secs(10))
             .expect("Expected watch changes but observed none.")
@@ -105,16 +100,20 @@ impl TestCase {
         Some(all_events)
     }
 
+    fn apply_changes(&mut self, changes: Vec<watch::ChangeEvent>) {
+        self.db.apply_changes(changes, Some(&self.configuration));
+    }
+
     fn update_search_path_settings(
         &mut self,
-        f: impl FnOnce(&SearchPathSettings) -> SearchPathSettings,
+        configuration: SearchPathConfiguration,
     ) -> anyhow::Result<()> {
         let program = Program::get(self.db());
 
-        let new_settings = f(&self.search_path_settings);
+        self.configuration.search_paths = configuration.clone();
+        let new_settings = configuration.into_settings(self.db.workspace().root(&self.db));
 
-        program.update_search_paths(&mut self.db, new_settings.clone())?;
-        self.search_path_settings = new_settings;
+        program.update_search_paths(&mut self.db, &new_settings)?;
 
         if let Some(watcher) = &mut self.watcher {
             watcher.update(&self.db);
@@ -179,17 +178,14 @@ fn setup<F>(setup_files: F) -> anyhow::Result<TestCase>
 where
     F: SetupFiles,
 {
-    setup_with_search_paths(setup_files, |_root, workspace_path| SearchPathSettings {
-        extra_paths: vec![],
-        src_root: workspace_path.to_path_buf(),
-        custom_typeshed: None,
-        site_packages: vec![],
+    setup_with_search_paths(setup_files, |_root, _workspace_path| {
+        SearchPathConfiguration::default()
     })
 }
 
 fn setup_with_search_paths<F>(
     setup_files: F,
-    create_search_paths: impl FnOnce(&SystemPath, &SystemPath) -> SearchPathSettings,
+    create_search_paths: impl FnOnce(&SystemPath, &SystemPath) -> SearchPathConfiguration,
 ) -> anyhow::Result<TestCase>
 where
     F: SetupFiles,
@@ -221,25 +217,34 @@ where
 
     let system = OsSystem::new(&workspace_path);
 
-    let workspace = WorkspaceMetadata::from_path(&workspace_path, &system)?;
-    let search_path_settings = create_search_paths(&root_path, workspace.root());
+    let search_paths = create_search_paths(&root_path, &workspace_path);
 
-    for path in search_path_settings
+    for path in search_paths
         .extra_paths
         .iter()
-        .chain(search_path_settings.site_packages.iter())
-        .chain(search_path_settings.custom_typeshed.iter())
+        .flatten()
+        .chain(search_paths.custom_typeshed.iter())
+        .chain(search_paths.site_packages.iter().flat_map(|site_packages| {
+            if let SitePackages::Known(path) = site_packages {
+                path.as_slice()
+            } else {
+                &[]
+            }
+        }))
     {
         std::fs::create_dir_all(path.as_std_path())
             .with_context(|| format!("Failed to create search path '{path}'"))?;
     }
 
-    let settings = ProgramSettings {
-        target_version: PythonVersion::default(),
-        search_paths: search_path_settings.clone(),
+    let configuration = Configuration {
+        target_version: Some(PythonVersion::PY312),
+        search_paths,
     };
 
-    let db = RootDatabase::new(workspace, settings, system)?;
+    let workspace =
+        WorkspaceMetadata::from_path(&workspace_path, &system, Some(configuration.clone()))?;
+
+    let db = RootDatabase::new(workspace, system)?;
 
     let (sender, receiver) = crossbeam::channel::unbounded();
     let watcher = directory_watcher(move |events| sender.send(events).unwrap())
@@ -254,7 +259,7 @@ where
         watcher: Some(watcher),
         _temp_dir: temp_dir,
         root_dir: root_path,
-        search_path_settings,
+        configuration,
     };
 
     // Sometimes the file watcher reports changes for events that happened before the watcher was started.
@@ -307,7 +312,7 @@ fn new_file() -> anyhow::Result<()> {
 
     let changes = case.stop_watch();
 
-    case.db_mut().apply_changes(changes);
+    case.apply_changes(changes);
 
     let foo = case.system_file(&foo_path).expect("foo.py to exist.");
 
@@ -330,7 +335,7 @@ fn new_ignored_file() -> anyhow::Result<()> {
 
     let changes = case.stop_watch();
 
-    case.db_mut().apply_changes(changes);
+    case.apply_changes(changes);
 
     assert!(case.system_file(&foo_path).is_ok());
     assert_eq!(&case.collect_package_files(&bar_path), &[bar_file]);
@@ -354,7 +359,7 @@ fn changed_file() -> anyhow::Result<()> {
 
     assert!(!changes.is_empty());
 
-    case.db_mut().apply_changes(changes);
+    case.apply_changes(changes);
 
     assert_eq!(source_text(case.db(), foo).as_str(), "print('Version 2')");
     assert_eq!(&case.collect_package_files(&foo_path), &[foo]);
@@ -377,7 +382,7 @@ fn deleted_file() -> anyhow::Result<()> {
 
     let changes = case.stop_watch();
 
-    case.db_mut().apply_changes(changes);
+    case.apply_changes(changes);
 
     assert!(!foo.exists(case.db()));
     assert_eq!(&case.collect_package_files(&foo_path), &[] as &[File]);
@@ -409,7 +414,7 @@ fn move_file_to_trash() -> anyhow::Result<()> {
 
     let changes = case.stop_watch();
 
-    case.db_mut().apply_changes(changes);
+    case.apply_changes(changes);
 
     assert!(!foo.exists(case.db()));
     assert_eq!(&case.collect_package_files(&foo_path), &[] as &[File]);
@@ -441,7 +446,7 @@ fn move_file_to_workspace() -> anyhow::Result<()> {
 
     let changes = case.stop_watch();
 
-    case.db_mut().apply_changes(changes);
+    case.apply_changes(changes);
 
     let foo_in_workspace = case.system_file(&foo_in_workspace_path)?;
 
@@ -469,7 +474,7 @@ fn rename_file() -> anyhow::Result<()> {
 
     let changes = case.stop_watch();
 
-    case.db_mut().apply_changes(changes);
+    case.apply_changes(changes);
 
     assert!(!foo.exists(case.db()));
 
@@ -510,7 +515,7 @@ fn directory_moved_to_workspace() -> anyhow::Result<()> {
 
     let changes = case.stop_watch();
 
-    case.db_mut().apply_changes(changes);
+    case.apply_changes(changes);
 
     let init_file = case
         .system_file(sub_new_path.join("__init__.py"))
@@ -561,7 +566,7 @@ fn directory_moved_to_trash() -> anyhow::Result<()> {
 
     let changes = case.stop_watch();
 
-    case.db_mut().apply_changes(changes);
+    case.apply_changes(changes);
 
     // `import sub.a` should no longer resolve
     assert!(resolve_module(case.db().upcast(), ModuleName::new_static("sub.a").unwrap()).is_none());
@@ -615,7 +620,7 @@ fn directory_renamed() -> anyhow::Result<()> {
 
     let changes = case.stop_watch();
 
-    case.db_mut().apply_changes(changes);
+    case.apply_changes(changes);
 
     // `import sub.a` should no longer resolve
     assert!(resolve_module(case.db().upcast(), ModuleName::new_static("sub.a").unwrap()).is_none());
@@ -680,7 +685,7 @@ fn directory_deleted() -> anyhow::Result<()> {
 
     let changes = case.stop_watch();
 
-    case.db_mut().apply_changes(changes);
+    case.apply_changes(changes);
 
     // `import sub.a` should no longer resolve
     assert!(resolve_module(case.db().upcast(), ModuleName::new_static("sub.a").unwrap()).is_none());
@@ -694,15 +699,13 @@ fn directory_deleted() -> anyhow::Result<()> {
 
 #[test]
 fn search_path() -> anyhow::Result<()> {
-    let mut case =
-        setup_with_search_paths([("bar.py", "import sub.a")], |root_path, workspace_path| {
-            SearchPathSettings {
-                extra_paths: vec![],
-                src_root: workspace_path.to_path_buf(),
-                custom_typeshed: None,
-                site_packages: vec![root_path.join("site_packages")],
-            }
-        })?;
+    let mut case = setup_with_search_paths(
+        [("bar.py", "import sub.a")],
+        |root_path, _workspace_path| SearchPathConfiguration {
+            site_packages: Some(SitePackages::Known(vec![root_path.join("site_packages")])),
+            ..SearchPathConfiguration::default()
+        },
+    )?;
 
     let site_packages = case.root_path().join("site_packages");
 
@@ -715,7 +718,7 @@ fn search_path() -> anyhow::Result<()> {
 
     let changes = case.stop_watch();
 
-    case.db_mut().apply_changes(changes);
+    case.apply_changes(changes);
 
     assert!(resolve_module(case.db().upcast(), ModuleName::new_static("a").unwrap()).is_some());
     assert_eq!(
@@ -736,9 +739,9 @@ fn add_search_path() -> anyhow::Result<()> {
     assert!(resolve_module(case.db().upcast(), ModuleName::new_static("a").unwrap()).is_none());
 
     // Register site-packages as a search path.
-    case.update_search_path_settings(|settings| SearchPathSettings {
-        site_packages: vec![site_packages.clone()],
-        ..settings.clone()
+    case.update_search_path_settings(SearchPathConfiguration {
+        site_packages: Some(SitePackages::Known(vec![site_packages.clone()])),
+        ..SearchPathConfiguration::default()
     })
     .expect("Search path settings to be valid");
 
@@ -746,7 +749,7 @@ fn add_search_path() -> anyhow::Result<()> {
 
     let changes = case.stop_watch();
 
-    case.db_mut().apply_changes(changes);
+    case.apply_changes(changes);
 
     assert!(resolve_module(case.db().upcast(), ModuleName::new_static("a").unwrap()).is_some());
 
@@ -755,21 +758,19 @@ fn add_search_path() -> anyhow::Result<()> {
 
 #[test]
 fn remove_search_path() -> anyhow::Result<()> {
-    let mut case =
-        setup_with_search_paths([("bar.py", "import sub.a")], |root_path, workspace_path| {
-            SearchPathSettings {
-                extra_paths: vec![],
-                src_root: workspace_path.to_path_buf(),
-                custom_typeshed: None,
-                site_packages: vec![root_path.join("site_packages")],
-            }
-        })?;
+    let mut case = setup_with_search_paths(
+        [("bar.py", "import sub.a")],
+        |root_path, _workspace_path| SearchPathConfiguration {
+            site_packages: Some(SitePackages::Known(vec![root_path.join("site_packages")])),
+            ..SearchPathConfiguration::default()
+        },
+    )?;
 
     // Remove site packages from the search path settings.
     let site_packages = case.root_path().join("site_packages");
-    case.update_search_path_settings(|settings| SearchPathSettings {
-        site_packages: vec![],
-        ..settings.clone()
+    case.update_search_path_settings(SearchPathConfiguration {
+        site_packages: None,
+        ..SearchPathConfiguration::default()
     })
     .expect("Search path settings to be valid");
 
@@ -782,6 +783,48 @@ fn remove_search_path() -> anyhow::Result<()> {
     Ok(())
 }
 
+#[test]
+fn changed_versions_file() -> anyhow::Result<()> {
+    let mut case = setup_with_search_paths(
+        |root_path: &SystemPath, workspace_path: &SystemPath| {
+            std::fs::write(workspace_path.join("bar.py").as_std_path(), "import sub.a")?;
+            std::fs::create_dir_all(root_path.join("typeshed/stdlib").as_std_path())?;
+            std::fs::write(root_path.join("typeshed/stdlib/VERSIONS").as_std_path(), "")?;
+            std::fs::write(
+                root_path.join("typeshed/stdlib/os.pyi").as_std_path(),
+                "# not important",
+            )?;
+
+            Ok(())
+        },
+        |root_path, _workspace_path| SearchPathConfiguration {
+            custom_typeshed: Some(root_path.join("typeshed")),
+            ..SearchPathConfiguration::default()
+        },
+    )?;
+
+    // Unset the custom typeshed directory.
+    assert_eq!(
+        resolve_module(case.db(), ModuleName::new("os").unwrap()),
+        None
+    );
+
+    std::fs::write(
+        case.root_path()
+            .join("typeshed/stdlib/VERSIONS")
+            .as_std_path(),
+        "os: 3.0-",
+    )?;
+
+    let changes = case.stop_watch();
+
+    case.apply_changes(changes);
+
+    assert!(resolve_module(case.db(), ModuleName::new("os").unwrap()).is_some());
+
+    Ok(())
+}
+
 /// Watch a workspace that contains two files where one file is a hardlink to another.
 ///
 /// Setup:
@@ -828,7 +871,7 @@ fn hard_links_in_workspace() -> anyhow::Result<()> {
 
     let changes = case.stop_watch();
 
-    case.db_mut().apply_changes(changes);
+    case.apply_changes(changes);
 
     assert_eq!(source_text(case.db(), foo).as_str(), "print('Version 2')");
 
@@ -899,7 +942,7 @@ fn hard_links_to_target_outside_workspace() -> anyhow::Result<()> {
 
     let changes = case.stop_watch();
 
-    case.db_mut().apply_changes(changes);
+    case.apply_changes(changes);
 
     assert_eq!(source_text(case.db(), bar).as_str(), "print('Version 2')");
 
@@ -938,7 +981,7 @@ mod unix {
 
         let changes = case.stop_watch();
 
-        case.db_mut().apply_changes(changes);
+        case.apply_changes(changes);
 
         assert_eq!(
             foo.permissions(case.db()),
@@ -1023,7 +1066,7 @@ mod unix {
 
         let changes = case.take_watch_changes();
 
-        case.db_mut().apply_changes(changes);
+        case.apply_changes(changes);
 
         assert_eq!(
             source_text(case.db(), baz.file()).as_str(),
@@ -1036,7 +1079,7 @@ mod unix {
 
         let changes = case.stop_watch();
 
-        case.db_mut().apply_changes(changes);
+        case.apply_changes(changes);
 
         assert_eq!(
             source_text(case.db(), baz.file()).as_str(),
@@ -1107,7 +1150,7 @@ mod unix {
 
         let changes = case.stop_watch();
 
-        case.db_mut().apply_changes(changes);
+        case.apply_changes(changes);
 
         // The file watcher is guaranteed to emit one event for the changed file, but it isn't specified
         // if the event is emitted for the "original" or linked path because both paths are watched.
@@ -1176,11 +1219,11 @@ mod unix {
 
                 Ok(())
             },
-            |_root, workspace| SearchPathSettings {
-                extra_paths: vec![],
-                src_root: workspace.to_path_buf(),
-                custom_typeshed: None,
-                site_packages: vec![workspace.join(".venv/lib/python3.12/site-packages")],
+            |_root, workspace| SearchPathConfiguration {
+                site_packages: Some(SitePackages::Known(vec![
+                    workspace.join(".venv/lib/python3.12/site-packages")
+                ])),
+                ..SearchPathConfiguration::default()
             },
         )?;
 
@@ -1215,7 +1258,7 @@ mod unix {
 
         let changes = case.stop_watch();
 
-        case.db_mut().apply_changes(changes);
+        case.apply_changes(changes);
 
         assert_eq!(
             source_text(case.db(), baz_original_file).as_str(),

crates/red_knot_python_semantic/Cargo.toml

@@ -17,6 +17,7 @@ ruff_python_ast = { workspace = true }
 ruff_python_stdlib = { workspace = true }
 ruff_source_file = { workspace = true }
 ruff_text_size = { workspace = true }
+ruff_python_literal = { workspace = true }
 
 anyhow = { workspace = true }
 bitflags = { workspace = true }
@@ -26,6 +27,7 @@ countme = { workspace = true }
 once_cell = { workspace = true }
 ordermap = { workspace = true }
 salsa = { workspace = true }
+thiserror = { workspace = true }
 tracing = { workspace = true }
 rustc-hash = { workspace = true }
 hashbrown = { workspace = true }

crates/red_knot_python_semantic/src/lib.rs

@@ -5,7 +5,7 @@ use rustc_hash::FxHasher;
 pub use db::Db;
 pub use module_name::ModuleName;
 pub use module_resolver::{resolve_module, system_module_search_paths, vendored_typeshed_stubs};
-pub use program::{Program, ProgramSettings, SearchPathSettings};
+pub use program::{Program, ProgramSettings, SearchPathSettings, SitePackages};
 pub use python_version::PythonVersion;
 pub use semantic_model::{HasTy, SemanticModel};
 
@@ -19,6 +19,7 @@ mod program;
 mod python_version;
 pub mod semantic_index;
 mod semantic_model;
+pub(crate) mod site_packages;
 pub mod types;
 
 type FxOrderSet<V> = ordermap::set::OrderSet<V, BuildHasherDefault<FxHasher>>;

crates/red_knot_python_semantic/src/module_resolver/mod.rs

@@ -13,7 +13,6 @@ use resolver::SearchPathIterator;
 mod module;
 mod path;
 mod resolver;
-mod state;
 mod typeshed;
 
 #[cfg(test)]

crates/red_knot_python_semantic/src/module_resolver/path.rs

@@ -9,11 +9,11 @@ use ruff_db::files::{system_path_to_file, vendored_path_to_file, File, FileError
 use ruff_db::system::{System, SystemPath, SystemPathBuf};
 use ruff_db::vendored::{VendoredPath, VendoredPathBuf};
 
+use super::typeshed::{typeshed_versions, TypeshedVersionsParseError, TypeshedVersionsQueryResult};
 use crate::db::Db;
 use crate::module_name::ModuleName;
-
-use super::state::ResolverState;
-use super::typeshed::{TypeshedVersionsParseError, TypeshedVersionsQueryResult};
+use crate::module_resolver::resolver::ResolverContext;
+use crate::site_packages::SitePackagesDiscoveryError;
 
 /// A path that points to a Python module.
 ///
@@ -60,7 +60,7 @@ impl ModulePath {
     }
 
     #[must_use]
-    pub(crate) fn is_directory(&self, resolver: &ResolverState) -> bool {
+    pub(super) fn is_directory(&self, resolver: &ResolverContext) -> bool {
         let ModulePath {
             search_path,
             relative_path,
@@ -74,7 +74,7 @@ impl ModulePath {
                     == Err(FileError::IsADirectory)
             }
             SearchPathInner::StandardLibraryCustom(stdlib_root) => {
-                match query_stdlib_version(Some(stdlib_root), relative_path, resolver) {
+                match query_stdlib_version(relative_path, resolver) {
                     TypeshedVersionsQueryResult::DoesNotExist => false,
                     TypeshedVersionsQueryResult::Exists
                     | TypeshedVersionsQueryResult::MaybeExists => {
@@ -84,7 +84,7 @@ impl ModulePath {
                 }
             }
             SearchPathInner::StandardLibraryVendored(stdlib_root) => {
-                match query_stdlib_version(None, relative_path, resolver) {
+                match query_stdlib_version(relative_path, resolver) {
                     TypeshedVersionsQueryResult::DoesNotExist => false,
                     TypeshedVersionsQueryResult::Exists
                     | TypeshedVersionsQueryResult::MaybeExists => resolver
@@ -96,7 +96,7 @@ impl ModulePath {
     }
 
     #[must_use]
-    pub(crate) fn is_regular_package(&self, resolver: &ResolverState) -> bool {
+    pub(super) fn is_regular_package(&self, resolver: &ResolverContext) -> bool {
         let ModulePath {
             search_path,
             relative_path,
@@ -113,7 +113,7 @@ impl ModulePath {
                         .is_ok()
             }
             SearchPathInner::StandardLibraryCustom(search_path) => {
-                match query_stdlib_version(Some(search_path), relative_path, resolver) {
+                match query_stdlib_version(relative_path, resolver) {
                     TypeshedVersionsQueryResult::DoesNotExist => false,
                     TypeshedVersionsQueryResult::Exists
                     | TypeshedVersionsQueryResult::MaybeExists => system_path_to_file(
@@ -124,7 +124,7 @@ impl ModulePath {
                 }
             }
             SearchPathInner::StandardLibraryVendored(search_path) => {
-                match query_stdlib_version(None, relative_path, resolver) {
+                match query_stdlib_version(relative_path, resolver) {
                     TypeshedVersionsQueryResult::DoesNotExist => false,
                     TypeshedVersionsQueryResult::Exists
                     | TypeshedVersionsQueryResult::MaybeExists => resolver
@@ -136,7 +136,7 @@ impl ModulePath {
     }
 
     #[must_use]
-    pub(crate) fn to_file(&self, resolver: &ResolverState) -> Option<File> {
+    pub(super) fn to_file(&self, resolver: &ResolverContext) -> Option<File> {
         let db = resolver.db.upcast();
         let ModulePath {
             search_path,
@@ -150,7 +150,7 @@ impl ModulePath {
                 system_path_to_file(db, search_path.join(relative_path)).ok()
             }
             SearchPathInner::StandardLibraryCustom(stdlib_root) => {
-                match query_stdlib_version(Some(stdlib_root), relative_path, resolver) {
+                match query_stdlib_version(relative_path, resolver) {
                     TypeshedVersionsQueryResult::DoesNotExist => None,
                     TypeshedVersionsQueryResult::Exists
                     | TypeshedVersionsQueryResult::MaybeExists => {
@@ -159,7 +159,7 @@ impl ModulePath {
                 }
             }
             SearchPathInner::StandardLibraryVendored(stdlib_root) => {
-                match query_stdlib_version(None, relative_path, resolver) {
+                match query_stdlib_version(relative_path, resolver) {
                     TypeshedVersionsQueryResult::DoesNotExist => None,
                     TypeshedVersionsQueryResult::Exists
                     | TypeshedVersionsQueryResult::MaybeExists => {
@@ -273,19 +273,15 @@ fn stdlib_path_to_module_name(relative_path: &Utf8Path) -> Option<ModuleName> {
 
 #[must_use]
 fn query_stdlib_version(
-    custom_stdlib_root: Option<&SystemPath>,
     relative_path: &Utf8Path,
-    resolver: &ResolverState,
+    context: &ResolverContext,
 ) -> TypeshedVersionsQueryResult {
     let Some(module_name) = stdlib_path_to_module_name(relative_path) else {
         return TypeshedVersionsQueryResult::DoesNotExist;
     };
-    let ResolverState {
-        db,
-        typeshed_versions,
-        target_version,
-    } = resolver;
-    typeshed_versions.query_module(*db, &module_name, custom_stdlib_root, *target_version)
+    let ResolverContext { db, target_version } = context;
+
+    typeshed_versions(*db).query_module(&module_name, *target_version)
 }
 
 /// Enumeration describing the various ways in which validation of a search path might fail.
@@ -293,7 +289,7 @@ fn query_stdlib_version(
 /// If validation fails for a search path derived from the user settings,
 /// a message must be displayed to the user,
 /// as type checking cannot be done reliably in these circumstances.
-#[derive(Debug, PartialEq, Eq)]
+#[derive(Debug)]
 pub(crate) enum SearchPathValidationError {
     /// The path provided by the user was not a directory
     NotADirectory(SystemPathBuf),
@@ -304,18 +300,20 @@ pub(crate) enum SearchPathValidationError {
     NoStdlibSubdirectory(SystemPathBuf),
 
     /// The typeshed path provided by the user is a directory,
-    /// but no `stdlib/VERSIONS` file exists.
+    /// but `stdlib/VERSIONS` could not be read.
     /// (This is only relevant for stdlib search paths.)
-    NoVersionsFile(SystemPathBuf),
-
-    /// `stdlib/VERSIONS` is a directory.
-    /// (This is only relevant for stdlib search paths.)
-    VersionsIsADirectory(SystemPathBuf),
+    FailedToReadVersionsFile {
+        path: SystemPathBuf,
+        error: std::io::Error,
+    },
 
     /// The path provided by the user is a directory,
     /// and a `stdlib/VERSIONS` file exists, but it fails to parse.
     /// (This is only relevant for stdlib search paths.)
     VersionsParseError(TypeshedVersionsParseError),
+
+    /// Failed to discover the site-packages for the configured virtual environment.
+    SitePackagesDiscovery(SitePackagesDiscoveryError),
 }
 
 impl fmt::Display for SearchPathValidationError {
@@ -325,9 +323,16 @@ impl fmt::Display for SearchPathValidationError {
             Self::NoStdlibSubdirectory(path) => {
                 write!(f, "The directory at {path} has no `stdlib/` subdirectory")
             }
-            Self::NoVersionsFile(path) => write!(f, "Expected a file at {path}/stdlib/VERSIONS"),
-            Self::VersionsIsADirectory(path) => write!(f, "{path}/stdlib/VERSIONS is a directory."),
+            Self::FailedToReadVersionsFile { path, error } => {
+                write!(
+                    f,
+                    "Failed to read the custom typeshed versions file '{path}': {error}"
+                )
+            }
             Self::VersionsParseError(underlying_error) => underlying_error.fmt(f),
+            SearchPathValidationError::SitePackagesDiscovery(error) => {
+                write!(f, "Failed to discover the site-packages directory: {error}")
+            }
         }
     }
 }
@@ -342,6 +347,18 @@ impl std::error::Error for SearchPathValidationError {
     }
 }
 
+impl From<TypeshedVersionsParseError> for SearchPathValidationError {
+    fn from(value: TypeshedVersionsParseError) -> Self {
+        Self::VersionsParseError(value)
+    }
+}
+
+impl From<SitePackagesDiscoveryError> for SearchPathValidationError {
+    fn from(value: SitePackagesDiscoveryError) -> Self {
+        Self::SitePackagesDiscovery(value)
+    }
+}
+
 type SearchPathResult<T> = Result<T, SearchPathValidationError>;
 
 #[derive(Debug, Clone, PartialEq, Eq, Hash)]
@@ -384,11 +401,10 @@ pub(crate) struct SearchPath(Arc<SearchPathInner>);
 
 impl SearchPath {
     fn directory_path(system: &dyn System, root: SystemPathBuf) -> SearchPathResult<SystemPathBuf> {
-        let canonicalized = system.canonicalize_path(&root).unwrap_or(root);
-        if system.is_directory(&canonicalized) {
-            Ok(canonicalized)
+        if system.is_directory(&root) {
+            Ok(root)
         } else {
-            Err(SearchPathValidationError::NotADirectory(canonicalized))
+            Err(SearchPathValidationError::NotADirectory(root))
         }
     }
 
@@ -407,32 +423,22 @@ impl SearchPath {
     }
 
     /// Create a new standard-library search path pointing to a custom directory on disk
-    pub(crate) fn custom_stdlib(db: &dyn Db, typeshed: SystemPathBuf) -> SearchPathResult<Self> {
+    pub(crate) fn custom_stdlib(db: &dyn Db, typeshed: &SystemPath) -> SearchPathResult<Self> {
         let system = db.system();
-        if !system.is_directory(&typeshed) {
+        if !system.is_directory(typeshed) {
             return Err(SearchPathValidationError::NotADirectory(
                 typeshed.to_path_buf(),
             ));
         }
+
         let stdlib =
             Self::directory_path(system, typeshed.join("stdlib")).map_err(|err| match err {
-                SearchPathValidationError::NotADirectory(path) => {
-                    SearchPathValidationError::NoStdlibSubdirectory(path)
+                SearchPathValidationError::NotADirectory(_) => {
+                    SearchPathValidationError::NoStdlibSubdirectory(typeshed.to_path_buf())
                 }
                 err => err,
             })?;
-        let typeshed_versions =
-            system_path_to_file(db.upcast(), stdlib.join("VERSIONS")).map_err(|err| match err {
-                FileError::NotFound => SearchPathValidationError::NoVersionsFile(typeshed),
-                FileError::IsADirectory => {
-                    SearchPathValidationError::VersionsIsADirectory(typeshed)
-                }
-            })?;
-        super::typeshed::parse_typeshed_versions(db, typeshed_versions)
-            .as_ref()
-            .map_err(|validation_error| {
-                SearchPathValidationError::VersionsParseError(validation_error.clone())
-            })?;
+
         Ok(Self(Arc::new(SearchPathInner::StandardLibraryCustom(
             stdlib,
         ))))
@@ -623,11 +629,11 @@ mod tests {
     use ruff_db::Db;
 
     use crate::db::tests::TestDb;
-
-    use super::*;
     use crate::module_resolver::testing::{FileSpec, MockedTypeshed, TestCase, TestCaseBuilder};
     use crate::python_version::PythonVersion;
 
+    use super::*;
+
     impl ModulePath {
         #[must_use]
         fn join(&self, component: &str) -> ModulePath {
@@ -638,15 +644,6 @@ mod tests {
     }
 
     impl SearchPath {
-        #[must_use]
-        pub(crate) fn is_stdlib_search_path(&self) -> bool {
-            matches!(
-                &*self.0,
-                SearchPathInner::StandardLibraryCustom(_)
-                    | SearchPathInner::StandardLibraryVendored(_)
-            )
-        }
-
         fn join(&self, component: &str) -> ModulePath {
             self.to_module_path().join(component)
         }
@@ -661,7 +658,7 @@ mod tests {
             .build();
 
         assert_eq!(
-            SearchPath::custom_stdlib(&db, stdlib.parent().unwrap().to_path_buf())
+            SearchPath::custom_stdlib(&db, stdlib.parent().unwrap())
                 .unwrap()
                 .to_module_path()
                 .with_py_extension(),
@@ -669,7 +666,7 @@ mod tests {
         );
 
         assert_eq!(
-            &SearchPath::custom_stdlib(&db, stdlib.parent().unwrap().to_path_buf())
+            &SearchPath::custom_stdlib(&db, stdlib.parent().unwrap())
                 .unwrap()
                 .join("foo")
                 .with_pyi_extension(),
@@ -780,7 +777,7 @@ mod tests {
         let TestCase { db, stdlib, .. } = TestCaseBuilder::new()
             .with_custom_typeshed(MockedTypeshed::default())
             .build();
-        SearchPath::custom_stdlib(&db, stdlib.parent().unwrap().to_path_buf())
+        SearchPath::custom_stdlib(&db, stdlib.parent().unwrap())
             .unwrap()
             .to_module_path()
             .push("bar.py");
@@ -792,7 +789,7 @@ mod tests {
         let TestCase { db, stdlib, .. } = TestCaseBuilder::new()
             .with_custom_typeshed(MockedTypeshed::default())
             .build();
-        SearchPath::custom_stdlib(&db, stdlib.parent().unwrap().to_path_buf())
+        SearchPath::custom_stdlib(&db, stdlib.parent().unwrap())
             .unwrap()
             .to_module_path()
             .push("bar.rs");
@@ -824,7 +821,7 @@ mod tests {
             .with_custom_typeshed(MockedTypeshed::default())
             .build();
 
-        let root = SearchPath::custom_stdlib(&db, stdlib.parent().unwrap().to_path_buf()).unwrap();
+        let root = SearchPath::custom_stdlib(&db, stdlib.parent().unwrap()).unwrap();
 
         // Must have a `.pyi` extension or no extension:
         let bad_absolute_path = SystemPath::new("foo/stdlib/x.py");
@@ -872,8 +869,7 @@ mod tests {
             .with_custom_typeshed(typeshed)
             .with_target_version(target_version)
             .build();
-        let stdlib =
-            SearchPath::custom_stdlib(&db, stdlib.parent().unwrap().to_path_buf()).unwrap();
+        let stdlib = SearchPath::custom_stdlib(&db, stdlib.parent().unwrap()).unwrap();
         (db, stdlib)
     }
 
@@ -898,7 +894,7 @@ mod tests {
         };
 
         let (db, stdlib_path) = py38_typeshed_test_case(TYPESHED);
-        let resolver = ResolverState::new(&db, PythonVersion::PY38);
+        let resolver = ResolverContext::new(&db, PythonVersion::PY38);
 
         let asyncio_regular_package = stdlib_path.join("asyncio");
         assert!(asyncio_regular_package.is_directory(&resolver));
@@ -926,7 +922,7 @@ mod tests {
         };
 
         let (db, stdlib_path) = py38_typeshed_test_case(TYPESHED);
-        let resolver = ResolverState::new(&db, PythonVersion::PY38);
+        let resolver = ResolverContext::new(&db, PythonVersion::PY38);
 
         let xml_namespace_package = stdlib_path.join("xml");
         assert!(xml_namespace_package.is_directory(&resolver));
@@ -948,7 +944,7 @@ mod tests {
         };
 
         let (db, stdlib_path) = py38_typeshed_test_case(TYPESHED);
-        let resolver = ResolverState::new(&db, PythonVersion::PY38);
+        let resolver = ResolverContext::new(&db, PythonVersion::PY38);
 
         let functools_module = stdlib_path.join("functools.pyi");
         assert!(functools_module.to_file(&resolver).is_some());
@@ -964,7 +960,7 @@ mod tests {
         };
 
         let (db, stdlib_path) = py38_typeshed_test_case(TYPESHED);
-        let resolver = ResolverState::new(&db, PythonVersion::PY38);
+        let resolver = ResolverContext::new(&db, PythonVersion::PY38);
 
         let collections_regular_package = stdlib_path.join("collections");
         assert_eq!(collections_regular_package.to_file(&resolver), None);
@@ -980,7 +976,7 @@ mod tests {
         };
 
         let (db, stdlib_path) = py38_typeshed_test_case(TYPESHED);
-        let resolver = ResolverState::new(&db, PythonVersion::PY38);
+        let resolver = ResolverContext::new(&db, PythonVersion::PY38);
 
         let importlib_namespace_package = stdlib_path.join("importlib");
         assert_eq!(importlib_namespace_package.to_file(&resolver), None);
@@ -1001,7 +997,7 @@ mod tests {
         };
 
         let (db, stdlib_path) = py38_typeshed_test_case(TYPESHED);
-        let resolver = ResolverState::new(&db, PythonVersion::PY38);
+        let resolver = ResolverContext::new(&db, PythonVersion::PY38);
 
         let non_existent = stdlib_path.join("doesnt_even_exist");
         assert_eq!(non_existent.to_file(&resolver), None);
@@ -1029,7 +1025,7 @@ mod tests {
         };
 
         let (db, stdlib_path) = py39_typeshed_test_case(TYPESHED);
-        let resolver = ResolverState::new(&db, PythonVersion::PY39);
+        let resolver = ResolverContext::new(&db, PythonVersion::PY39);
 
         // Since we've set the target version to Py39,
         // `collections` should now exist as a directory, according to VERSIONS...
@@ -1058,7 +1054,7 @@ mod tests {
         };
 
         let (db, stdlib_path) = py39_typeshed_test_case(TYPESHED);
-        let resolver = ResolverState::new(&db, PythonVersion::PY39);
+        let resolver = ResolverContext::new(&db, PythonVersion::PY39);
 
         // The `importlib` directory now also exists
         let importlib_namespace_package = stdlib_path.join("importlib");
@@ -1082,7 +1078,7 @@ mod tests {
         };
 
         let (db, stdlib_path) = py39_typeshed_test_case(TYPESHED);
-        let resolver = ResolverState::new(&db, PythonVersion::PY39);
+        let resolver = ResolverContext::new(&db, PythonVersion::PY39);
 
         // The `xml` package no longer exists on py39:
         let xml_namespace_package = stdlib_path.join("xml");

crates/red_knot_python_semantic/src/module_resolver/resolver.rs

@@ -1,19 +1,19 @@
+use rustc_hash::{FxBuildHasher, FxHashSet};
 use std::borrow::Cow;
 use std::iter::FusedIterator;
-
-use rustc_hash::{FxBuildHasher, FxHashSet};
+use std::ops::Deref;
 
 use ruff_db::files::{File, FilePath, FileRootKind};
-use ruff_db::system::{DirectoryEntry, SystemPath, SystemPathBuf};
-use ruff_db::vendored::VendoredPath;
-
-use crate::db::Db;
-use crate::module_name::ModuleName;
-use crate::{Program, SearchPathSettings};
+use ruff_db::system::{DirectoryEntry, System, SystemPath, SystemPathBuf};
+use ruff_db::vendored::{VendoredFileSystem, VendoredPath};
 
 use super::module::{Module, ModuleKind};
 use super::path::{ModulePath, SearchPath, SearchPathValidationError};
-use super::state::ResolverState;
+use crate::db::Db;
+use crate::module_name::ModuleName;
+use crate::module_resolver::typeshed::{vendored_typeshed_versions, TypeshedVersions};
+use crate::site_packages::VirtualEnvironment;
+use crate::{Program, PythonVersion, SearchPathSettings, SitePackages};
 
 /// Resolves a module name to a module.
 pub fn resolve_module(db: &dyn Db, module_name: ModuleName) -> Option<Module> {
@@ -122,7 +122,7 @@ pub(crate) fn search_paths(db: &dyn Db) -> SearchPathIterator {
     Program::get(db).search_paths(db).iter(db)
 }
 
-#[derive(Debug, PartialEq, Eq, Default)]
+#[derive(Debug, PartialEq, Eq)]
 pub(crate) struct SearchPaths {
     /// Search paths that have been statically determined purely from reading Ruff's configuration settings.
     /// These shouldn't ever change unless the config settings themselves change.
@@ -135,6 +135,8 @@ pub(crate) struct SearchPaths {
     /// in terms of module-resolution priority until we've discovered the editable installs
     /// for the first `site-packages` path
     site_packages: Vec<SearchPath>,
+
+    typeshed_versions: ResolvedTypeshedVersions,
 }
 
 impl SearchPaths {
@@ -146,8 +148,14 @@ impl SearchPaths {
     /// [module resolution order]: https://typing.readthedocs.io/en/latest/spec/distributing.html#import-resolution-ordering
     pub(crate) fn from_settings(
         db: &dyn Db,
-        settings: SearchPathSettings,
+        settings: &SearchPathSettings,
     ) -> Result<Self, SearchPathValidationError> {
+        fn canonicalize(path: &SystemPath, system: &dyn System) -> SystemPathBuf {
+            system
+                .canonicalize_path(path)
+                .unwrap_or_else(|_| path.to_path_buf())
+        }
+
         let SearchPathSettings {
             extra_paths,
             src_root,
@@ -161,45 +169,65 @@ impl SearchPaths {
         let mut static_paths = vec![];
 
         for path in extra_paths {
-            tracing::debug!("Adding static extra search-path '{path}'");
+            let path = canonicalize(path, system);
+            files.try_add_root(db.upcast(), &path, FileRootKind::LibrarySearchPath);
+            tracing::debug!("Adding extra search-path '{path}'");
 
-            let search_path = SearchPath::extra(system, path)?;
-            files.try_add_root(
-                db.upcast(),
-                search_path.as_system_path().unwrap(),
-                FileRootKind::LibrarySearchPath,
-            );
-            static_paths.push(search_path);
+            static_paths.push(SearchPath::extra(system, path)?);
         }
 
         tracing::debug!("Adding first-party search path '{src_root}'");
-        static_paths.push(SearchPath::first_party(system, src_root)?);
+        static_paths.push(SearchPath::first_party(system, src_root.to_path_buf())?);
 
-        static_paths.push(if let Some(custom_typeshed) = custom_typeshed {
+        let (typeshed_versions, stdlib_path) = if let Some(custom_typeshed) = custom_typeshed {
+            let custom_typeshed = canonicalize(custom_typeshed, system);
             tracing::debug!("Adding custom-stdlib search path '{custom_typeshed}'");
 
-            let search_path = SearchPath::custom_stdlib(db, custom_typeshed)?;
             files.try_add_root(
                 db.upcast(),
-                search_path.as_system_path().unwrap(),
+                &custom_typeshed,
                 FileRootKind::LibrarySearchPath,
             );
-            search_path
+
+            let versions_path = custom_typeshed.join("stdlib/VERSIONS");
+
+            let versions_content = system.read_to_string(&versions_path).map_err(|error| {
+                SearchPathValidationError::FailedToReadVersionsFile {
+                    path: versions_path,
+                    error,
+                }
+            })?;
+
+            let parsed: TypeshedVersions = versions_content.parse()?;
+
+            let search_path = SearchPath::custom_stdlib(db, &custom_typeshed)?;
+
+            (ResolvedTypeshedVersions::Custom(parsed), search_path)
         } else {
-            SearchPath::vendored_stdlib()
-        });
+            tracing::debug!("Using vendored stdlib");
+            (
+                ResolvedTypeshedVersions::Vendored(vendored_typeshed_versions()),
+                SearchPath::vendored_stdlib(),
+            )
+        };
+
+        static_paths.push(stdlib_path);
+
+        let site_packages_paths = match site_packages_paths {
+            SitePackages::Derived { venv_path } => VirtualEnvironment::new(venv_path, system)
+                .and_then(|venv| venv.site_packages_directories(system))?,
+            SitePackages::Known(paths) => paths
+                .iter()
+                .map(|path| canonicalize(path, system))
+                .collect(),
+        };
 
         let mut site_packages: Vec<_> = Vec::with_capacity(site_packages_paths.len());
 
         for path in site_packages_paths {
             tracing::debug!("Adding site-packages search path '{path}'");
-            let search_path = SearchPath::site_packages(system, path)?;
-            files.try_add_root(
-                db.upcast(),
-                search_path.as_system_path().unwrap(),
-                FileRootKind::LibrarySearchPath,
-            );
-            site_packages.push(search_path);
+            files.try_add_root(db.upcast(), &path, FileRootKind::LibrarySearchPath);
+            site_packages.push(SearchPath::site_packages(system, path)?);
         }
 
         // TODO vendor typeshed's third-party stubs as well as the stdlib and fallback to them as a final step
@@ -224,16 +252,48 @@ impl SearchPaths {
         Ok(SearchPaths {
             static_paths,
             site_packages,
+            typeshed_versions,
         })
     }
 
-    pub(crate) fn iter<'a>(&'a self, db: &'a dyn Db) -> SearchPathIterator<'a> {
+    pub(super) fn iter<'a>(&'a self, db: &'a dyn Db) -> SearchPathIterator<'a> {
         SearchPathIterator {
             db,
             static_paths: self.static_paths.iter(),
             dynamic_paths: None,
         }
     }
+
+    pub(crate) fn custom_stdlib(&self) -> Option<&SystemPath> {
+        self.static_paths.iter().find_map(|search_path| {
+            if search_path.is_standard_library() {
+                search_path.as_system_path()
+            } else {
+                None
+            }
+        })
+    }
+
+    pub(super) fn typeshed_versions(&self) -> &TypeshedVersions {
+        &self.typeshed_versions
+    }
+}
+
+#[derive(Debug, PartialEq, Eq)]
+enum ResolvedTypeshedVersions {
+    Vendored(&'static TypeshedVersions),
+    Custom(TypeshedVersions),
+}
+
+impl Deref for ResolvedTypeshedVersions {
+    type Target = TypeshedVersions;
+
+    fn deref(&self) -> &Self::Target {
+        match self {
+            ResolvedTypeshedVersions::Vendored(versions) => versions,
+            ResolvedTypeshedVersions::Custom(versions) => versions,
+        }
+    }
 }
 
 /// Collect all dynamic search paths. For each `site-packages` path:
@@ -251,6 +311,7 @@ pub(crate) fn dynamic_resolution_paths(db: &dyn Db) -> Vec<SearchPath> {
     let SearchPaths {
         static_paths,
         site_packages,
+        typeshed_versions: _,
     } = Program::get(db).search_paths(db);
 
     let mut dynamic_paths = Vec::new();
@@ -315,12 +376,16 @@ pub(crate) fn dynamic_resolution_paths(db: &dyn Db) -> Vec<SearchPath> {
         let installations = all_pth_files.iter().flat_map(PthFile::items);
 
         for installation in installations {
+            let installation = system
+                .canonicalize_path(&installation)
+                .unwrap_or(installation);
+
             if existing_paths.insert(Cow::Owned(installation.clone())) {
-                match SearchPath::editable(system, installation) {
+                match SearchPath::editable(system, installation.clone()) {
                     Ok(search_path) => {
                         tracing::debug!(
                             "Adding editable installation to module resolution path {path}",
-                            path = search_path.as_system_path().unwrap()
+                            path = installation
                         );
                         dynamic_paths.push(search_path);
                     }
@@ -482,7 +547,7 @@ struct ModuleNameIngredient<'db> {
 fn resolve_name(db: &dyn Db, name: &ModuleName) -> Option<(SearchPath, File, ModuleKind)> {
     let program = Program::get(db);
     let target_version = program.target_version(db);
-    let resolver_state = ResolverState::new(db, target_version);
+    let resolver_state = ResolverContext::new(db, target_version);
     let is_builtin_module =
         ruff_python_stdlib::sys::is_builtin_module(target_version.minor, name.as_str());
 
@@ -545,7 +610,7 @@ fn resolve_name(db: &dyn Db, name: &ModuleName) -> Option<(SearchPath, File, Mod
 fn resolve_package<'a, 'db, I>(
     module_search_path: &SearchPath,
     components: I,
-    resolver_state: &ResolverState<'db>,
+    resolver_state: &ResolverContext<'db>,
 ) -> Result<ResolvedPackage, PackageKind>
 where
     I: Iterator<Item = &'a str>,
@@ -627,6 +692,21 @@ impl PackageKind {
     }
 }
 
+pub(super) struct ResolverContext<'db> {
+    pub(super) db: &'db dyn Db,
+    pub(super) target_version: PythonVersion,
+}
+
+impl<'db> ResolverContext<'db> {
+    pub(super) fn new(db: &'db dyn Db, target_version: PythonVersion) -> Self {
+        Self { db, target_version }
+    }
+
+    pub(super) fn vendored(&self) -> &VendoredFileSystem {
+        self.db.vendored()
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use ruff_db::files::{system_path_to_file, File, FilePath};
@@ -781,7 +861,7 @@ mod tests {
                 "Search path for {module_name} was unexpectedly {search_path:?}"
             );
             assert!(
-                search_path.is_stdlib_search_path(),
+                search_path.is_standard_library(),
                 "Expected a stdlib search path, but got {search_path:?}"
             );
         }
@@ -877,7 +957,7 @@ mod tests {
                 "Search path for {module_name} was unexpectedly {search_path:?}"
             );
             assert!(
-                search_path.is_stdlib_search_path(),
+                search_path.is_standard_library(),
                 "Expected a stdlib search path, but got {search_path:?}"
             );
         }
@@ -1194,13 +1274,13 @@ mod tests {
 
         Program::from_settings(
             &db,
-            ProgramSettings {
+            &ProgramSettings {
                 target_version: PythonVersion::PY38,
                 search_paths: SearchPathSettings {
                     extra_paths: vec![],
                     src_root: src.clone(),
                     custom_typeshed: Some(custom_typeshed.clone()),
-                    site_packages: vec![site_packages],
+                    site_packages: SitePackages::Known(vec![site_packages]),
                 },
             },
         )
@@ -1699,13 +1779,16 @@ not_a_directory
 
         Program::from_settings(
             &db,
-            ProgramSettings {
+            &ProgramSettings {
                 target_version: PythonVersion::default(),
                 search_paths: SearchPathSettings {
                     extra_paths: vec![],
                     src_root: SystemPathBuf::from("/src"),
                     custom_typeshed: None,
-                    site_packages: vec![venv_site_packages, system_site_packages],
+                    site_packages: SitePackages::Known(vec![
+                        venv_site_packages,
+                        system_site_packages,
+                    ]),
                 },
             },
         )

crates/red_knot_python_semantic/src/module_resolver/state.rs

@@ -1,25 +0,0 @@
-use ruff_db::vendored::VendoredFileSystem;
-
-use super::typeshed::LazyTypeshedVersions;
-use crate::db::Db;
-use crate::python_version::PythonVersion;
-
-pub(crate) struct ResolverState<'db> {
-    pub(crate) db: &'db dyn Db,
-    pub(crate) typeshed_versions: LazyTypeshedVersions<'db>,
-    pub(crate) target_version: PythonVersion,
-}
-
-impl<'db> ResolverState<'db> {
-    pub(crate) fn new(db: &'db dyn Db, target_version: PythonVersion) -> Self {
-        Self {
-            db,
-            typeshed_versions: LazyTypeshedVersions::new(),
-            target_version,
-        }
-    }
-
-    pub(crate) fn vendored(&self) -> &VendoredFileSystem {
-        self.db.vendored()
-    }
-}

crates/red_knot_python_semantic/src/module_resolver/testing.rs

@@ -4,7 +4,7 @@ use ruff_db::vendored::VendoredPathBuf;
 use crate::db::tests::TestDb;
 use crate::program::{Program, SearchPathSettings};
 use crate::python_version::PythonVersion;
-use crate::ProgramSettings;
+use crate::{ProgramSettings, SitePackages};
 
 /// A test case for the module resolver.
 ///
@@ -179,6 +179,7 @@ impl TestCaseBuilder<UnspecifiedTypeshed> {
             first_party_files,
             site_packages_files,
         } = self;
+
         TestCaseBuilder {
             typeshed_option: typeshed,
             target_version,
@@ -195,6 +196,7 @@ impl TestCaseBuilder<UnspecifiedTypeshed> {
             site_packages,
             target_version,
         } = self.with_custom_typeshed(MockedTypeshed::default()).build();
+
         TestCase {
             db,
             src,
@@ -223,13 +225,13 @@ impl TestCaseBuilder<MockedTypeshed> {
 
         Program::from_settings(
             &db,
-            ProgramSettings {
+            &ProgramSettings {
                 target_version,
                 search_paths: SearchPathSettings {
                     extra_paths: vec![],
                     src_root: src.clone(),
                     custom_typeshed: Some(typeshed.clone()),
-                    site_packages: vec![site_packages.clone()],
+                    site_packages: SitePackages::Known(vec![site_packages.clone()]),
                 },
             },
         )
@@ -279,13 +281,11 @@ impl TestCaseBuilder<VendoredTypeshed> {
 
         Program::from_settings(
             &db,
-            ProgramSettings {
+            &ProgramSettings {
                 target_version,
                 search_paths: SearchPathSettings {
-                    extra_paths: vec![],
-                    src_root: src.clone(),
-                    custom_typeshed: None,
-                    site_packages: vec![site_packages.clone()],
+                    site_packages: SitePackages::Known(vec![site_packages.clone()]),
+                    ..SearchPathSettings::new(src.clone())
                 },
             },
         )

crates/red_knot_python_semantic/src/module_resolver/typeshed/mod.rs

@@ -1,6 +1,6 @@
 pub use self::vendored::vendored_typeshed_stubs;
 pub(super) use self::versions::{
-    parse_typeshed_versions, LazyTypeshedVersions, TypeshedVersionsParseError,
+    typeshed_versions, vendored_typeshed_versions, TypeshedVersions, TypeshedVersionsParseError,
     TypeshedVersionsQueryResult,
 };
 

crates/red_knot_python_semantic/src/module_resolver/typeshed/versions.rs

@@ -1,4 +1,3 @@
-use std::cell::OnceCell;
 use std::collections::BTreeMap;
 use std::fmt;
 use std::num::{NonZeroU16, NonZeroUsize};
@@ -6,78 +5,12 @@ use std::ops::{RangeFrom, RangeInclusive};
 use std::str::FromStr;
 
 use once_cell::sync::Lazy;
-use ruff_db::system::SystemPath;
 use rustc_hash::FxHashMap;
 
-use ruff_db::files::{system_path_to_file, File};
-
 use super::vendored::vendored_typeshed_stubs;
 use crate::db::Db;
 use crate::module_name::ModuleName;
-use crate::python_version::PythonVersion;
-
-#[derive(Debug)]
-pub(crate) struct LazyTypeshedVersions<'db>(OnceCell<&'db TypeshedVersions>);
-
-impl<'db> LazyTypeshedVersions<'db> {
-    #[must_use]
-    pub(crate) fn new() -> Self {
-        Self(OnceCell::new())
-    }
-
-    /// Query whether a module exists at runtime in the stdlib on a certain Python version.
-    ///
-    /// Simply probing whether a file exists in typeshed is insufficient for this question,
-    /// as a module in the stdlib may have been added in Python 3.10, but the typeshed stub
-    /// will still be available (either in a custom typeshed dir or in our vendored copy)
-    /// even if the user specified Python 3.8 as the target version.
-    ///
-    /// For top-level modules and packages, the VERSIONS file can always provide an unambiguous answer
-    /// as to whether the module exists on the specified target version. However, VERSIONS does not
-    /// provide comprehensive information on all submodules, meaning that this method sometimes
-    /// returns [`TypeshedVersionsQueryResult::MaybeExists`].
-    /// See [`TypeshedVersionsQueryResult`] for more details.
-    #[must_use]
-    pub(crate) fn query_module(
-        &self,
-        db: &'db dyn Db,
-        module: &ModuleName,
-        stdlib_root: Option<&SystemPath>,
-        target_version: PythonVersion,
-    ) -> TypeshedVersionsQueryResult {
-        let versions = self.0.get_or_init(|| {
-            let versions_path = if let Some(system_path) = stdlib_root {
-                system_path.join("VERSIONS")
-            } else {
-                return &VENDORED_VERSIONS;
-            };
-            let Ok(versions_file) = system_path_to_file(db.upcast(), &versions_path) else {
-                todo!(
-                    "Still need to figure out how to handle VERSIONS files being deleted \
-                    from custom typeshed directories! Expected a file to exist at {versions_path}"
-                )
-            };
-            // TODO(Alex/Micha): If VERSIONS is invalid,
-            // this should invalidate not just the specific module resolution we're currently attempting,
-            // but all type inference that depends on any standard-library types.
-            // Unwrapping here is not correct...
-            parse_typeshed_versions(db, versions_file).as_ref().unwrap()
-        });
-        versions.query_module(module, target_version)
-    }
-}
-
-#[salsa::tracked(return_ref)]
-pub(crate) fn parse_typeshed_versions(
-    db: &dyn Db,
-    versions_file: File,
-) -> Result<TypeshedVersions, TypeshedVersionsParseError> {
-    // TODO: Handle IO errors
-    let file_content = versions_file
-        .read_to_string(db.upcast())
-        .unwrap_or_default();
-    file_content.parse()
-}
+use crate::{Program, PythonVersion};
 
 static VENDORED_VERSIONS: Lazy<TypeshedVersions> = Lazy::new(|| {
     TypeshedVersions::from_str(
@@ -88,6 +21,14 @@ static VENDORED_VERSIONS: Lazy<TypeshedVersions> = Lazy::new(|| {
     .unwrap()
 });
 
+pub(crate) fn vendored_typeshed_versions() -> &'static TypeshedVersions {
+    &VENDORED_VERSIONS
+}
+
+pub(crate) fn typeshed_versions(db: &dyn Db) -> &TypeshedVersions {
+    Program::get(db).search_paths(db).typeshed_versions()
+}
+
 #[derive(Debug, PartialEq, Eq, Clone)]
 pub(crate) struct TypeshedVersionsParseError {
     line_number: Option<NonZeroU16>,
@@ -174,7 +115,7 @@ impl TypeshedVersions {
     }
 
     #[must_use]
-    fn query_module(
+    pub(in crate::module_resolver) fn query_module(
         &self,
         module: &ModuleName,
         target_version: PythonVersion,
@@ -204,7 +145,7 @@ impl TypeshedVersions {
     }
 }
 
-/// Possible answers [`LazyTypeshedVersions::query_module()`] could give to the question:
+/// Possible answers [`TypeshedVersions::query_module()`] could give to the question:
 /// "Does this module exist in the stdlib at runtime on a certain target version?"
 #[derive(Debug, Copy, PartialEq, Eq, Clone, Hash)]
 pub(crate) enum TypeshedVersionsQueryResult {

crates/red_knot_python_semantic/src/node_key.rs

@@ -1,4 +1,4 @@
-use ruff_python_ast::{AnyNodeRef, AstNode, NodeKind};
+use ruff_python_ast::{AnyNodeRef, NodeKind};
 use ruff_text_size::{Ranged, TextRange};
 
 /// Compact key for a node for use in a hash map.
@@ -11,19 +11,7 @@ pub(super) struct NodeKey {
 }
 
 impl NodeKey {
-    #[inline]
-    pub(super) fn from_node<'a, N>(node: &N) -> Self
-    where
-        N: AstNode,
-    {
-        NodeKey {
-            kind: node.kind(),
-            range: node.range(),
-        }
-    }
-
-    #[inline]
-    pub(super) fn from_ref<'a, N>(node: N) -> Self
+    pub(super) fn from_node<'a, N>(node: N) -> Self
     where
         N: Into<AnyNodeRef<'a>>,
     {

crates/red_knot_python_semantic/src/program.rs

@@ -3,7 +3,7 @@ use anyhow::Context;
 use salsa::Durability;
 use salsa::Setter;
 
-use ruff_db::system::SystemPathBuf;
+use ruff_db::system::{SystemPath, SystemPathBuf};
 
 use crate::module_resolver::SearchPaths;
 use crate::Db;
@@ -12,13 +12,12 @@ use crate::Db;
 pub struct Program {
     pub target_version: PythonVersion,
 
-    #[default]
     #[return_ref]
     pub(crate) search_paths: SearchPaths,
 }
 
 impl Program {
-    pub fn from_settings(db: &dyn Db, settings: ProgramSettings) -> anyhow::Result<Self> {
+    pub fn from_settings(db: &dyn Db, settings: &ProgramSettings) -> anyhow::Result<Self> {
         let ProgramSettings {
             target_version,
             search_paths,
@@ -29,16 +28,15 @@ impl Program {
         let search_paths = SearchPaths::from_settings(db, search_paths)
             .with_context(|| "Invalid search path settings")?;
 
-        Ok(Program::builder(settings.target_version)
+        Ok(Program::builder(settings.target_version, search_paths)
             .durability(Durability::HIGH)
-            .search_paths(search_paths)
             .new(db))
     }
 
     pub fn update_search_paths(
-        &self,
+        self,
         db: &mut dyn Db,
-        search_path_settings: SearchPathSettings,
+        search_path_settings: &SearchPathSettings,
     ) -> anyhow::Result<()> {
         let search_paths = SearchPaths::from_settings(db, search_path_settings)?;
 
@@ -49,16 +47,20 @@ impl Program {
 
         Ok(())
     }
+
+    pub fn custom_stdlib_search_path(self, db: &dyn Db) -> Option<&SystemPath> {
+        self.search_paths(db).custom_stdlib()
+    }
 }
 
-#[derive(Debug, Eq, PartialEq)]
+#[derive(Clone, Debug, Eq, PartialEq)]
 pub struct ProgramSettings {
     pub target_version: PythonVersion,
     pub search_paths: SearchPathSettings,
 }
 
 /// Configures the search paths for module resolution.
-#[derive(Eq, PartialEq, Debug, Clone, Default)]
+#[derive(Eq, PartialEq, Debug, Clone)]
 pub struct SearchPathSettings {
     /// List of user-provided paths that should take first priority in the module resolution.
     /// Examples in other type checkers are mypy's MYPYPATH environment variable,
@@ -74,5 +76,25 @@ pub struct SearchPathSettings {
     pub custom_typeshed: Option<SystemPathBuf>,
 
     /// The path to the user's `site-packages` directory, where third-party packages from ``PyPI`` are installed.
-    pub site_packages: Vec<SystemPathBuf>,
+    pub site_packages: SitePackages,
+}
+
+impl SearchPathSettings {
+    pub fn new(src_root: SystemPathBuf) -> Self {
+        Self {
+            src_root,
+            extra_paths: vec![],
+            custom_typeshed: None,
+            site_packages: SitePackages::Known(vec![]),
+        }
+    }
+}
+
+#[derive(Debug, Clone, Eq, PartialEq)]
+pub enum SitePackages {
+    Derived {
+        venv_path: SystemPathBuf,
+    },
+    /// Resolved site packages paths
+    Known(Vec<SystemPathBuf>),
 }

crates/red_knot_python_semantic/src/semantic_index.rs

@@ -790,6 +790,56 @@ def f(a: str, /, b: str, c: int = 1, *args, d: int = 2, **kwargs):
         assert_eq!(names(&inner_comprehension_symbol_table), vec!["x"]);
     }
 
+    #[test]
+    fn with_item_definition() {
+        let TestCase { db, file } = test_case(
+            "
+with item1 as x, item2 as y:
+    pass
+",
+        );
+
+        let index = semantic_index(&db, file);
+        let global_table = index.symbol_table(FileScopeId::global());
+
+        assert_eq!(names(&global_table), vec!["item1", "x", "item2", "y"]);
+
+        let use_def = index.use_def_map(FileScopeId::global());
+        for name in ["x", "y"] {
+            let Some(definition) = use_def.first_public_definition(
+                global_table.symbol_id_by_name(name).expect("symbol exists"),
+            ) else {
+                panic!("Expected with item definition for {name}");
+            };
+            assert!(matches!(definition.node(&db), DefinitionKind::WithItem(_)));
+        }
+    }
+
+    #[test]
+    fn with_item_unpacked_definition() {
+        let TestCase { db, file } = test_case(
+            "
+with context() as (x, y):
+    pass
+",
+        );
+
+        let index = semantic_index(&db, file);
+        let global_table = index.symbol_table(FileScopeId::global());
+
+        assert_eq!(names(&global_table), vec!["context", "x", "y"]);
+
+        let use_def = index.use_def_map(FileScopeId::global());
+        for name in ["x", "y"] {
+            let Some(definition) = use_def.first_public_definition(
+                global_table.symbol_id_by_name(name).expect("symbol exists"),
+            ) else {
+                panic!("Expected with item definition for {name}");
+            };
+            assert!(matches!(definition.node(&db), DefinitionKind::WithItem(_)));
+        }
+    }
+
     #[test]
     fn dupes() {
         let TestCase { db, file } = test_case(

crates/red_knot_python_semantic/src/semantic_index/ast_ids.rs

@@ -197,14 +197,12 @@ pub(crate) mod node_key {
     pub(crate) struct ExpressionNodeKey(NodeKey);
 
     impl From<ast::ExpressionRef<'_>> for ExpressionNodeKey {
-        #[inline]
         fn from(value: ast::ExpressionRef<'_>) -> Self {
-            Self(NodeKey::from_ref(value))
+            Self(NodeKey::from_node(value))
         }
     }
 
     impl From<&ast::Expr> for ExpressionNodeKey {
-        #[inline]
         fn from(value: &ast::Expr) -> Self {
             Self(NodeKey::from_node(value))
         }

crates/red_knot_python_semantic/src/semantic_index/builder.rs

@@ -26,6 +26,8 @@ use crate::semantic_index::use_def::{FlowSnapshot, UseDefMapBuilder};
 use crate::semantic_index::SemanticIndex;
 use crate::Db;
 
+use super::definition::WithItemDefinitionNodeRef;
+
 pub(super) struct SemanticIndexBuilder<'db> {
     // Builder state
     db: &'db dyn Db,
@@ -561,6 +563,18 @@ where
                     self.flow_merge(break_state);
                 }
             }
+            ast::Stmt::With(ast::StmtWith { items, body, .. }) => {
+                for item in items {
+                    self.visit_expr(&item.context_expr);
+                    if let Some(optional_vars) = item.optional_vars.as_deref() {
+                        self.add_standalone_expression(&item.context_expr);
+                        self.current_assignment = Some(item.into());
+                        self.visit_expr(optional_vars);
+                        self.current_assignment = None;
+                    }
+                }
+                self.visit_body(body);
+            }
             ast::Stmt::Break(_) => {
                 self.loop_break_states.push(self.flow_snapshot());
             }
@@ -622,6 +636,15 @@ where
                                 ComprehensionDefinitionNodeRef { node, first },
                             );
                         }
+                        Some(CurrentAssignment::WithItem(with_item)) => {
+                            self.add_definition(
+                                symbol,
+                                WithItemDefinitionNodeRef {
+                                    node: with_item,
+                                    target: name_node,
+                                },
+                            );
+                        }
                         None => {}
                     }
                 }
@@ -635,11 +658,11 @@ where
             }
             ast::Expr::Named(node) => {
                 debug_assert!(self.current_assignment.is_none());
-                self.current_assignment = Some(node.into());
                 // TODO walrus in comprehensions is implicitly nonlocal
+                self.visit_expr(&node.value);
+                self.current_assignment = Some(node.into());
                 self.visit_expr(&node.target);
                 self.current_assignment = None;
-                self.visit_expr(&node.value);
             }
             ast::Expr::Lambda(lambda) => {
                 if let Some(parameters) = &lambda.parameters {
@@ -778,6 +801,7 @@ enum CurrentAssignment<'a> {
         node: &'a ast::Comprehension,
         first: bool,
     },
+    WithItem(&'a ast::WithItem),
 }
 
 impl<'a> From<&'a ast::StmtAssign> for CurrentAssignment<'a> {
@@ -803,3 +827,9 @@ impl<'a> From<&'a ast::ExprNamed> for CurrentAssignment<'a> {
         Self::Named(value)
     }
 }
+
+impl<'a> From<&'a ast::WithItem> for CurrentAssignment<'a> {
+    fn from(value: &'a ast::WithItem) -> Self {
+        Self::WithItem(value)
+    }
+}

crates/red_knot_python_semantic/src/semantic_index/definition.rs

@@ -47,6 +47,7 @@ pub(crate) enum DefinitionNodeRef<'a> {
     AugmentedAssignment(&'a ast::StmtAugAssign),
     Comprehension(ComprehensionDefinitionNodeRef<'a>),
     Parameter(ast::AnyParameterRef<'a>),
+    WithItem(WithItemDefinitionNodeRef<'a>),
 }
 
 impl<'a> From<&'a ast::StmtFunctionDef> for DefinitionNodeRef<'a> {
@@ -97,6 +98,12 @@ impl<'a> From<AssignmentDefinitionNodeRef<'a>> for DefinitionNodeRef<'a> {
     }
 }
 
+impl<'a> From<WithItemDefinitionNodeRef<'a>> for DefinitionNodeRef<'a> {
+    fn from(node_ref: WithItemDefinitionNodeRef<'a>) -> Self {
+        Self::WithItem(node_ref)
+    }
+}
+
 impl<'a> From<ComprehensionDefinitionNodeRef<'a>> for DefinitionNodeRef<'a> {
     fn from(node: ComprehensionDefinitionNodeRef<'a>) -> Self {
         Self::Comprehension(node)
@@ -121,6 +128,12 @@ pub(crate) struct AssignmentDefinitionNodeRef<'a> {
     pub(crate) target: &'a ast::ExprName,
 }
 
+#[derive(Copy, Clone, Debug)]
+pub(crate) struct WithItemDefinitionNodeRef<'a> {
+    pub(crate) node: &'a ast::WithItem,
+    pub(crate) target: &'a ast::ExprName,
+}
+
 #[derive(Copy, Clone, Debug)]
 pub(crate) struct ComprehensionDefinitionNodeRef<'a> {
     pub(crate) node: &'a ast::Comprehension,
@@ -175,6 +188,12 @@ impl DefinitionNodeRef<'_> {
                     DefinitionKind::ParameterWithDefault(AstNodeRef::new(parsed, parameter))
                 }
             },
+            DefinitionNodeRef::WithItem(WithItemDefinitionNodeRef { node, target }) => {
+                DefinitionKind::WithItem(WithItemDefinitionKind {
+                    node: AstNodeRef::new(parsed.clone(), node),
+                    target: AstNodeRef::new(parsed, target),
+                })
+            }
         }
     }
 
@@ -198,6 +217,7 @@ impl DefinitionNodeRef<'_> {
                 ast::AnyParameterRef::Variadic(parameter) => parameter.into(),
                 ast::AnyParameterRef::NonVariadic(parameter) => parameter.into(),
             },
+            Self::WithItem(WithItemDefinitionNodeRef { node: _, target }) => target.into(),
         }
     }
 }
@@ -215,6 +235,7 @@ pub enum DefinitionKind {
     Comprehension(ComprehensionDefinitionKind),
     Parameter(AstNodeRef<ast::Parameter>),
     ParameterWithDefault(AstNodeRef<ast::ParameterWithDefault>),
+    WithItem(WithItemDefinitionKind),
 }
 
 #[derive(Clone, Debug)]
@@ -250,7 +271,6 @@ impl ImportFromDefinitionKind {
 }
 
 #[derive(Clone, Debug)]
-#[allow(dead_code)]
 pub struct AssignmentDefinitionKind {
     assignment: AstNodeRef<ast::StmtAssign>,
     target: AstNodeRef<ast::ExprName>,
@@ -266,6 +286,22 @@ impl AssignmentDefinitionKind {
     }
 }
 
+#[derive(Clone, Debug)]
+pub struct WithItemDefinitionKind {
+    node: AstNodeRef<ast::WithItem>,
+    target: AstNodeRef<ast::ExprName>,
+}
+
+impl WithItemDefinitionKind {
+    pub(crate) fn node(&self) -> &ast::WithItem {
+        self.node.node()
+    }
+
+    pub(crate) fn target(&self) -> &ast::ExprName {
+        self.target.node()
+    }
+}
+
 #[derive(Copy, Clone, Eq, PartialEq, Hash, Debug)]
 pub(crate) struct DefinitionNodeKey(NodeKey);
 

crates/red_knot_python_semantic/src/semantic_model.rs

@@ -184,14 +184,9 @@ mod tests {
 
         Program::from_settings(
             &db,
-            ProgramSettings {
+            &ProgramSettings {
                 target_version: PythonVersion::default(),
-                search_paths: SearchPathSettings {
-                    extra_paths: vec![],
-                    src_root: SystemPathBuf::from("/src"),
-                    site_packages: vec![],
-                    custom_typeshed: None,
-                },
+                search_paths: SearchPathSettings::new(SystemPathBuf::from("/src")),
             },
         )?;
 

crates/red_knot_python_semantic/src/site_packages.rs

@@ -13,9 +13,10 @@ use std::io;
 use std::num::NonZeroUsize;
 use std::ops::Deref;
 
-use red_knot_python_semantic::PythonVersion;
 use ruff_db::system::{System, SystemPath, SystemPathBuf};
 
+use crate::PythonVersion;
+
 type SitePackagesDiscoveryResult<T> = Result<T, SitePackagesDiscoveryError>;
 
 /// Abstraction for a Python virtual environment.
@@ -24,7 +25,7 @@ type SitePackagesDiscoveryResult<T> = Result<T, SitePackagesDiscoveryError>;
 /// The format of this file is not defined anywhere, and exactly which keys are present
 /// depends on the tool that was used to create the virtual environment.
 #[derive(Debug)]
-pub struct VirtualEnvironment {
+pub(crate) struct VirtualEnvironment {
     venv_path: SysPrefixPath,
     base_executable_home_path: PythonHomePath,
     include_system_site_packages: bool,
@@ -41,7 +42,7 @@ pub struct VirtualEnvironment {
 }
 
 impl VirtualEnvironment {
-    pub fn new(
+    pub(crate) fn new(
         path: impl AsRef<SystemPath>,
         system: &dyn System,
     ) -> SitePackagesDiscoveryResult<Self> {
@@ -157,7 +158,7 @@ impl VirtualEnvironment {
     /// Return a list of `site-packages` directories that are available from this virtual environment
     ///
     /// See the documentation for `site_packages_dir_from_sys_prefix` for more details.
-    pub fn site_packages_directories(
+    pub(crate) fn site_packages_directories(
         &self,
         system: &dyn System,
     ) -> SitePackagesDiscoveryResult<Vec<SystemPathBuf>> {
@@ -204,7 +205,7 @@ System site-packages will not be used for module resolution.",
 }
 
 #[derive(Debug, thiserror::Error)]
-pub enum SitePackagesDiscoveryError {
+pub(crate) enum SitePackagesDiscoveryError {
     #[error("Invalid --venv-path argument: {0} could not be canonicalized")]
     VenvDirCanonicalizationError(SystemPathBuf, #[source] io::Error),
     #[error("Invalid --venv-path argument: {0} does not point to a directory on disk")]
@@ -221,7 +222,7 @@ pub enum SitePackagesDiscoveryError {
 
 /// The various ways in which parsing a `pyvenv.cfg` file could fail
 #[derive(Debug)]
-pub enum PyvenvCfgParseErrorKind {
+pub(crate) enum PyvenvCfgParseErrorKind {
     TooManyEquals { line_number: NonZeroUsize },
     MalformedKeyValuePair { line_number: NonZeroUsize },
     NoHomeKey,
@@ -370,7 +371,7 @@ fn site_packages_directory_from_sys_prefix(
 ///
 /// [`sys.prefix`]: https://docs.python.org/3/library/sys.html#sys.prefix
 #[derive(Debug, PartialEq, Eq, Clone)]
-pub struct SysPrefixPath(SystemPathBuf);
+pub(crate) struct SysPrefixPath(SystemPathBuf);
 
 impl SysPrefixPath {
     fn new(

crates/red_knot_python_semantic/src/types.rs

@@ -181,6 +181,8 @@ pub enum Type<'db> {
     IntLiteral(i64),
     /// A boolean literal, either `True` or `False`.
     BooleanLiteral(bool),
+    /// A bytes literal
+    BytesLiteral(BytesLiteralType<'db>),
     // TODO protocols, callable types, overloads, generics, type vars
 }
 
@@ -276,6 +278,10 @@ impl<'db> Type<'db> {
                 Type::Unknown
             }
             Type::BooleanLiteral(_) => Type::Unknown,
+            Type::BytesLiteral(_) => {
+                // TODO defer to Type::Instance(<bytes from typeshed>).member
+                Type::Unknown
+            }
         }
     }
 
@@ -372,6 +378,12 @@ pub struct IntersectionType<'db> {
     negative: FxOrderSet<Type<'db>>,
 }
 
+#[salsa::interned]
+pub struct BytesLiteralType<'db> {
+    #[return_ref]
+    value: Box<[u8]>,
+}
+
 #[cfg(test)]
 mod tests {
     use anyhow::Context;
@@ -392,14 +404,9 @@ mod tests {
 
         Program::from_settings(
             &db,
-            ProgramSettings {
+            &ProgramSettings {
                 target_version: PythonVersion::default(),
-                search_paths: SearchPathSettings {
-                    extra_paths: Vec::new(),
-                    src_root: SystemPathBuf::from("/src"),
-                    site_packages: vec![],
-                    custom_typeshed: None,
-                },
+                search_paths: SearchPathSettings::new(SystemPathBuf::from("/src")),
             },
         )
         .expect("Valid search path settings");

crates/red_knot_python_semantic/src/types/display.rs

@@ -2,6 +2,9 @@
 
 use std::fmt::{Display, Formatter};
 
+use ruff_python_ast::str::Quote;
+use ruff_python_literal::escape::AsciiEscape;
+
 use crate::types::{IntersectionType, Type, UnionType};
 use crate::Db;
 
@@ -38,6 +41,14 @@ impl Display for DisplayType<'_> {
             Type::BooleanLiteral(boolean) => {
                 write!(f, "Literal[{}]", if *boolean { "True" } else { "False" })
             }
+            Type::BytesLiteral(bytes) => {
+                let escape =
+                    AsciiEscape::with_preferred_quote(bytes.value(self.db).as_ref(), Quote::Double);
+
+                f.write_str("Literal[")?;
+                escape.bytes_repr().write(f)?;
+                f.write_str("]")
+            }
         }
     }
 }

crates/red_knot_python_semantic/src/types/infer.rs

@@ -43,8 +43,8 @@ use crate::semantic_index::symbol::{FileScopeId, NodeWithScopeKind, NodeWithScop
 use crate::semantic_index::SemanticIndex;
 use crate::types::diagnostic::{TypeCheckDiagnostic, TypeCheckDiagnostics};
 use crate::types::{
-    builtins_symbol_ty_by_name, definitions_ty, global_symbol_ty_by_name, ClassType, FunctionType,
-    Name, Type, UnionBuilder,
+    builtins_symbol_ty_by_name, definitions_ty, global_symbol_ty_by_name, BytesLiteralType,
+    ClassType, FunctionType, Name, Type, UnionBuilder,
 };
 use crate::Db;
 
@@ -333,6 +333,9 @@ impl<'db> TypeInferenceBuilder<'db> {
             DefinitionKind::ParameterWithDefault(parameter_with_default) => {
                 self.infer_parameter_with_default_definition(parameter_with_default, definition);
             }
+            DefinitionKind::WithItem(with_item) => {
+                self.infer_with_item_definition(with_item.target(), with_item.node(), definition);
+            }
         }
     }
 
@@ -618,13 +621,42 @@ impl<'db> TypeInferenceBuilder<'db> {
         } = with_statement;
 
         for item in items {
-            self.infer_expression(&item.context_expr);
-            self.infer_optional_expression(item.optional_vars.as_deref());
+            match item.optional_vars.as_deref() {
+                Some(ast::Expr::Name(name)) => {
+                    self.infer_definition(name);
+                }
+                _ => {
+                    // TODO infer definitions in unpacking assignment
+                    self.infer_expression(&item.context_expr);
+                }
+            }
         }
 
         self.infer_body(body);
     }
 
+    fn infer_with_item_definition(
+        &mut self,
+        target: &ast::ExprName,
+        with_item: &ast::WithItem,
+        definition: Definition<'db>,
+    ) {
+        let expression = self.index.expression(&with_item.context_expr);
+        let result = infer_expression_types(self.db, expression);
+        self.extend(result);
+
+        // TODO(dhruvmanila): The correct type inference here is the return type of the __enter__
+        // method of the context manager.
+        let context_expr_ty = self
+            .types
+            .expression_ty(with_item.context_expr.scoped_ast_id(self.db, self.scope));
+
+        self.types
+            .expressions
+            .insert(target.scoped_ast_id(self.db, self.scope), context_expr_ty);
+        self.types.definitions.insert(definition, context_expr_ty);
+    }
+
     fn infer_match_statement(&mut self, match_statement: &ast::StmtMatch) {
         let ast::StmtMatch {
             range: _,
@@ -1174,9 +1206,12 @@ impl<'db> TypeInferenceBuilder<'db> {
     }
 
     #[allow(clippy::unused_self)]
-    fn infer_bytes_literal_expression(&mut self, _literal: &ast::ExprBytesLiteral) -> Type<'db> {
-        // TODO
-        Type::Unknown
+    fn infer_bytes_literal_expression(&mut self, literal: &ast::ExprBytesLiteral) -> Type<'db> {
+        // TODO: ignoring r/R prefixes for now, should normalize bytes values
+        Type::BytesLiteral(BytesLiteralType::new(
+            self.db,
+            literal.value.bytes().collect(),
+        ))
     }
 
     fn infer_fstring_expression(&mut self, fstring: &ast::ExprFString) -> Type<'db> {
@@ -1652,6 +1687,7 @@ impl<'db> TypeInferenceBuilder<'db> {
         let left_ty = self.infer_expression(left);
         let right_ty = self.infer_expression(right);
 
+        // TODO flatten the matches by matching on (left_ty, right_ty, op)
         match left_ty {
             Type::Any => Type::Any,
             Type::Unknown => Type::Unknown,
@@ -1690,6 +1726,22 @@ impl<'db> TypeInferenceBuilder<'db> {
                     _ => Type::Unknown, // TODO
                 }
             }
+            Type::BytesLiteral(lhs) => {
+                match right_ty {
+                    Type::BytesLiteral(rhs) => {
+                        match op {
+                            ast::Operator::Add => Type::BytesLiteral(BytesLiteralType::new(
+                                self.db,
+                                [lhs.value(self.db).as_ref(), rhs.value(self.db).as_ref()]
+                                    .concat()
+                                    .into_boxed_slice(),
+                            )),
+                            _ => Type::Unknown, // TODO
+                        }
+                    }
+                    _ => Type::Unknown, // TODO
+                }
+            }
             _ => Type::Unknown, // TODO
         }
     }
@@ -1866,14 +1918,9 @@ mod tests {
 
         Program::from_settings(
             &db,
-            ProgramSettings {
+            &ProgramSettings {
                 target_version: PythonVersion::default(),
-                search_paths: SearchPathSettings {
-                    extra_paths: Vec::new(),
-                    src_root,
-                    site_packages: vec![],
-                    custom_typeshed: None,
-                },
+                search_paths: SearchPathSettings::new(src_root),
             },
         )
         .expect("Valid search path settings");
@@ -1893,13 +1940,11 @@ mod tests {
 
         Program::from_settings(
             &db,
-            ProgramSettings {
+            &ProgramSettings {
                 target_version: PythonVersion::default(),
                 search_paths: SearchPathSettings {
-                    extra_paths: Vec::new(),
-                    src_root,
-                    site_packages: vec![],
                     custom_typeshed: Some(SystemPathBuf::from(typeshed)),
+                    ..SearchPathSettings::new(src_root)
                 },
             },
         )
@@ -2210,6 +2255,28 @@ mod tests {
         Ok(())
     }
 
+    #[test]
+    fn bytes_type() -> anyhow::Result<()> {
+        let mut db = setup_db();
+
+        db.write_dedented(
+            "src/a.py",
+            "
+            w = b'red' b'knot'
+            x = b'hello'
+            y = b'world' + b'!'
+            z = b'\\xff\\x00'
+            ",
+        )?;
+
+        assert_public_ty(&db, "src/a.py", "w", "Literal[b\"redknot\"]");
+        assert_public_ty(&db, "src/a.py", "x", "Literal[b\"hello\"]");
+        assert_public_ty(&db, "src/a.py", "y", "Literal[b\"world!\"]");
+        assert_public_ty(&db, "src/a.py", "z", "Literal[b\"\\xff\\x00\"]");
+
+        Ok(())
+    }
+
     #[test]
     fn resolve_union() -> anyhow::Result<()> {
         let mut db = setup_db();
@@ -2265,6 +2332,23 @@ mod tests {
         Ok(())
     }
 
+    #[test]
+    fn walrus_self_plus_one() -> anyhow::Result<()> {
+        let mut db = setup_db();
+
+        db.write_dedented(
+            "src/a.py",
+            "
+            x = 0
+            (x := x + 1)
+            ",
+        )?;
+
+        assert_public_ty(&db, "src/a.py", "x", "Literal[1]");
+
+        Ok(())
+    }
+
     #[test]
     fn ifexpr() -> anyhow::Result<()> {
         let mut db = setup_db();

crates/red_knot_server/Cargo.toml

@@ -11,7 +11,6 @@ repository = { workspace = true }
 license = { workspace = true }
 
 [dependencies]
-red_knot_python_semantic = { workspace = true }
 red_knot_workspace = { workspace = true }
 ruff_db = { workspace = true }
 ruff_notebook = { workspace = true }

crates/red_knot_server/src/server.rs

@@ -3,11 +3,11 @@
 use std::num::NonZeroUsize;
 use std::panic::PanicInfo;
 
-use lsp_server as lsp;
-use lsp_types as types;
+use lsp_server::Message;
 use lsp_types::{
-    ClientCapabilities, DiagnosticOptions, NotebookCellSelector, NotebookDocumentSyncOptions,
-    NotebookSelector, TextDocumentSyncCapability, TextDocumentSyncOptions,
+    ClientCapabilities, DiagnosticOptions, DiagnosticServerCapabilities, MessageType,
+    ServerCapabilities, TextDocumentSyncCapability, TextDocumentSyncKind, TextDocumentSyncOptions,
+    Url,
 };
 
 use self::connection::{Connection, ConnectionInitializer};
@@ -74,7 +74,7 @@ impl Server {
             init_params.client_info.as_ref(),
         );
 
-        let mut workspace_for_url = |url: lsp_types::Url| {
+        let mut workspace_for_url = |url: Url| {
             let Some(workspace_settings) = workspace_settings.as_mut() else {
                 return (url, ClientSettings::default());
             };
@@ -93,13 +93,18 @@ impl Server {
             }).collect())
             .or_else(|| {
                 tracing::warn!("No workspace(s) were provided during initialization. Using the current working directory as a default workspace...");
-                let uri = types::Url::from_file_path(std::env::current_dir().ok()?).ok()?;
+                let uri = Url::from_file_path(std::env::current_dir().ok()?).ok()?;
                 Some(vec![workspace_for_url(uri)])
             })
             .ok_or_else(|| {
                 anyhow::anyhow!("Failed to get the current working directory while creating a default workspace.")
             })?;
 
+        if workspaces.len() > 1 {
+            // TODO(dhruvmanila): Support multi-root workspaces
+            anyhow::bail!("Multi-root workspaces are not supported yet");
+        }
+
         Ok(Self {
             connection,
             worker_threads,
@@ -149,7 +154,7 @@ impl Server {
             try_show_message(
                 "The Ruff language server exited with a panic. See the logs for more details."
                     .to_string(),
-                lsp_types::MessageType::ERROR,
+                MessageType::ERROR,
             )
             .ok();
         }));
@@ -182,9 +187,9 @@ impl Server {
                 break;
             }
             let task = match msg {
-                lsp::Message::Request(req) => api::request(req),
-                lsp::Message::Notification(notification) => api::notification(notification),
-                lsp::Message::Response(response) => scheduler.response(response),
+                Message::Request(req) => api::request(req),
+                Message::Notification(notification) => api::notification(notification),
+                Message::Response(response) => scheduler.response(response),
             };
             scheduler.dispatch(task);
         }
@@ -206,28 +211,17 @@ impl Server {
             .unwrap_or_default()
     }
 
-    fn server_capabilities(position_encoding: PositionEncoding) -> types::ServerCapabilities {
-        types::ServerCapabilities {
+    fn server_capabilities(position_encoding: PositionEncoding) -> ServerCapabilities {
+        ServerCapabilities {
             position_encoding: Some(position_encoding.into()),
-            diagnostic_provider: Some(types::DiagnosticServerCapabilities::Options(
-                DiagnosticOptions {
-                    identifier: Some(crate::DIAGNOSTIC_NAME.into()),
-                    ..Default::default()
-                },
-            )),
-            notebook_document_sync: Some(types::OneOf::Left(NotebookDocumentSyncOptions {
-                save: Some(false),
-                notebook_selector: [NotebookSelector::ByCells {
-                    notebook: None,
-                    cells: vec![NotebookCellSelector {
-                        language: "python".to_string(),
-                    }],
-                }]
-                .to_vec(),
+            diagnostic_provider: Some(DiagnosticServerCapabilities::Options(DiagnosticOptions {
+                identifier: Some(crate::DIAGNOSTIC_NAME.into()),
+                ..Default::default()
             })),
             text_document_sync: Some(TextDocumentSyncCapability::Options(
                 TextDocumentSyncOptions {
                     open_close: Some(true),
+                    change: Some(TextDocumentSyncKind::INCREMENTAL),
                     ..Default::default()
                 },
             )),

crates/red_knot_server/src/server/api.rs

@@ -1,13 +1,15 @@
-use crate::{server::schedule::Task, session::Session, system::url_to_system_path};
 use lsp_server as server;
 
+use crate::server::schedule::Task;
+use crate::session::Session;
+use crate::system::{url_to_any_system_path, AnySystemPath};
+
 mod diagnostics;
 mod notifications;
 mod requests;
 mod traits;
 
 use notifications as notification;
-use red_knot_workspace::db::RootDatabase;
 use requests as request;
 
 use self::traits::{NotificationHandler, RequestHandler};
@@ -43,6 +45,7 @@ pub(super) fn notification<'a>(notif: server::Notification) -> Task<'a> {
     match notif.method.as_str() {
         notification::DidCloseTextDocumentHandler::METHOD => local_notification_task::<notification::DidCloseTextDocumentHandler>(notif),
         notification::DidOpenTextDocumentHandler::METHOD => local_notification_task::<notification::DidOpenTextDocumentHandler>(notif),
+        notification::DidChangeTextDocumentHandler::METHOD => local_notification_task::<notification::DidChangeTextDocumentHandler>(notif),
         notification::DidOpenNotebookHandler::METHOD => {
             local_notification_task::<notification::DidOpenNotebookHandler>(notif)
         }
@@ -82,12 +85,18 @@ fn background_request_task<'a, R: traits::BackgroundDocumentRequestHandler>(
     Ok(Task::background(schedule, move |session: &Session| {
         let url = R::document_url(&params).into_owned();
 
-        let Ok(path) = url_to_system_path(&url) else {
+        let Ok(path) = url_to_any_system_path(&url) else {
             return Box::new(|_, _| {});
         };
-        let db = session
-            .workspace_db_for_path(path.as_std_path())
-            .map(RootDatabase::snapshot);
+        let db = match path {
+            AnySystemPath::System(path) => {
+                match session.workspace_db_for_path(path.as_std_path()) {
+                    Some(db) => db.snapshot(),
+                    None => session.default_workspace_db().snapshot(),
+                }
+            }
+            AnySystemPath::SystemVirtual(_) => session.default_workspace_db().snapshot(),
+        };
 
         let Some(snapshot) = session.take_snapshot(url) else {
             return Box::new(|_, _| {});

crates/red_knot_server/src/server/api/notifications.rs

@@ -1,9 +1,11 @@
+mod did_change;
 mod did_close;
 mod did_close_notebook;
 mod did_open;
 mod did_open_notebook;
 mod set_trace;
 
+pub(super) use did_change::DidChangeTextDocumentHandler;
 pub(super) use did_close::DidCloseTextDocumentHandler;
 pub(super) use did_close_notebook::DidCloseNotebookHandler;
 pub(super) use did_open::DidOpenTextDocumentHandler;

crates/red_knot_server/src/server/api/notifications/did_change.rs

@@ -0,0 +1,55 @@
+use lsp_server::ErrorCode;
+use lsp_types::notification::DidChangeTextDocument;
+use lsp_types::DidChangeTextDocumentParams;
+
+use red_knot_workspace::watch::ChangeEvent;
+
+use crate::server::api::traits::{NotificationHandler, SyncNotificationHandler};
+use crate::server::api::LSPResult;
+use crate::server::client::{Notifier, Requester};
+use crate::server::Result;
+use crate::session::Session;
+use crate::system::{url_to_any_system_path, AnySystemPath};
+
+pub(crate) struct DidChangeTextDocumentHandler;
+
+impl NotificationHandler for DidChangeTextDocumentHandler {
+    type NotificationType = DidChangeTextDocument;
+}
+
+impl SyncNotificationHandler for DidChangeTextDocumentHandler {
+    fn run(
+        session: &mut Session,
+        _notifier: Notifier,
+        _requester: &mut Requester,
+        params: DidChangeTextDocumentParams,
+    ) -> Result<()> {
+        let Ok(path) = url_to_any_system_path(&params.text_document.uri) else {
+            return Ok(());
+        };
+
+        let key = session.key_from_url(params.text_document.uri);
+
+        session
+            .update_text_document(&key, params.content_changes, params.text_document.version)
+            .with_failure_code(ErrorCode::InternalError)?;
+
+        match path {
+            AnySystemPath::System(path) => {
+                let db = match session.workspace_db_for_path_mut(path.as_std_path()) {
+                    Some(db) => db,
+                    None => session.default_workspace_db_mut(),
+                };
+                db.apply_changes(vec![ChangeEvent::file_content_changed(path)], None);
+            }
+            AnySystemPath::SystemVirtual(virtual_path) => {
+                let db = session.default_workspace_db_mut();
+                db.apply_changes(vec![ChangeEvent::ChangedVirtual(virtual_path)], None);
+            }
+        }
+
+        // TODO(dhruvmanila): Publish diagnostics if the client doesnt support pull diagnostics
+
+        Ok(())
+    }
+}

crates/red_knot_server/src/server/api/notifications/did_close.rs

@@ -1,8 +1,7 @@
 use lsp_server::ErrorCode;
 use lsp_types::notification::DidCloseTextDocument;
 use lsp_types::DidCloseTextDocumentParams;
-
-use ruff_db::files::File;
+use red_knot_workspace::watch::ChangeEvent;
 
 use crate::server::api::diagnostics::clear_diagnostics;
 use crate::server::api::traits::{NotificationHandler, SyncNotificationHandler};
@@ -10,7 +9,7 @@ use crate::server::api::LSPResult;
 use crate::server::client::{Notifier, Requester};
 use crate::server::Result;
 use crate::session::Session;
-use crate::system::url_to_system_path;
+use crate::system::{url_to_any_system_path, AnySystemPath};
 
 pub(crate) struct DidCloseTextDocumentHandler;
 
@@ -25,7 +24,7 @@ impl SyncNotificationHandler for DidCloseTextDocumentHandler {
         _requester: &mut Requester,
         params: DidCloseTextDocumentParams,
     ) -> Result<()> {
-        let Ok(path) = url_to_system_path(&params.text_document.uri) else {
+        let Ok(path) = url_to_any_system_path(&params.text_document.uri) else {
             return Ok(());
         };
 
@@ -34,8 +33,9 @@ impl SyncNotificationHandler for DidCloseTextDocumentHandler {
             .close_document(&key)
             .with_failure_code(ErrorCode::InternalError)?;
 
-        if let Some(db) = session.workspace_db_for_path_mut(path.as_std_path()) {
-            File::sync_path(db, &path);
+        if let AnySystemPath::SystemVirtual(virtual_path) = path {
+            let db = session.default_workspace_db_mut();
+            db.apply_changes(vec![ChangeEvent::DeletedVirtual(virtual_path)], None);
         }
 
         clear_diagnostics(key.url(), &notifier)?;

crates/red_knot_server/src/server/api/notifications/did_close_notebook.rs

@@ -1,14 +1,14 @@
 use lsp_types::notification::DidCloseNotebookDocument;
 use lsp_types::DidCloseNotebookDocumentParams;
 
-use ruff_db::files::File;
+use red_knot_workspace::watch::ChangeEvent;
 
 use crate::server::api::traits::{NotificationHandler, SyncNotificationHandler};
 use crate::server::api::LSPResult;
 use crate::server::client::{Notifier, Requester};
 use crate::server::Result;
 use crate::session::Session;
-use crate::system::url_to_system_path;
+use crate::system::{url_to_any_system_path, AnySystemPath};
 
 pub(crate) struct DidCloseNotebookHandler;
 
@@ -23,7 +23,7 @@ impl SyncNotificationHandler for DidCloseNotebookHandler {
         _requester: &mut Requester,
         params: DidCloseNotebookDocumentParams,
     ) -> Result<()> {
-        let Ok(path) = url_to_system_path(&params.notebook_document.uri) else {
+        let Ok(path) = url_to_any_system_path(&params.notebook_document.uri) else {
             return Ok(());
         };
 
@@ -32,8 +32,9 @@ impl SyncNotificationHandler for DidCloseNotebookHandler {
             .close_document(&key)
             .with_failure_code(lsp_server::ErrorCode::InternalError)?;
 
-        if let Some(db) = session.workspace_db_for_path_mut(path.as_std_path()) {
-            File::sync_path(db, &path);
+        if let AnySystemPath::SystemVirtual(virtual_path) = path {
+            let db = session.default_workspace_db_mut();
+            db.apply_changes(vec![ChangeEvent::DeletedVirtual(virtual_path)], None);
         }
 
         Ok(())

crates/red_knot_server/src/server/api/notifications/did_open.rs

@@ -1,13 +1,14 @@
 use lsp_types::notification::DidOpenTextDocument;
 use lsp_types::DidOpenTextDocumentParams;
 
-use ruff_db::files::system_path_to_file;
+use red_knot_workspace::watch::ChangeEvent;
+use ruff_db::Db;
 
 use crate::server::api::traits::{NotificationHandler, SyncNotificationHandler};
 use crate::server::client::{Notifier, Requester};
 use crate::server::Result;
 use crate::session::Session;
-use crate::system::url_to_system_path;
+use crate::system::{url_to_any_system_path, AnySystemPath};
 use crate::TextDocument;
 
 pub(crate) struct DidOpenTextDocumentHandler;
@@ -23,17 +24,25 @@ impl SyncNotificationHandler for DidOpenTextDocumentHandler {
         _requester: &mut Requester,
         params: DidOpenTextDocumentParams,
     ) -> Result<()> {
-        let Ok(path) = url_to_system_path(&params.text_document.uri) else {
+        let Ok(path) = url_to_any_system_path(&params.text_document.uri) else {
             return Ok(());
         };
 
         let document = TextDocument::new(params.text_document.text, params.text_document.version);
         session.open_text_document(params.text_document.uri, document);
 
-        if let Some(db) = session.workspace_db_for_path_mut(path.as_std_path()) {
-            // TODO(dhruvmanila): Store the `file` in `DocumentController`
-            let file = system_path_to_file(db, &path).unwrap();
-            file.sync(db);
+        match path {
+            AnySystemPath::System(path) => {
+                let db = match session.workspace_db_for_path_mut(path.as_std_path()) {
+                    Some(db) => db,
+                    None => session.default_workspace_db_mut(),
+                };
+                db.apply_changes(vec![ChangeEvent::Opened(path)], None);
+            }
+            AnySystemPath::SystemVirtual(virtual_path) => {
+                let db = session.default_workspace_db_mut();
+                db.files().virtual_file(db, &virtual_path);
+            }
         }
 
         // TODO(dhruvmanila): Publish diagnostics if the client doesn't support pull diagnostics

crates/red_knot_server/src/server/api/notifications/did_open_notebook.rs

@@ -2,7 +2,8 @@ use lsp_server::ErrorCode;
 use lsp_types::notification::DidOpenNotebookDocument;
 use lsp_types::DidOpenNotebookDocumentParams;
 
-use ruff_db::files::system_path_to_file;
+use red_knot_workspace::watch::ChangeEvent;
+use ruff_db::Db;
 
 use crate::edit::NotebookDocument;
 use crate::server::api::traits::{NotificationHandler, SyncNotificationHandler};
@@ -10,7 +11,7 @@ use crate::server::api::LSPResult;
 use crate::server::client::{Notifier, Requester};
 use crate::server::Result;
 use crate::session::Session;
-use crate::system::url_to_system_path;
+use crate::system::{url_to_any_system_path, AnySystemPath};
 
 pub(crate) struct DidOpenNotebookHandler;
 
@@ -25,7 +26,7 @@ impl SyncNotificationHandler for DidOpenNotebookHandler {
         _requester: &mut Requester,
         params: DidOpenNotebookDocumentParams,
     ) -> Result<()> {
-        let Ok(path) = url_to_system_path(&params.notebook_document.uri) else {
+        let Ok(path) = url_to_any_system_path(&params.notebook_document.uri) else {
             return Ok(());
         };
 
@@ -38,10 +39,18 @@ impl SyncNotificationHandler for DidOpenNotebookHandler {
         .with_failure_code(ErrorCode::InternalError)?;
         session.open_notebook_document(params.notebook_document.uri.clone(), notebook);
 
-        if let Some(db) = session.workspace_db_for_path_mut(path.as_std_path()) {
-            // TODO(dhruvmanila): Store the `file` in `DocumentController`
-            let file = system_path_to_file(db, &path).unwrap();
-            file.sync(db);
+        match path {
+            AnySystemPath::System(path) => {
+                let db = match session.workspace_db_for_path_mut(path.as_std_path()) {
+                    Some(db) => db,
+                    None => session.default_workspace_db_mut(),
+                };
+                db.apply_changes(vec![ChangeEvent::Opened(path)], None);
+            }
+            AnySystemPath::SystemVirtual(virtual_path) => {
+                let db = session.default_workspace_db_mut();
+                db.files().virtual_file(db, &virtual_path);
+            }
         }
 
         // TODO(dhruvmanila): Publish diagnostics if the client doesn't support pull diagnostics

crates/red_knot_server/src/server/api/requests/diagnostic.rs

@@ -26,13 +26,11 @@ impl BackgroundDocumentRequestHandler for DocumentDiagnosticRequestHandler {
 
     fn run_with_snapshot(
         snapshot: DocumentSnapshot,
-        db: Option<RootDatabase>,
+        db: RootDatabase,
         _notifier: Notifier,
         _params: DocumentDiagnosticParams,
     ) -> Result<DocumentDiagnosticReportResult> {
-        let diagnostics = db
-            .map(|db| compute_diagnostics(&snapshot, &db))
-            .unwrap_or_default();
+        let diagnostics = compute_diagnostics(&snapshot, &db);
 
         Ok(DocumentDiagnosticReportResult::Report(
             DocumentDiagnosticReport::Full(RelatedFullDocumentDiagnosticReport {
@@ -48,10 +46,19 @@ impl BackgroundDocumentRequestHandler for DocumentDiagnosticRequestHandler {
 
 fn compute_diagnostics(snapshot: &DocumentSnapshot, db: &RootDatabase) -> Vec<Diagnostic> {
     let Some(file) = snapshot.file(db) else {
+        tracing::info!(
+            "No file found for snapshot for '{}'",
+            snapshot.query().file_url()
+        );
         return vec![];
     };
-    let Ok(diagnostics) = db.check_file(file) else {
-        return vec![];
+
+    let diagnostics = match db.check_file(file) {
+        Ok(diagnostics) => diagnostics,
+        Err(cancelled) => {
+            tracing::info!("Diagnostics computation {cancelled}");
+            return vec![];
+        }
     };
 
     diagnostics
@@ -65,12 +72,12 @@ fn to_lsp_diagnostic(message: &str) -> Diagnostic {
     let words = message.split(':').collect::<Vec<_>>();
 
     let (range, message) = match words.as_slice() {
-        [_filename, line, column, message] => {
-            let line = line.parse::<u32>().unwrap_or_default();
+        [_, _, line, column, message] | [_, line, column, message] => {
+            let line = line.parse::<u32>().unwrap_or_default().saturating_sub(1);
             let column = column.parse::<u32>().unwrap_or_default();
             (
                 Range::new(
-                    Position::new(line.saturating_sub(1), column.saturating_sub(1)),
+                    Position::new(line, column.saturating_sub(1)),
                     Position::new(line, column),
                 ),
                 message.trim(),

crates/red_knot_server/src/server/api/traits.rs

@@ -34,7 +34,7 @@ pub(super) trait BackgroundDocumentRequestHandler: RequestHandler {
 
     fn run_with_snapshot(
         snapshot: DocumentSnapshot,
-        db: Option<RootDatabase>,
+        db: RootDatabase,
         notifier: Notifier,
         params: <<Self as RequestHandler>::RequestType as Request>::Params,
     ) -> super::Result<<<Self as RequestHandler>::RequestType as Request>::Result>;

crates/red_knot_server/src/session.rs

@@ -6,16 +6,16 @@ use std::path::{Path, PathBuf};
 use std::sync::Arc;
 
 use anyhow::anyhow;
-use lsp_types::{ClientCapabilities, Url};
+use lsp_types::{ClientCapabilities, TextDocumentContentChangeEvent, Url};
 
-use red_knot_python_semantic::{ProgramSettings, PythonVersion, SearchPathSettings};
 use red_knot_workspace::db::RootDatabase;
 use red_knot_workspace::workspace::WorkspaceMetadata;
 use ruff_db::files::{system_path_to_file, File};
 use ruff_db::system::SystemPath;
+use ruff_db::Db;
 
-use crate::edit::{DocumentKey, NotebookDocument};
-use crate::system::{url_to_system_path, LSPSystem};
+use crate::edit::{DocumentKey, DocumentVersion, NotebookDocument};
+use crate::system::{url_to_any_system_path, AnySystemPath, LSPSystem};
 use crate::{PositionEncoding, TextDocument};
 
 pub(crate) use self::capabilities::ResolvedClientCapabilities;
@@ -67,19 +67,10 @@ impl Session {
                 .ok_or_else(|| anyhow!("Workspace path is not a valid UTF-8 path: {:?}", path))?;
             let system = LSPSystem::new(index.clone());
 
-            let metadata = WorkspaceMetadata::from_path(system_path, &system)?;
             // TODO(dhruvmanila): Get the values from the client settings
-            let program_settings = ProgramSettings {
-                target_version: PythonVersion::default(),
-                search_paths: SearchPathSettings {
-                    extra_paths: vec![],
-                    src_root: system_path.to_path_buf(),
-                    site_packages: vec![],
-                    custom_typeshed: None,
-                },
-            };
+            let metadata = WorkspaceMetadata::from_path(system_path, &system, None)?;
             // TODO(micha): Handle the case where the program settings are incorrect more gracefully.
-            workspaces.insert(path, RootDatabase::new(metadata, program_settings, system)?);
+            workspaces.insert(path, RootDatabase::new(metadata, system)?);
         }
 
         Ok(Self {
@@ -92,6 +83,12 @@ impl Session {
         })
     }
 
+    // TODO(dhruvmanila): Ideally, we should have a single method for `workspace_db_for_path_mut`
+    // and `default_workspace_db_mut` but the borrow checker doesn't allow that.
+    // https://github.com/astral-sh/ruff/pull/13041#discussion_r1726725437
+
+    /// Returns a reference to the workspace [`RootDatabase`] corresponding to the given path, if
+    /// any.
     pub(crate) fn workspace_db_for_path(&self, path: impl AsRef<Path>) -> Option<&RootDatabase> {
         self.workspaces
             .range(..=path.as_ref().to_path_buf())
@@ -99,6 +96,8 @@ impl Session {
             .map(|(_, db)| db)
     }
 
+    /// Returns a mutable reference to the workspace [`RootDatabase`] corresponding to the given
+    /// path, if any.
     pub(crate) fn workspace_db_for_path_mut(
         &mut self,
         path: impl AsRef<Path>,
@@ -109,6 +108,19 @@ impl Session {
             .map(|(_, db)| db)
     }
 
+    /// Returns a reference to the default workspace [`RootDatabase`]. The default workspace is the
+    /// minimum root path in the workspace map.
+    pub(crate) fn default_workspace_db(&self) -> &RootDatabase {
+        // SAFETY: Currently, red knot only support a single workspace.
+        self.workspaces.values().next().unwrap()
+    }
+
+    /// Returns a mutable reference to the default workspace [`RootDatabase`].
+    pub(crate) fn default_workspace_db_mut(&mut self) -> &mut RootDatabase {
+        // SAFETY: Currently, red knot only support a single workspace.
+        self.workspaces.values_mut().next().unwrap()
+    }
+
     pub fn key_from_url(&self, url: Url) -> DocumentKey {
         self.index().key_from_url(url)
     }
@@ -135,6 +147,20 @@ impl Session {
         self.index_mut().open_text_document(url, document);
     }
 
+    /// Updates a text document at the associated `key`.
+    ///
+    /// The document key must point to a text document, or this will throw an error.
+    pub(crate) fn update_text_document(
+        &mut self,
+        key: &DocumentKey,
+        content_changes: Vec<TextDocumentContentChangeEvent>,
+        new_version: DocumentVersion,
+    ) -> crate::Result<()> {
+        let position_encoding = self.position_encoding;
+        self.index_mut()
+            .update_text_document(key, content_changes, new_version, position_encoding)
+    }
+
     /// De-registers a document, specified by its key.
     /// Calling this multiple times for the same document is a logic error.
     pub(crate) fn close_document(&mut self, key: &DocumentKey) -> crate::Result<()> {
@@ -221,6 +247,7 @@ impl Drop for MutIndexGuard<'_> {
 
 /// An immutable snapshot of `Session` that references
 /// a specific document.
+#[derive(Debug)]
 pub struct DocumentSnapshot {
     resolved_client_capabilities: Arc<ResolvedClientCapabilities>,
     document_ref: index::DocumentQuery,
@@ -241,7 +268,12 @@ impl DocumentSnapshot {
     }
 
     pub(crate) fn file(&self, db: &RootDatabase) -> Option<File> {
-        let path = url_to_system_path(self.document_ref.file_url()).ok()?;
-        system_path_to_file(db, path).ok()
+        match url_to_any_system_path(self.document_ref.file_url()).ok()? {
+            AnySystemPath::System(path) => system_path_to_file(db, path).ok(),
+            AnySystemPath::SystemVirtual(virtual_path) => db
+                .files()
+                .try_virtual_file(&virtual_path)
+                .map(|virtual_file| virtual_file.file()),
+        }
     }
 }

crates/red_knot_server/src/system.rs

@@ -8,27 +8,40 @@ use ruff_db::file_revision::FileRevision;
 use ruff_db::system::walk_directory::WalkDirectoryBuilder;
 use ruff_db::system::{
     DirectoryEntry, FileType, Metadata, OsSystem, Result, System, SystemPath, SystemPathBuf,
-    SystemVirtualPath,
+    SystemVirtualPath, SystemVirtualPathBuf,
 };
 use ruff_notebook::{Notebook, NotebookError};
 
 use crate::session::index::Index;
 use crate::DocumentQuery;
 
-/// Converts the given [`Url`] to a [`SystemPathBuf`].
+/// Converts the given [`Url`] to an [`AnySystemPath`].
+///
+/// If the URL scheme is `file`, then the path is converted to a [`SystemPathBuf`]. Otherwise, the
+/// URL is converted to a [`SystemVirtualPathBuf`].
 ///
 /// This fails in the following cases:
-/// * The URL scheme is not `file`.
 /// * The URL cannot be converted to a file path (refer to [`Url::to_file_path`]).
 /// * If the URL is not a valid UTF-8 string.
-pub(crate) fn url_to_system_path(url: &Url) -> std::result::Result<SystemPathBuf, ()> {
+pub(crate) fn url_to_any_system_path(url: &Url) -> std::result::Result<AnySystemPath, ()> {
     if url.scheme() == "file" {
-        Ok(SystemPathBuf::from_path_buf(url.to_file_path()?).map_err(|_| ())?)
+        Ok(AnySystemPath::System(
+            SystemPathBuf::from_path_buf(url.to_file_path()?).map_err(|_| ())?,
+        ))
     } else {
-        Err(())
+        Ok(AnySystemPath::SystemVirtual(
+            SystemVirtualPath::new(url.as_str()).to_path_buf(),
+        ))
     }
 }
 
+/// Represents either a [`SystemPath`] or a [`SystemVirtualPath`].
+#[derive(Debug)]
+pub(crate) enum AnySystemPath {
+    System(SystemPathBuf),
+    SystemVirtual(SystemVirtualPathBuf),
+}
+
 #[derive(Debug)]
 pub(crate) struct LSPSystem {
     /// A read-only copy of the index where the server stores all the open documents and settings.
@@ -144,19 +157,6 @@ impl System for LSPSystem {
         }
     }
 
-    fn virtual_path_metadata(&self, path: &SystemVirtualPath) -> Result<Metadata> {
-        // Virtual paths only exists in the LSP system, so we don't need to check the OS system.
-        let document = self
-            .system_virtual_path_to_document_ref(path)?
-            .ok_or_else(|| virtual_path_not_found(path))?;
-
-        Ok(Metadata::new(
-            document_revision(&document),
-            None,
-            FileType::File,
-        ))
-    }
-
     fn read_virtual_path_to_string(&self, path: &SystemVirtualPath) -> Result<String> {
         let document = self
             .system_virtual_path_to_document_ref(path)?

crates/red_knot_wasm/src/lib.rs

@@ -3,8 +3,8 @@ use std::any::Any;
 use js_sys::Error;
 use wasm_bindgen::prelude::*;
 
-use red_knot_python_semantic::{ProgramSettings, SearchPathSettings};
 use red_knot_workspace::db::RootDatabase;
+use red_knot_workspace::workspace::settings::Configuration;
 use red_knot_workspace::workspace::WorkspaceMetadata;
 use ruff_db::files::{system_path_to_file, File};
 use ruff_db::system::walk_directory::WalkDirectoryBuilder;
@@ -41,16 +41,17 @@ impl Workspace {
     #[wasm_bindgen(constructor)]
     pub fn new(root: &str, settings: &Settings) -> Result<Workspace, Error> {
         let system = WasmSystem::new(SystemPath::new(root));
-        let workspace =
-            WorkspaceMetadata::from_path(SystemPath::new(root), &system).map_err(into_error)?;
+        let workspace = WorkspaceMetadata::from_path(
+            SystemPath::new(root),
+            &system,
+            Some(Configuration {
+                target_version: Some(settings.target_version.into()),
+                ..Configuration::default()
+            }),
+        )
+        .map_err(into_error)?;
 
-        let program_settings = ProgramSettings {
-            target_version: settings.target_version.into(),
-            search_paths: SearchPathSettings::default(),
-        };
-
-        let db =
-            RootDatabase::new(workspace, program_settings, system.clone()).map_err(into_error)?;
+        let db = RootDatabase::new(workspace, system.clone()).map_err(into_error)?;
 
         Ok(Self { db, system })
     }
@@ -233,13 +234,6 @@ impl System for WasmSystem {
         Notebook::from_source_code(&content)
     }
 
-    fn virtual_path_metadata(
-        &self,
-        _path: &SystemVirtualPath,
-    ) -> ruff_db::system::Result<Metadata> {
-        Err(not_found())
-    }
-
     fn read_virtual_path_to_string(
         &self,
         _path: &SystemVirtualPath,

crates/red_knot_workspace/Cargo.toml

@@ -24,7 +24,6 @@ crossbeam = { workspace = true }
 notify = { workspace = true }
 rustc-hash = { workspace = true }
 salsa = { workspace = true }
-thiserror = { workspace = true }
 tracing = { workspace = true }
 
 [dev-dependencies]

crates/red_knot_workspace/resources/test/corpus/04_assign_named_expr.py

@@ -0,0 +1,2 @@
+x = 0
+(x := x + 1)

crates/red_knot_workspace/resources/test/corpus/10_if_with_named_expr.py

@@ -0,0 +1,3 @@
+x = 0
+if x := x + 1:
+    pass

crates/red_knot_workspace/resources/test/corpus/85_match_guard_with_named_expr.py

@@ -0,0 +1,3 @@
+match x:
+    case [1, 0] if x := x[:0]:
+        y = 1

crates/red_knot_workspace/src/db.rs

@@ -1,15 +1,14 @@
 use std::panic::RefUnwindSafe;
 use std::sync::Arc;
 
-use red_knot_python_semantic::{
-    vendored_typeshed_stubs, Db as SemanticDb, Program, ProgramSettings,
-};
+use salsa::plumbing::ZalsaDatabase;
+use salsa::{Cancelled, Event};
+
+use red_knot_python_semantic::{vendored_typeshed_stubs, Db as SemanticDb, Program};
 use ruff_db::files::{File, Files};
 use ruff_db::system::System;
 use ruff_db::vendored::VendoredFileSystem;
 use ruff_db::{Db as SourceDb, Upcast};
-use salsa::plumbing::ZalsaDatabase;
-use salsa::{Cancelled, Event};
 
 use crate::workspace::{check_file, Workspace, WorkspaceMetadata};
 
@@ -27,11 +26,7 @@ pub struct RootDatabase {
 }
 
 impl RootDatabase {
-    pub fn new<S>(
-        workspace: WorkspaceMetadata,
-        settings: ProgramSettings,
-        system: S,
-    ) -> anyhow::Result<Self>
+    pub fn new<S>(workspace: WorkspaceMetadata, system: S) -> anyhow::Result<Self>
     where
         S: System + 'static + Send + Sync + RefUnwindSafe,
     {
@@ -42,11 +37,11 @@ impl RootDatabase {
             system: Arc::new(system),
         };
 
-        let workspace = Workspace::from_metadata(&db, workspace);
         // Initialize the `Program` singleton
-        Program::from_settings(&db, settings)?;
+        Program::from_settings(&db, workspace.settings().program())?;
+
+        db.workspace = Some(Workspace::from_metadata(&db, workspace));
 
-        db.workspace = Some(workspace);
         Ok(db)
     }
 
@@ -160,9 +155,10 @@ impl Db for RootDatabase {}
 
 #[cfg(test)]
 pub(crate) mod tests {
-    use salsa::Event;
     use std::sync::Arc;
 
+    use salsa::Event;
+
     use red_knot_python_semantic::{vendored_typeshed_stubs, Db as SemanticDb};
     use ruff_db::files::Files;
     use ruff_db::system::{DbWithTestSystem, System, TestSystem};

crates/red_knot_workspace/src/db/changes.rs

@@ -1,22 +1,33 @@
-use rustc_hash::FxHashSet;
-
+use red_knot_python_semantic::Program;
 use ruff_db::files::{system_path_to_file, File, Files};
 use ruff_db::system::walk_directory::WalkState;
 use ruff_db::system::SystemPath;
 use ruff_db::Db;
+use rustc_hash::FxHashSet;
 
 use crate::db::RootDatabase;
 use crate::watch;
 use crate::watch::{CreatedKind, DeletedKind};
+use crate::workspace::settings::Configuration;
 use crate::workspace::WorkspaceMetadata;
 
 impl RootDatabase {
-    #[tracing::instrument(level = "debug", skip(self, changes))]
-    pub fn apply_changes(&mut self, changes: Vec<watch::ChangeEvent>) {
+    #[tracing::instrument(level = "debug", skip(self, changes, base_configuration))]
+    pub fn apply_changes(
+        &mut self,
+        changes: Vec<watch::ChangeEvent>,
+        base_configuration: Option<&Configuration>,
+    ) {
         let workspace = self.workspace();
         let workspace_path = workspace.root(self).to_path_buf();
+        let program = Program::get(self);
+        let custom_stdlib_versions_path = program
+            .custom_stdlib_search_path(self)
+            .map(|path| path.join("VERSIONS"));
 
         let mut workspace_change = false;
+        // Changes to a custom stdlib path's VERSIONS
+        let mut custom_stdlib_change = false;
         // Packages that need reloading
         let mut changed_packages = FxHashSet::default();
         // Paths that were added
@@ -39,7 +50,7 @@ impl RootDatabase {
         };
 
         for change in changes {
-            if let Some(path) = change.path() {
+            if let Some(path) = change.system_path() {
                 if matches!(
                     path.file_name(),
                     Some(".gitignore" | ".ignore" | "ruff.toml" | ".ruff.toml" | "pyproject.toml")
@@ -54,10 +65,15 @@ impl RootDatabase {
 
                     continue;
                 }
+
+                if Some(path) == custom_stdlib_versions_path.as_deref() {
+                    custom_stdlib_change = true;
+                }
             }
 
             match change {
-                watch::ChangeEvent::Changed { path, kind: _ } => sync_path(self, &path),
+                watch::ChangeEvent::Changed { path, kind: _ }
+                | watch::ChangeEvent::Opened(path) => sync_path(self, &path),
 
                 watch::ChangeEvent::Created { kind, path } => {
                     match kind {
@@ -100,7 +116,13 @@ impl RootDatabase {
                     } else {
                         sync_recursively(self, &path);
 
-                        // TODO: Remove after converting `package.files()` to a salsa query.
+                        if custom_stdlib_versions_path
+                            .as_ref()
+                            .is_some_and(|versions_path| versions_path.starts_with(&path))
+                        {
+                            custom_stdlib_change = true;
+                        }
+
                         if let Some(package) = workspace.package(self, &path) {
                             changed_packages.insert(package);
                         } else {
@@ -109,6 +131,17 @@ impl RootDatabase {
                     }
                 }
 
+                watch::ChangeEvent::CreatedVirtual(path)
+                | watch::ChangeEvent::ChangedVirtual(path) => {
+                    File::sync_virtual_path(self, &path);
+                }
+
+                watch::ChangeEvent::DeletedVirtual(path) => {
+                    if let Some(virtual_file) = self.files().try_virtual_file(&path) {
+                        virtual_file.close(self);
+                    }
+                }
+
                 watch::ChangeEvent::Rescan => {
                     workspace_change = true;
                     Files::sync_all(self);
@@ -118,7 +151,11 @@ impl RootDatabase {
         }
 
         if workspace_change {
-            match WorkspaceMetadata::from_path(&workspace_path, self.system()) {
+            match WorkspaceMetadata::from_path(
+                &workspace_path,
+                self.system(),
+                base_configuration.cloned(),
+            ) {
                 Ok(metadata) => {
                     tracing::debug!("Reloading workspace after structural change.");
                     // TODO: Handle changes in the program settings.
@@ -130,6 +167,11 @@ impl RootDatabase {
             }
 
             return;
+        } else if custom_stdlib_change {
+            let search_paths = workspace.search_path_settings(self).clone();
+            if let Err(error) = program.update_search_paths(self, &search_paths) {
+                tracing::error!("Failed to set the new search paths: {error}");
+            }
         }
 
         let mut added_paths = added_paths.into_iter().filter(|path| {

crates/red_knot_workspace/src/lib.rs

@@ -1,5 +1,4 @@
 pub mod db;
 pub mod lint;
-pub mod site_packages;
 pub mod watch;
 pub mod workspace;

crates/red_knot_workspace/src/lint.rs

@@ -7,7 +7,7 @@ use red_knot_python_semantic::types::Type;
 use red_knot_python_semantic::{HasTy, ModuleName, SemanticModel};
 use ruff_db::files::File;
 use ruff_db::parsed::{parsed_module, ParsedModule};
-use ruff_db::source::{line_index, source_text, SourceText};
+use ruff_db::source::{source_text, SourceText};
 use ruff_python_ast as ast;
 use ruff_python_ast::visitor::{walk_expr, walk_stmt, Visitor};
 use ruff_text_size::{Ranged, TextSize};
@@ -48,19 +48,6 @@ pub(crate) fn lint_syntax(db: &dyn Db, file_id: File) -> Vec<String> {
         };
         visitor.visit_body(&ast.body);
         diagnostics = visitor.diagnostics;
-    } else {
-        let path = file_id.path(db);
-        let line_index = line_index(db.upcast(), file_id);
-        diagnostics.extend(parsed.errors().iter().map(|err| {
-            let source_location = line_index.source_location(err.location.start(), source.as_str());
-            format!(
-                "{}:{}:{}: {}",
-                path.as_str(),
-                source_location.row,
-                source_location.column,
-                err,
-            )
-        }));
     }
 
     diagnostics
@@ -286,14 +273,9 @@ mod tests {
 
         Program::from_settings(
             &db,
-            ProgramSettings {
+            &ProgramSettings {
                 target_version: PythonVersion::default(),
-                search_paths: SearchPathSettings {
-                    extra_paths: Vec::new(),
-                    src_root,
-                    site_packages: vec![],
-                    custom_typeshed: None,
-                },
+                search_paths: SearchPathSettings::new(src_root),
             },
         )
         .expect("Valid program settings");

crates/red_knot_workspace/src/watch.rs

@@ -1,4 +1,4 @@
-use ruff_db::system::{SystemPath, SystemPathBuf};
+use ruff_db::system::{SystemPath, SystemPathBuf, SystemVirtualPathBuf};
 pub use watcher::{directory_watcher, EventHandler, Watcher};
 pub use workspace_watcher::WorkspaceWatcher;
 
@@ -20,6 +20,9 @@ mod workspace_watcher;
 /// event instead of emitting an event for each file or subdirectory in that path.
 #[derive(Debug, PartialEq, Eq)]
 pub enum ChangeEvent {
+    /// The file corresponding to the given path was opened in an editor.
+    Opened(SystemPathBuf),
+
     /// A new path was created
     Created {
         path: SystemPathBuf,
@@ -38,6 +41,15 @@ pub enum ChangeEvent {
         kind: DeletedKind,
     },
 
+    /// A new virtual path was created.
+    CreatedVirtual(SystemVirtualPathBuf),
+
+    /// The content of a virtual path was changed.
+    ChangedVirtual(SystemVirtualPathBuf),
+
+    /// A virtual path was deleted.
+    DeletedVirtual(SystemVirtualPathBuf),
+
     /// The file watcher failed to observe some changes and now is out of sync with the file system.
     ///
     /// This can happen if many files are changed at once. The consumer should rescan all files to catch up
@@ -46,16 +58,27 @@ pub enum ChangeEvent {
 }
 
 impl ChangeEvent {
-    pub fn file_name(&self) -> Option<&str> {
-        self.path().and_then(|path| path.file_name())
+    /// Creates a new [`Changed`] event for the file content at the given path.
+    ///
+    /// [`Changed`]: ChangeEvent::Changed
+    pub fn file_content_changed(path: SystemPathBuf) -> ChangeEvent {
+        ChangeEvent::Changed {
+            path,
+            kind: ChangedKind::FileContent,
+        }
     }
 
-    pub fn path(&self) -> Option<&SystemPath> {
+    pub fn file_name(&self) -> Option<&str> {
+        self.system_path().and_then(|path| path.file_name())
+    }
+
+    pub fn system_path(&self) -> Option<&SystemPath> {
         match self {
-            ChangeEvent::Created { path, .. }
+            ChangeEvent::Opened(path)
+            | ChangeEvent::Created { path, .. }
             | ChangeEvent::Changed { path, .. }
             | ChangeEvent::Deleted { path, .. } => Some(path),
-            ChangeEvent::Rescan => None,
+            _ => None,
         }
     }
 }

crates/red_knot_workspace/src/workspace.rs

@@ -5,6 +5,8 @@ use salsa::{Durability, Setter as _};
 
 pub use metadata::{PackageMetadata, WorkspaceMetadata};
 use red_knot_python_semantic::types::check_types;
+use red_knot_python_semantic::SearchPathSettings;
+use ruff_db::parsed::parsed_module;
 use ruff_db::source::{line_index, source_text, SourceDiagnostic};
 use ruff_db::{
     files::{system_path_to_file, File},
@@ -21,6 +23,7 @@ use crate::{
 
 mod files;
 mod metadata;
+pub mod settings;
 
 /// The project workspace as a Salsa ingredient.
 ///
@@ -81,6 +84,10 @@ pub struct Workspace {
     /// The (first-party) packages in this workspace.
     #[return_ref]
     package_tree: BTreeMap<SystemPathBuf, Package>,
+
+    /// The unresolved search path configuration.
+    #[return_ref]
+    pub search_path_settings: SearchPathSettings,
 }
 
 /// A first-party package in a workspace.
@@ -109,10 +116,14 @@ impl Workspace {
             packages.insert(package.root.clone(), Package::from_metadata(db, package));
         }
 
-        Workspace::builder(metadata.root, packages)
-            .durability(Durability::MEDIUM)
-            .open_fileset_durability(Durability::LOW)
-            .new(db)
+        Workspace::builder(
+            metadata.root,
+            packages,
+            metadata.settings.program.search_paths,
+        )
+        .durability(Durability::MEDIUM)
+        .open_fileset_durability(Durability::LOW)
+        .new(db)
     }
 
     pub fn root(self, db: &dyn Db) -> &SystemPath {
@@ -143,6 +154,11 @@ impl Workspace {
             new_packages.insert(path, package);
         }
 
+        if &metadata.settings.program.search_paths != self.search_path_settings(db) {
+            self.set_search_path_settings(db)
+                .to(metadata.settings.program.search_paths);
+        }
+
         self.set_package_tree(db).to(new_packages);
     }
 
@@ -331,11 +347,7 @@ impl Package {
                     tracing::debug_span!("index_package_files", package = %self.name(db)).entered();
 
                 let files = discover_package_files(db, self.root(db));
-                tracing::info!(
-                    "Indexed {} files for package '{}'",
-                    files.len(),
-                    self.name(db)
-                );
+                tracing::info!("Found {} files in package '{}'", files.len(), self.name(db));
                 vacant.set(files)
             }
             Index::Indexed(indexed) => indexed,
@@ -393,6 +405,17 @@ pub(super) fn check_file(db: &dyn Db, file: File) -> Vec<String> {
         return diagnostics;
     }
 
+    let parsed = parsed_module(db.upcast(), file);
+
+    if !parsed.errors().is_empty() {
+        let path = file.path(db);
+        let line_index = line_index(db.upcast(), file);
+        diagnostics.extend(parsed.errors().iter().map(|err| {
+            let source_location = line_index.source_location(err.location.start(), source.as_str());
+            format!("{path}:{source_location}: {message}", message = err.error)
+        }));
+    }
+
     for diagnostic in check_types(db.upcast(), file) {
         let index = line_index(db.upcast(), diagnostic.file());
         let location = index.source_location(diagnostic.start(), source.as_str());

crates/red_knot_workspace/src/workspace/metadata.rs

@@ -1,3 +1,4 @@
+use crate::workspace::settings::{Configuration, WorkspaceSettings};
 use ruff_db::system::{System, SystemPath, SystemPathBuf};
 use ruff_python_ast::name::Name;
 
@@ -7,6 +8,8 @@ pub struct WorkspaceMetadata {
 
     /// The (first-party) packages in this workspace.
     pub(super) packages: Vec<PackageMetadata>,
+
+    pub(super) settings: WorkspaceSettings,
 }
 
 /// A first-party package in a workspace.
@@ -21,7 +24,11 @@ pub struct PackageMetadata {
 
 impl WorkspaceMetadata {
     /// Discovers the closest workspace at `path` and returns its metadata.
-    pub fn from_path(path: &SystemPath, system: &dyn System) -> anyhow::Result<WorkspaceMetadata> {
+    pub fn from_path(
+        path: &SystemPath,
+        system: &dyn System,
+        base_configuration: Option<Configuration>,
+    ) -> anyhow::Result<WorkspaceMetadata> {
         assert!(
             system.is_directory(path),
             "Workspace root path must be a directory"
@@ -38,9 +45,20 @@ impl WorkspaceMetadata {
             root: root.clone(),
         };
 
+        // TODO: Load the configuration from disk.
+        let mut configuration = Configuration::default();
+
+        if let Some(base_configuration) = base_configuration {
+            configuration.extend(base_configuration);
+        }
+
+        // TODO: Respect the package configurations when resolving settings (e.g. for the target version).
+        let settings = configuration.into_workspace_settings(&root);
+
         let workspace = WorkspaceMetadata {
             root,
             packages: vec![package],
+            settings,
         };
 
         Ok(workspace)
@@ -53,6 +71,10 @@ impl WorkspaceMetadata {
     pub fn packages(&self) -> &[PackageMetadata] {
         &self.packages
     }
+
+    pub fn settings(&self) -> &WorkspaceSettings {
+        &self.settings
+    }
 }
 
 impl PackageMetadata {

crates/red_knot_workspace/src/workspace/settings.rs

@@ -0,0 +1,89 @@
+use red_knot_python_semantic::{ProgramSettings, PythonVersion, SearchPathSettings, SitePackages};
+use ruff_db::system::{SystemPath, SystemPathBuf};
+
+/// The resolved configurations.
+///
+/// The main difference to [`Configuration`] is that default values are filled in.
+#[derive(Debug, Clone)]
+pub struct WorkspaceSettings {
+    pub(super) program: ProgramSettings,
+}
+
+impl WorkspaceSettings {
+    pub fn program(&self) -> &ProgramSettings {
+        &self.program
+    }
+}
+
+/// The configuration for the workspace or a package.
+#[derive(Debug, Default, Clone)]
+pub struct Configuration {
+    pub target_version: Option<PythonVersion>,
+    pub search_paths: SearchPathConfiguration,
+}
+
+impl Configuration {
+    /// Extends this configuration by using the values from `with` for all values that are absent in `self`.
+    pub fn extend(&mut self, with: Configuration) {
+        self.target_version = self.target_version.or(with.target_version);
+        self.search_paths.extend(with.search_paths);
+    }
+
+    pub fn into_workspace_settings(self, workspace_root: &SystemPath) -> WorkspaceSettings {
+        WorkspaceSettings {
+            program: ProgramSettings {
+                target_version: self.target_version.unwrap_or_default(),
+                search_paths: self.search_paths.into_settings(workspace_root),
+            },
+        }
+    }
+}
+
+#[derive(Debug, Default, Clone, Eq, PartialEq)]
+pub struct SearchPathConfiguration {
+    /// List of user-provided paths that should take first priority in the module resolution.
+    /// Examples in other type checkers are mypy's MYPYPATH environment variable,
+    /// or pyright's stubPath configuration setting.
+    pub extra_paths: Option<Vec<SystemPathBuf>>,
+
+    /// The root of the workspace, used for finding first-party modules.
+    pub src_root: Option<SystemPathBuf>,
+
+    /// Optional path to a "custom typeshed" directory on disk for us to use for standard-library types.
+    /// If this is not provided, we will fallback to our vendored typeshed stubs for the stdlib,
+    /// bundled as a zip file in the binary
+    pub custom_typeshed: Option<SystemPathBuf>,
+
+    /// The path to the user's `site-packages` directory, where third-party packages from ``PyPI`` are installed.
+    pub site_packages: Option<SitePackages>,
+}
+
+impl SearchPathConfiguration {
+    pub fn into_settings(self, workspace_root: &SystemPath) -> SearchPathSettings {
+        let site_packages = self.site_packages.unwrap_or(SitePackages::Known(vec![]));
+
+        SearchPathSettings {
+            extra_paths: self.extra_paths.unwrap_or_default(),
+            src_root: self
+                .src_root
+                .unwrap_or_else(|| workspace_root.to_path_buf()),
+            custom_typeshed: self.custom_typeshed,
+            site_packages,
+        }
+    }
+
+    pub fn extend(&mut self, with: SearchPathConfiguration) {
+        if let Some(extra_paths) = with.extra_paths {
+            self.extra_paths.get_or_insert(extra_paths);
+        }
+        if let Some(src_root) = with.src_root {
+            self.src_root.get_or_insert(src_root);
+        }
+        if let Some(custom_typeshed) = with.custom_typeshed {
+            self.custom_typeshed.get_or_insert(custom_typeshed);
+        }
+        if let Some(site_packages) = with.site_packages {
+            self.site_packages.get_or_insert(site_packages);
+        }
+    }
+}

crates/red_knot_workspace/tests/check.rs

@@ -1,6 +1,7 @@
-use red_knot_python_semantic::{
-    HasTy, ProgramSettings, PythonVersion, SearchPathSettings, SemanticModel,
-};
+use std::fs;
+use std::path::PathBuf;
+
+use red_knot_python_semantic::{HasTy, SemanticModel};
 use red_knot_workspace::db::RootDatabase;
 use red_knot_workspace::workspace::WorkspaceMetadata;
 use ruff_db::files::{system_path_to_file, File};
@@ -9,23 +10,11 @@ use ruff_db::system::{OsSystem, SystemPath, SystemPathBuf};
 use ruff_python_ast::visitor::source_order;
 use ruff_python_ast::visitor::source_order::SourceOrderVisitor;
 use ruff_python_ast::{Alias, Expr, Parameter, ParameterWithDefault, Stmt};
-use std::fs;
-use std::path::PathBuf;
 
-fn setup_db(workspace_root: SystemPathBuf) -> anyhow::Result<RootDatabase> {
-    let system = OsSystem::new(&workspace_root);
-    let workspace = WorkspaceMetadata::from_path(&workspace_root, &system)?;
-    let search_paths = SearchPathSettings {
-        extra_paths: vec![],
-        src_root: workspace_root,
-        custom_typeshed: None,
-        site_packages: vec![],
-    };
-    let settings = ProgramSettings {
-        target_version: PythonVersion::default(),
-        search_paths,
-    };
-    RootDatabase::new(workspace, settings, system)
+fn setup_db(workspace_root: &SystemPath) -> anyhow::Result<RootDatabase> {
+    let system = OsSystem::new(workspace_root);
+    let workspace = WorkspaceMetadata::from_path(workspace_root, &system, None)?;
+    RootDatabase::new(workspace, system)
 }
 
 /// Test that all snippets in testcorpus can be checked without panic
@@ -33,8 +22,9 @@ fn setup_db(workspace_root: SystemPathBuf) -> anyhow::Result<RootDatabase> {
 #[allow(clippy::print_stdout)]
 fn corpus_no_panic() -> anyhow::Result<()> {
     let corpus = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("resources/test/corpus");
-    let system_corpus = SystemPath::from_std_path(&corpus).expect("corpus path to be UTF8");
-    let db = setup_db(system_corpus.to_path_buf())?;
+    let system_corpus =
+        SystemPathBuf::from_path_buf(corpus.clone()).expect("corpus path to be UTF8");
+    let db = setup_db(&system_corpus)?;
 
     for path in fs::read_dir(&corpus).expect("corpus to be a directory") {
         let path = path.expect("path to not be an error").path();

crates/ruff/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "ruff"
-version = "0.6.1"
+version = "0.6.2"
 publish = true
 authors = { workspace = true }
 edition = { workspace = true }

crates/ruff_benchmark/benches/red_knot.rs

@@ -1,8 +1,9 @@
 #![allow(clippy::disallowed_names)]
 
-use red_knot_python_semantic::{ProgramSettings, PythonVersion, SearchPathSettings};
+use red_knot_python_semantic::PythonVersion;
 use red_knot_workspace::db::RootDatabase;
 use red_knot_workspace::watch::{ChangeEvent, ChangedKind};
+use red_knot_workspace::workspace::settings::Configuration;
 use red_knot_workspace::workspace::WorkspaceMetadata;
 use ruff_benchmark::criterion::{criterion_group, criterion_main, BatchSize, Criterion};
 use ruff_benchmark::TestFile;
@@ -86,18 +87,17 @@ fn setup_case() -> Case {
     .unwrap();
 
     let src_root = SystemPath::new("/src");
-    let metadata = WorkspaceMetadata::from_path(src_root, &system).unwrap();
-    let settings = ProgramSettings {
-        target_version: PythonVersion::PY312,
-        search_paths: SearchPathSettings {
-            extra_paths: vec![],
-            src_root: src_root.to_path_buf(),
-            site_packages: vec![],
-            custom_typeshed: None,
-        },
-    };
+    let metadata = WorkspaceMetadata::from_path(
+        src_root,
+        &system,
+        Some(Configuration {
+            target_version: Some(PythonVersion::PY312),
+            ..Configuration::default()
+        }),
+    )
+    .unwrap();
 
-    let mut db = RootDatabase::new(metadata, settings, system).unwrap();
+    let mut db = RootDatabase::new(metadata, system).unwrap();
     let parser = system_path_to_file(&db, parser_path).unwrap();
 
     db.workspace().open_file(&mut db, parser);
@@ -131,10 +131,13 @@ fn benchmark_incremental(criterion: &mut Criterion) {
             |case| {
                 let Case { db, .. } = case;
 
-                db.apply_changes(vec![ChangeEvent::Changed {
-                    path: case.re_path.to_path_buf(),
-                    kind: ChangedKind::FileContent,
-                }]);
+                db.apply_changes(
+                    vec![ChangeEvent::Changed {
+                        path: case.re_path.to_path_buf(),
+                        kind: ChangedKind::FileContent,
+                    }],
+                    None,
+                );
 
                 let result = db.check().unwrap();
 

crates/ruff_db/src/files.rs

@@ -12,7 +12,7 @@ use ruff_notebook::{Notebook, NotebookError};
 use crate::file_revision::FileRevision;
 use crate::files::file_root::FileRoots;
 use crate::files::private::FileStatus;
-use crate::system::{Metadata, SystemPath, SystemPathBuf, SystemVirtualPath, SystemVirtualPathBuf};
+use crate::system::{SystemPath, SystemPathBuf, SystemVirtualPath, SystemVirtualPathBuf};
 use crate::vendored::{VendoredPath, VendoredPathBuf};
 use crate::{vendored, Db, FxDashMap};
 
@@ -60,8 +60,8 @@ struct FilesInner {
     /// so that queries that depend on the existence of a file are re-executed when the file is created.
     system_by_path: FxDashMap<SystemPathBuf, File>,
 
-    /// Lookup table that maps [`SystemVirtualPathBuf`]s to salsa interned [`File`] instances.
-    system_virtual_by_path: FxDashMap<SystemVirtualPathBuf, File>,
+    /// Lookup table that maps [`SystemVirtualPathBuf`]s to [`VirtualFile`] instances.
+    system_virtual_by_path: FxDashMap<SystemVirtualPathBuf, VirtualFile>,
 
     /// Lookup table that maps vendored files to the salsa [`File`] ingredients.
     vendored_by_path: FxDashMap<VendoredPathBuf, File>,
@@ -147,31 +147,31 @@ impl Files {
         Ok(file)
     }
 
-    /// Looks up a virtual file by its `path`.
+    /// Create a new virtual file at the given path and store it for future lookups.
     ///
-    /// For a non-existing file, creates a new salsa [`File`] ingredient and stores it for future lookups.
-    ///
-    /// The operations fails if the system failed to provide a metadata for the path.
-    pub fn add_virtual_file(&self, db: &dyn Db, path: &SystemVirtualPath) -> Option<File> {
-        let file = match self.inner.system_virtual_by_path.entry(path.to_path_buf()) {
-            Entry::Occupied(entry) => *entry.get(),
-            Entry::Vacant(entry) => {
-                let metadata = db.system().virtual_path_metadata(path).ok()?;
+    /// This will always create a new file, overwriting any existing file at `path` in the internal
+    /// storage.
+    pub fn virtual_file(&self, db: &dyn Db, path: &SystemVirtualPath) -> VirtualFile {
+        tracing::trace!("Adding virtual file {}", path);
+        let virtual_file = VirtualFile(
+            File::builder(FilePath::SystemVirtual(path.to_path_buf()))
+                .status(FileStatus::Exists)
+                .revision(FileRevision::zero())
+                .permissions(None)
+                .new(db),
+        );
+        self.inner
+            .system_virtual_by_path
+            .insert(path.to_path_buf(), virtual_file);
+        virtual_file
+    }
 
-                tracing::trace!("Adding virtual file '{}'", path);
-
-                let file = File::builder(FilePath::SystemVirtual(path.to_path_buf()))
-                    .revision(metadata.revision())
-                    .permissions(metadata.permissions())
-                    .new(db);
-
-                entry.insert(file);
-
-                file
-            }
-        };
-
-        Some(file)
+    /// Tries to look up a virtual file by its path. Returns `None` if no such file exists yet.
+    pub fn try_virtual_file(&self, path: &SystemVirtualPath) -> Option<VirtualFile> {
+        self.inner
+            .system_virtual_by_path
+            .get(&path.to_path_buf())
+            .map(|entry| *entry.value())
     }
 
     /// Looks up the closest  root for `path`. Returns `None` if `path` isn't enclosed by any source root.
@@ -318,6 +318,9 @@ impl File {
             }
             FilePath::Vendored(vendored) => db.vendored().read_to_string(vendored),
             FilePath::SystemVirtual(system_virtual) => {
+                // Add a dependency on the revision to ensure the operation gets re-executed when the file changes.
+                let _ = self.revision(db);
+
                 db.system().read_virtual_path_to_string(system_virtual)
             }
         }
@@ -342,6 +345,9 @@ impl File {
                 "Reading a notebook from the vendored file system is not supported.",
             ))),
             FilePath::SystemVirtual(system_virtual) => {
+                // Add a dependency on the revision to ensure the operation gets re-executed when the file changes.
+                let _ = self.revision(db);
+
                 db.system().read_virtual_path_to_notebook(system_virtual)
             }
         }
@@ -354,6 +360,13 @@ impl File {
         Self::sync_system_path(db, &absolute, None);
     }
 
+    /// Increments the revision for the virtual file at `path`.
+    pub fn sync_virtual_path(db: &mut dyn Db, path: &SystemVirtualPath) {
+        if let Some(virtual_file) = db.files().try_virtual_file(path) {
+            virtual_file.sync(db);
+        }
+    }
+
     /// Syncs the [`File`]'s state with the state of the file on the system.
     pub fn sync(self, db: &mut dyn Db) {
         let path = self.path(db).clone();
@@ -366,29 +379,20 @@ impl File {
             FilePath::Vendored(_) => {
                 // Readonly, can never be out of date.
             }
-            FilePath::SystemVirtual(system_virtual) => {
-                Self::sync_system_virtual_path(db, &system_virtual, self);
+            FilePath::SystemVirtual(_) => {
+                VirtualFile(self).sync(db);
             }
         }
     }
 
+    /// Private method providing the implementation for [`Self::sync_path`] and [`Self::sync`] for
+    /// system paths.
     fn sync_system_path(db: &mut dyn Db, path: &SystemPath, file: Option<File>) {
         let Some(file) = file.or_else(|| db.files().try_system(db, path)) else {
             return;
         };
-        let metadata = db.system().path_metadata(path);
-        Self::sync_impl(db, metadata, file);
-    }
 
-    fn sync_system_virtual_path(db: &mut dyn Db, path: &SystemVirtualPath, file: File) {
-        let metadata = db.system().virtual_path_metadata(path);
-        Self::sync_impl(db, metadata, file);
-    }
-
-    /// Private method providing the implementation for [`Self::sync_system_path`] and
-    /// [`Self::sync_system_virtual_path`].
-    fn sync_impl(db: &mut dyn Db, metadata: crate::system::Result<Metadata>, file: File) {
-        let (status, revision, permission) = match metadata {
+        let (status, revision, permission) = match db.system().path_metadata(path) {
             Ok(metadata) if metadata.file_type().is_file() => (
                 FileStatus::Exists,
                 metadata.revision(),
@@ -422,6 +426,35 @@ impl File {
     }
 }
 
+/// A virtual file that doesn't exist on the file system.
+///
+/// This is a wrapper around a [`File`] that provides additional methods to interact with a virtual
+/// file.
+#[derive(Copy, Clone)]
+pub struct VirtualFile(File);
+
+impl VirtualFile {
+    /// Returns the underlying [`File`].
+    pub fn file(&self) -> File {
+        self.0
+    }
+
+    /// Increments the revision of the underlying [`File`].
+    fn sync(&self, db: &mut dyn Db) {
+        let file = self.0;
+        tracing::debug!("Updating the revision of '{}'", file.path(db));
+        let current_revision = file.revision(db);
+        file.set_revision(db)
+            .to(FileRevision::new(current_revision.as_u128() + 1));
+    }
+
+    /// Closes the virtual file.
+    pub fn close(&self, db: &mut dyn Db) {
+        tracing::debug!("Closing virtual file '{}'", self.0.path(db));
+        self.0.set_status(db).to(FileStatus::NotFound);
+    }
+}
+
 // The types in here need to be public because they're salsa ingredients but we
 // don't want them to be publicly accessible. That's why we put them into a private module.
 mod private {

crates/ruff_db/src/parsed.rs

@@ -121,9 +121,9 @@ mod tests {
 
         db.write_virtual_file(path, "x = 10");
 
-        let file = db.files().add_virtual_file(&db, path).unwrap();
+        let virtual_file = db.files().virtual_file(&db, path);
 
-        let parsed = parsed_module(&db, file);
+        let parsed = parsed_module(&db, virtual_file.file());
 
         assert!(parsed.is_valid());
 
@@ -137,9 +137,9 @@ mod tests {
 
         db.write_virtual_file(path, "%timeit a = b");
 
-        let file = db.files().add_virtual_file(&db, path).unwrap();
+        let virtual_file = db.files().virtual_file(&db, path);
 
-        let parsed = parsed_module(&db, file);
+        let parsed = parsed_module(&db, virtual_file.file());
 
         assert!(parsed.is_valid());
 

crates/ruff_db/src/system.rs

@@ -63,9 +63,6 @@ pub trait System: Debug {
     /// representation fall-back to deserializing the notebook from a string.
     fn read_to_notebook(&self, path: &SystemPath) -> std::result::Result<Notebook, NotebookError>;
 
-    /// Reads the metadata of the virtual file at `path`.
-    fn virtual_path_metadata(&self, path: &SystemVirtualPath) -> Result<Metadata>;
-
     /// Reads the content of the virtual file at `path` into a [`String`].
     fn read_virtual_path_to_string(&self, path: &SystemVirtualPath) -> Result<String>;
 

crates/ruff_db/src/system/memory_fs.rs

@@ -136,22 +136,6 @@ impl MemoryFileSystem {
         ruff_notebook::Notebook::from_source_code(&content)
     }
 
-    pub(crate) fn virtual_path_metadata(
-        &self,
-        path: impl AsRef<SystemVirtualPath>,
-    ) -> Result<Metadata> {
-        let virtual_files = self.inner.virtual_files.read().unwrap();
-        let file = virtual_files
-            .get(&path.as_ref().to_path_buf())
-            .ok_or_else(not_found)?;
-
-        Ok(Metadata {
-            revision: file.last_modified.into(),
-            permissions: Some(MemoryFileSystem::PERMISSION),
-            file_type: FileType::File,
-        })
-    }
-
     pub(crate) fn read_virtual_path_to_string(
         &self,
         path: impl AsRef<SystemVirtualPath>,

crates/ruff_db/src/system/os.rs

@@ -77,10 +77,6 @@ impl System for OsSystem {
         Notebook::from_path(path.as_std_path())
     }
 
-    fn virtual_path_metadata(&self, _path: &SystemVirtualPath) -> Result<Metadata> {
-        Err(not_found())
-    }
-
     fn read_virtual_path_to_string(&self, _path: &SystemVirtualPath) -> Result<String> {
         Err(not_found())
     }

crates/ruff_db/src/system/test.rs

@@ -69,13 +69,6 @@ impl System for TestSystem {
         }
     }
 
-    fn virtual_path_metadata(&self, path: &SystemVirtualPath) -> Result<Metadata> {
-        match &self.inner {
-            TestSystemInner::Stub(fs) => fs.virtual_path_metadata(path),
-            TestSystemInner::System(system) => system.virtual_path_metadata(path),
-        }
-    }
-
     fn read_virtual_path_to_string(&self, path: &SystemVirtualPath) -> Result<String> {
         match &self.inner {
             TestSystemInner::Stub(fs) => fs.read_virtual_path_to_string(path),

crates/ruff_linter/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "ruff_linter"
-version = "0.6.1"
+version = "0.6.2"
 publish = false
 authors = { workspace = true }
 edition = { workspace = true }

crates/ruff_linter/resources/test/fixtures/flake8_simplify/SIM115.py

@@ -47,7 +47,6 @@ with contextlib.ExitStack() as exit_stack:
 open("filename", "w").close()
 pathlib.Path("filename").open("w").close()
 
-
 # OK (custom context manager)
 class MyFile:
     def __init__(self, filename: str):
@@ -58,3 +57,189 @@ class MyFile:
 
     def __exit__(self, exc_type, exc_val, exc_tb):
         self.file.close()
+
+
+import tempfile
+import tarfile
+from tarfile import TarFile
+import zipfile
+import io
+import codecs
+import bz2
+import gzip
+import dbm
+import dbm.gnu
+import dbm.ndbm
+import dbm.dumb
+import lzma
+import shelve
+import tokenize
+import wave
+import fileinput
+
+f = tempfile.NamedTemporaryFile()
+f = tempfile.TemporaryFile()
+f = tempfile.SpooledTemporaryFile()
+f = tarfile.open("foo.tar")
+f = TarFile("foo.tar").open()
+f = tarfile.TarFile("foo.tar").open()
+f = tarfile.TarFile().open()
+f = zipfile.ZipFile("foo.zip").open("foo.txt")
+f = io.open("foo.txt")
+f = io.open_code("foo.txt")
+f = codecs.open("foo.txt")
+f = bz2.open("foo.txt")
+f = gzip.open("foo.txt")
+f = dbm.open("foo.db")
+f = dbm.gnu.open("foo.db")
+f = dbm.ndbm.open("foo.db")
+f = dbm.dumb.open("foo.db")
+f = lzma.open("foo.xz")
+f = lzma.LZMAFile("foo.xz")
+f = shelve.open("foo.db")
+f = tokenize.open("foo.py")
+f = wave.open("foo.wav")
+f = tarfile.TarFile.taropen("foo.tar")
+f = fileinput.input("foo.txt")
+f = fileinput.FileInput("foo.txt")
+
+with contextlib.suppress(Exception):
+    # The following line is for example's sake.
+    # For some f's above, this would raise an error (since it'd be f.readline() etc.)
+    data = f.read()
+
+f.close()
+
+# OK
+with tempfile.TemporaryFile() as f:
+    data = f.read()
+
+# OK
+with tarfile.open("foo.tar") as f:
+    data = f.add("foo.txt")
+
+# OK
+with tarfile.TarFile("foo.tar") as f:
+    data = f.add("foo.txt")
+
+# OK
+with tarfile.TarFile("foo.tar").open() as f:
+    data = f.add("foo.txt")
+
+# OK
+with zipfile.ZipFile("foo.zip") as f:
+    data = f.read("foo.txt")
+
+# OK
+with zipfile.ZipFile("foo.zip").open("foo.txt") as f:
+    data = f.read()
+
+# OK
+with zipfile.ZipFile("foo.zip") as zf:
+    with zf.open("foo.txt") as f:
+        data = f.read()
+
+# OK
+with io.open("foo.txt") as f:
+    data = f.read()
+
+# OK
+with io.open_code("foo.txt") as f:
+    data = f.read()
+
+# OK
+with codecs.open("foo.txt") as f:
+    data = f.read()
+
+# OK
+with bz2.open("foo.txt") as f:
+    data = f.read()
+
+# OK
+with gzip.open("foo.txt") as f:
+    data = f.read()
+
+# OK
+with dbm.open("foo.db") as f:
+    data = f.get("foo")
+
+# OK
+with dbm.gnu.open("foo.db") as f:
+    data = f.get("foo")
+
+# OK
+with dbm.ndbm.open("foo.db") as f:
+    data = f.get("foo")
+
+# OK
+with dbm.dumb.open("foo.db") as f:
+    data = f.get("foo")
+
+# OK
+with lzma.open("foo.xz") as f:
+    data = f.read()
+
+# OK
+with lzma.LZMAFile("foo.xz") as f:
+    data = f.read()
+
+# OK
+with shelve.open("foo.db") as f:
+    data = f["foo"]
+
+# OK
+with tokenize.open("foo.py") as f:
+    data = f.read()
+
+# OK
+with wave.open("foo.wav") as f:
+    data = f.readframes(1024)
+
+# OK
+with tarfile.TarFile.taropen("foo.tar") as f:
+    data = f.add("foo.txt")
+
+# OK
+with fileinput.input("foo.txt") as f:
+    data = f.readline()
+
+# OK
+with fileinput.FileInput("foo.txt") as f:
+    data = f.readline()
+
+# OK (quick one-liner to clear file contents)
+tempfile.NamedTemporaryFile().close()
+tempfile.TemporaryFile().close()
+tempfile.SpooledTemporaryFile().close()
+tarfile.open("foo.tar").close()
+tarfile.TarFile("foo.tar").close()
+tarfile.TarFile("foo.tar").open().close()
+tarfile.TarFile.open("foo.tar").close()
+zipfile.ZipFile("foo.zip").close()
+zipfile.ZipFile("foo.zip").open("foo.txt").close()
+io.open("foo.txt").close()
+io.open_code("foo.txt").close()
+codecs.open("foo.txt").close()
+bz2.open("foo.txt").close()
+gzip.open("foo.txt").close()
+dbm.open("foo.db").close()
+dbm.gnu.open("foo.db").close()
+dbm.ndbm.open("foo.db").close()
+dbm.dumb.open("foo.db").close()
+lzma.open("foo.xz").close()
+lzma.LZMAFile("foo.xz").close()
+shelve.open("foo.db").close()
+tokenize.open("foo.py").close()
+wave.open("foo.wav").close()
+tarfile.TarFile.taropen("foo.tar").close()
+fileinput.input("foo.txt").close()
+fileinput.FileInput("foo.txt").close()
+
+def aliased():
+    from shelve import open as open_shelf
+    x = open_shelf("foo.dbm")
+    x.close()
+
+    from tarfile import TarFile as TF
+    f = TF("foo").open()
+    f.close()

crates/ruff_linter/resources/test/fixtures/ruff/RUF027_0.py

@@ -38,7 +38,7 @@ def tripled_quoted():
     c = a
     single_line = """ {a} """  # RUF027
     # RUF027
-    multi_line = a = """b { # comment
+    multi_line = """b { # comment
     c}  d
     """
 
@@ -72,3 +72,16 @@ def method_calls():
 def format_specifiers():
     a = 4
     b = "{a:b} {a:^5}"
+
+
+def implicitly_concatenated_fstring(x: int):
+    y = f"{x}" "{x}"  # RUF027
+    z = "{x}" f"{x}"  # RUF027
+
+
+def method_call_passed_to_logging(x: int):
+    # We special-case bindings that are passed to logging calls later on,
+    # but not bindings that involve arbitrary call expressions!
+    y = foo("{x}")  # RUF027
+    import logging
+    logging.log(0, y)

crates/ruff_linter/resources/test/fixtures/ruff/RUF027_1.py

@@ -68,3 +68,33 @@ def negative_cases():
     @app.get("/items/{item_id}")
     async def read_item(item_id):
         return {"item_id": item_id}
+
+
+# we shouldn't flag either of these, because the bindings are used elsewhere
+# in contexts that make it clear that they're not meant to be f-strings
+GLOBAL_STRING = "foo {bar}"
+LOGGING_TEMPLATE = "{foo} bar"
+
+def uses_global_strings():
+    print(GLOBAL_STRING.format(bar="whatever"))
+
+    import logging
+    logging.error(LOGGING_TEMPLATE, 42)
+
+
+def binding_defined_after_string():
+    if bool():
+        x = "{foo}"
+    else:
+        x = "{foo}"
+    foo = 42
+    print(x.format(foo=foo))
+
+
+def variable_immediately_used_after_more_complex_binding(arg: str, as_pypath: bool):
+    msg = (
+        "module or package not found: {arg} (missing __init__.py?)"
+        if as_pypath
+        else "file or directory not found: {arg}"
+    )
+    raise TypeError(msg.format(arg=arg))

crates/ruff_linter/src/checkers/ast/analyze/bindings.rs

@@ -15,6 +15,7 @@ pub(crate) fn bindings(checker: &mut Checker) {
         Rule::UnconventionalImportAlias,
         Rule::UnsortedDunderSlots,
         Rule::UnusedVariable,
+        Rule::MissingFStringSyntax,
     ]) {
         return;
     }
@@ -77,5 +78,11 @@ pub(crate) fn bindings(checker: &mut Checker) {
                 checker.diagnostics.push(diagnostic);
             }
         }
+        if checker.enabled(Rule::MissingFStringSyntax) {
+            if let Some(diagnostic) = ruff::rules::missing_fstring_syntax_binding(checker, binding)
+            {
+                checker.diagnostics.push(diagnostic);
+            }
+        }
     }
 }

crates/ruff_linter/src/checkers/ast/analyze/expression.rs

@@ -883,7 +883,7 @@ pub(crate) fn expression(expr: &Expr, checker: &mut Checker) {
                 flake8_simplify::rules::use_capital_environment_variables(checker, expr);
             }
             if checker.enabled(Rule::OpenFileWithContextHandler) {
-                flake8_simplify::rules::open_file_with_context_handler(checker, func);
+                flake8_simplify::rules::open_file_with_context_handler(checker, call);
             }
             if checker.enabled(Rule::DictGetWithNoneDefault) {
                 flake8_simplify::rules::dict_get_with_none_default(checker, expr);
@@ -1080,7 +1080,7 @@ pub(crate) fn expression(expr: &Expr, checker: &mut Checker) {
             }
             if checker.enabled(Rule::MissingFStringSyntax) {
                 for string_literal in value.literals() {
-                    ruff::rules::missing_fstring_syntax(checker, string_literal);
+                    ruff::rules::missing_fstring_syntax_expr(checker, string_literal);
                 }
             }
         }
@@ -1376,7 +1376,7 @@ pub(crate) fn expression(expr: &Expr, checker: &mut Checker) {
             }
             if checker.enabled(Rule::MissingFStringSyntax) {
                 for string_literal in value.as_slice() {
-                    ruff::rules::missing_fstring_syntax(checker, string_literal);
+                    ruff::rules::missing_fstring_syntax_expr(checker, string_literal);
                 }
             }
         }

crates/ruff_linter/src/rules/flake8_async/mod.rs

@@ -11,6 +11,7 @@ mod tests {
 
     use crate::assert_messages;
     use crate::registry::Rule;
+    use crate::settings::types::PythonVersion;
     use crate::settings::LinterSettings;
     use crate::test::test_path;
 
@@ -36,4 +37,18 @@ mod tests {
         assert_messages!(snapshot, diagnostics);
         Ok(())
     }
+
+    #[test_case(Path::new("ASYNC109_0.py"); "asyncio")]
+    #[test_case(Path::new("ASYNC109_1.py"); "trio")]
+    fn async109_python_310_or_older(path: &Path) -> Result<()> {
+        let diagnostics = test_path(
+            Path::new("flake8_async").join(path),
+            &LinterSettings {
+                target_version: PythonVersion::Py310,
+                ..LinterSettings::for_rule(Rule::AsyncFunctionWithTimeout)
+            },
+        )?;
+        assert_messages!(path.file_name().unwrap().to_str().unwrap(), diagnostics);
+        Ok(())
+    }
 }

crates/ruff_linter/src/rules/flake8_async/rules/async_function_with_timeout.rs

@@ -6,6 +6,7 @@ use ruff_text_size::Ranged;
 
 use crate::checkers::ast::Checker;
 use crate::rules::flake8_async::helpers::AsyncModule;
+use crate::settings::types::PythonVersion;
 
 /// ## What it does
 /// Checks for `async` functions with a `timeout` argument.
@@ -86,6 +87,11 @@ pub(crate) fn async_function_with_timeout(
         AsyncModule::AsyncIo
     };
 
+    // asyncio.timeout feature was first introduced in Python 3.11
+    if module == AsyncModule::AsyncIo && checker.settings.target_version < PythonVersion::Py311 {
+        return;
+    }
+
     checker.diagnostics.push(Diagnostic::new(
         AsyncFunctionWithTimeout { module },
         timeout.range(),

crates/ruff_linter/src/rules/flake8_async/snapshots/ruff_linter__rules__flake8_async__tests__ASYNC109_0.py.snap

@@ -0,0 +1,18 @@
+---
+source: crates/ruff_linter/src/rules/flake8_async/mod.rs
+---
+ASYNC109_0.py:8:16: ASYNC109 Async function definition with a `timeout` parameter
+  |
+8 | async def func(timeout):
+  |                ^^^^^^^ ASYNC109
+9 |     ...
+  |
+  = help: Use `trio.fail_after` instead
+
+ASYNC109_0.py:12:16: ASYNC109 Async function definition with a `timeout` parameter
+   |
+12 | async def func(timeout=10):
+   |                ^^^^^^^^^^ ASYNC109
+13 |     ...
+   |
+   = help: Use `trio.fail_after` instead

crates/ruff_linter/src/rules/flake8_async/snapshots/ruff_linter__rules__flake8_async__tests__ASYNC109_1.py.snap

@@ -0,0 +1,4 @@
+---
+source: crates/ruff_linter/src/rules/flake8_async/mod.rs
+---
+

crates/ruff_linter/src/rules/flake8_simplify/mod.rs

@@ -58,6 +58,7 @@ mod tests {
     }
 
     #[test_case(Rule::IfElseBlockInsteadOfIfExp, Path::new("SIM108.py"))]
+    #[test_case(Rule::OpenFileWithContextHandler, Path::new("SIM115.py"))]
     fn preview_rules(rule_code: Rule, path: &Path) -> Result<()> {
         let snapshot = format!(
             "preview__{}_{}",

crates/ruff_linter/src/rules/flake8_simplify/rules/open_file_with_context_handler.rs

@@ -8,14 +8,20 @@ use ruff_text_size::Ranged;
 use crate::checkers::ast::Checker;
 
 /// ## What it does
-/// Checks for uses of the builtin `open()` function without an associated context
-/// manager.
+/// Checks for cases where files are opened (e.g., using the builtin `open()` function)
+/// without using a context manager.
 ///
 /// ## Why is this bad?
 /// If a file is opened without a context manager, it is not guaranteed that
 /// the file will be closed (e.g., if an exception is raised), which can cause
 /// resource leaks.
 ///
+/// ## Preview-mode behavior
+/// If [preview] mode is enabled, this rule will detect a wide array of IO calls where
+/// context managers could be used, such as `tempfile.TemporaryFile()` or
+/// `tarfile.TarFile(...).gzopen()`. If preview mode is not enabled, only `open()`,
+/// `builtins.open()` and `pathlib.Path(...).open()` are detected.
+///
 /// ## Example
 /// ```python
 /// file = open("foo.txt")
@@ -37,7 +43,7 @@ pub struct OpenFileWithContextHandler;
 impl Violation for OpenFileWithContextHandler {
     #[derive_message_formats]
     fn message(&self) -> String {
-        format!("Use context handler for opening files")
+        format!("Use a context manager for opening files")
     }
 }
 
@@ -113,14 +119,14 @@ fn match_exit_stack(semantic: &SemanticModel) -> bool {
 }
 
 /// Return `true` if `func` is the builtin `open` or `pathlib.Path(...).open`.
-fn is_open(semantic: &SemanticModel, func: &Expr) -> bool {
+fn is_open(semantic: &SemanticModel, call: &ast::ExprCall) -> bool {
     // Ex) `open(...)`
-    if semantic.match_builtin_expr(func, "open") {
+    if semantic.match_builtin_expr(&call.func, "open") {
         return true;
     }
 
     // Ex) `pathlib.Path(...).open()`
-    let Expr::Attribute(ast::ExprAttribute { attr, value, .. }) = func else {
+    let Expr::Attribute(ast::ExprAttribute { attr, value, .. }) = &*call.func else {
         return false;
     };
 
@@ -140,6 +146,63 @@ fn is_open(semantic: &SemanticModel, func: &Expr) -> bool {
         .is_some_and(|qualified_name| matches!(qualified_name.segments(), ["pathlib", "Path"]))
 }
 
+/// Return `true` if the expression is an `open` call or temporary file constructor.
+fn is_open_preview(semantic: &SemanticModel, call: &ast::ExprCall) -> bool {
+    let func = &*call.func;
+
+    // Ex) `open(...)`
+    if let Some(qualified_name) = semantic.resolve_qualified_name(func) {
+        return matches!(
+            qualified_name.segments(),
+            [
+                "" | "builtins"
+                    | "bz2"
+                    | "codecs"
+                    | "dbm"
+                    | "gzip"
+                    | "tarfile"
+                    | "shelve"
+                    | "tokenize"
+                    | "wave",
+                "open"
+            ] | ["dbm", "gnu" | "ndbm" | "dumb", "open"]
+                | ["fileinput", "FileInput" | "input"]
+                | ["io", "open" | "open_code"]
+                | ["lzma", "LZMAFile" | "open"]
+                | ["tarfile", "TarFile", "taropen"]
+                | [
+                    "tempfile",
+                    "TemporaryFile" | "NamedTemporaryFile" | "SpooledTemporaryFile"
+                ]
+        );
+    }
+
+    // Ex) `pathlib.Path(...).open()`
+    let Expr::Attribute(ast::ExprAttribute { attr, value, .. }) = func else {
+        return false;
+    };
+
+    let Expr::Call(ast::ExprCall { func, .. }) = &**value else {
+        return false;
+    };
+
+    // E.g. for `pathlib.Path(...).open()`, `qualified_name_of_instance.segments() == ["pathlib", "Path"]`
+    let Some(qualified_name_of_instance) = semantic.resolve_qualified_name(func) else {
+        return false;
+    };
+
+    matches!(
+        (qualified_name_of_instance.segments(), &**attr),
+        (
+            ["pathlib", "Path"] | ["zipfile", "ZipFile"] | ["lzma", "LZMAFile"],
+            "open"
+        ) | (
+            ["tarfile", "TarFile"],
+            "open" | "taropen" | "gzopen" | "bz2open" | "xzopen"
+        )
+    )
+}
+
 /// Return `true` if the current expression is followed by a `close` call.
 fn is_closed(semantic: &SemanticModel) -> bool {
     let Some(expr) = semantic.current_expression_grandparent() else {
@@ -165,11 +228,17 @@ fn is_closed(semantic: &SemanticModel) -> bool {
 }
 
 /// SIM115
-pub(crate) fn open_file_with_context_handler(checker: &mut Checker, func: &Expr) {
+pub(crate) fn open_file_with_context_handler(checker: &mut Checker, call: &ast::ExprCall) {
     let semantic = checker.semantic();
 
-    if !is_open(semantic, func) {
-        return;
+    if checker.settings.preview.is_disabled() {
+        if !is_open(semantic, call) {
+            return;
+        }
+    } else {
+        if !is_open_preview(semantic, call) {
+            return;
+        }
     }
 
     // Ex) `open("foo.txt").close()`
@@ -201,7 +270,8 @@ pub(crate) fn open_file_with_context_handler(checker: &mut Checker, func: &Expr)
         }
     }
 
-    checker
-        .diagnostics
-        .push(Diagnostic::new(OpenFileWithContextHandler, func.range()));
+    checker.diagnostics.push(Diagnostic::new(
+        OpenFileWithContextHandler,
+        call.func.range(),
+    ));
 }

crates/ruff_linter/src/rules/flake8_simplify/snapshots/ruff_linter__rules__flake8_simplify__tests__SIM115_SIM115.py.snap

@@ -1,7 +1,7 @@
 ---
 source: crates/ruff_linter/src/rules/flake8_simplify/mod.rs
 ---
-SIM115.py:8:5: SIM115 Use context handler for opening files
+SIM115.py:8:5: SIM115 Use a context manager for opening files
    |
  7 | # SIM115
  8 | f = open("foo.txt")
@@ -10,7 +10,7 @@ SIM115.py:8:5: SIM115 Use context handler for opening files
 10 | f = pathlib.Path("foo.txt").open()
    |
 
-SIM115.py:9:5: SIM115 Use context handler for opening files
+SIM115.py:9:5: SIM115 Use a context manager for opening files
    |
  7 | # SIM115
  8 | f = open("foo.txt")
@@ -20,7 +20,7 @@ SIM115.py:9:5: SIM115 Use context handler for opening files
 11 | f = pl.Path("foo.txt").open()
    |
 
-SIM115.py:10:5: SIM115 Use context handler for opening files
+SIM115.py:10:5: SIM115 Use a context manager for opening files
    |
  8 | f = open("foo.txt")
  9 | f = Path("foo.txt").open()
@@ -30,7 +30,7 @@ SIM115.py:10:5: SIM115 Use context handler for opening files
 12 | f = P("foo.txt").open()
    |
 
-SIM115.py:11:5: SIM115 Use context handler for opening files
+SIM115.py:11:5: SIM115 Use a context manager for opening files
    |
  9 | f = Path("foo.txt").open()
 10 | f = pathlib.Path("foo.txt").open()
@@ -40,7 +40,7 @@ SIM115.py:11:5: SIM115 Use context handler for opening files
 13 | data = f.read()
    |
 
-SIM115.py:12:5: SIM115 Use context handler for opening files
+SIM115.py:12:5: SIM115 Use a context manager for opening files
    |
 10 | f = pathlib.Path("foo.txt").open()
 11 | f = pl.Path("foo.txt").open()
@@ -50,7 +50,7 @@ SIM115.py:12:5: SIM115 Use context handler for opening files
 14 | f.close()
    |
 
-SIM115.py:39:9: SIM115 Use context handler for opening files
+SIM115.py:39:9: SIM115 Use a context manager for opening files
    |
 37 | # SIM115
 38 | with contextlib.ExitStack():
@@ -59,5 +59,3 @@ SIM115.py:39:9: SIM115 Use context handler for opening files
 40 | 
 41 | # OK
    |
-
-

crates/ruff_linter/src/rules/flake8_simplify/snapshots/ruff_linter__rules__flake8_simplify__tests__preview__SIM115_SIM115.py.snap

@@ -0,0 +1,326 @@
+---
+source: crates/ruff_linter/src/rules/flake8_simplify/mod.rs
+---
+SIM115.py:8:5: SIM115 Use a context manager for opening files
+   |
+ 7 | # SIM115
+ 8 | f = open("foo.txt")
+   |     ^^^^ SIM115
+ 9 | f = Path("foo.txt").open()
+10 | f = pathlib.Path("foo.txt").open()
+   |
+
+SIM115.py:9:5: SIM115 Use a context manager for opening files
+   |
+ 7 | # SIM115
+ 8 | f = open("foo.txt")
+ 9 | f = Path("foo.txt").open()
+   |     ^^^^^^^^^^^^^^^^^^^^ SIM115
+10 | f = pathlib.Path("foo.txt").open()
+11 | f = pl.Path("foo.txt").open()
+   |
+
+SIM115.py:10:5: SIM115 Use a context manager for opening files
+   |
+ 8 | f = open("foo.txt")
+ 9 | f = Path("foo.txt").open()
+10 | f = pathlib.Path("foo.txt").open()
+   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ SIM115
+11 | f = pl.Path("foo.txt").open()
+12 | f = P("foo.txt").open()
+   |
+
+SIM115.py:11:5: SIM115 Use a context manager for opening files
+   |
+ 9 | f = Path("foo.txt").open()
+10 | f = pathlib.Path("foo.txt").open()
+11 | f = pl.Path("foo.txt").open()
+   |     ^^^^^^^^^^^^^^^^^^^^^^^ SIM115
+12 | f = P("foo.txt").open()
+13 | data = f.read()
+   |
+
+SIM115.py:12:5: SIM115 Use a context manager for opening files
+   |
+10 | f = pathlib.Path("foo.txt").open()
+11 | f = pl.Path("foo.txt").open()
+12 | f = P("foo.txt").open()
+   |     ^^^^^^^^^^^^^^^^^ SIM115
+13 | data = f.read()
+14 | f.close()
+   |
+
+SIM115.py:39:9: SIM115 Use a context manager for opening files
+   |
+37 | # SIM115
+38 | with contextlib.ExitStack():
+39 |     f = open("filename")
+   |         ^^^^ SIM115
+40 | 
+41 | # OK
+   |
+
+SIM115.py:80:5: SIM115 Use a context manager for opening files
+   |
+78 | import fileinput
+79 | 
+80 | f = tempfile.NamedTemporaryFile()
+   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^ SIM115
+81 | f = tempfile.TemporaryFile()
+82 | f = tempfile.SpooledTemporaryFile()
+   |
+
+SIM115.py:81:5: SIM115 Use a context manager for opening files
+   |
+80 | f = tempfile.NamedTemporaryFile()
+81 | f = tempfile.TemporaryFile()
+   |     ^^^^^^^^^^^^^^^^^^^^^^ SIM115
+82 | f = tempfile.SpooledTemporaryFile()
+83 | f = tarfile.open("foo.tar")
+   |
+
+SIM115.py:82:5: SIM115 Use a context manager for opening files
+   |
+80 | f = tempfile.NamedTemporaryFile()
+81 | f = tempfile.TemporaryFile()
+82 | f = tempfile.SpooledTemporaryFile()
+   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ SIM115
+83 | f = tarfile.open("foo.tar")
+84 | f = TarFile("foo.tar").open()
+   |
+
+SIM115.py:83:5: SIM115 Use a context manager for opening files
+   |
+81 | f = tempfile.TemporaryFile()
+82 | f = tempfile.SpooledTemporaryFile()
+83 | f = tarfile.open("foo.tar")
+   |     ^^^^^^^^^^^^ SIM115
+84 | f = TarFile("foo.tar").open()
+85 | f = tarfile.TarFile("foo.tar").open()
+   |
+
+SIM115.py:84:5: SIM115 Use a context manager for opening files
+   |
+82 | f = tempfile.SpooledTemporaryFile()
+83 | f = tarfile.open("foo.tar")
+84 | f = TarFile("foo.tar").open()
+   |     ^^^^^^^^^^^^^^^^^^^^^^^ SIM115
+85 | f = tarfile.TarFile("foo.tar").open()
+86 | f = tarfile.TarFile().open()
+   |
+
+SIM115.py:85:5: SIM115 Use a context manager for opening files
+   |
+83 | f = tarfile.open("foo.tar")
+84 | f = TarFile("foo.tar").open()
+85 | f = tarfile.TarFile("foo.tar").open()
+   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ SIM115
+86 | f = tarfile.TarFile().open()
+87 | f = zipfile.ZipFile("foo.zip").open("foo.txt")
+   |
+
+SIM115.py:86:5: SIM115 Use a context manager for opening files
+   |
+84 | f = TarFile("foo.tar").open()
+85 | f = tarfile.TarFile("foo.tar").open()
+86 | f = tarfile.TarFile().open()
+   |     ^^^^^^^^^^^^^^^^^^^^^^ SIM115
+87 | f = zipfile.ZipFile("foo.zip").open("foo.txt")
+88 | f = io.open("foo.txt")
+   |
+
+SIM115.py:87:5: SIM115 Use a context manager for opening files
+   |
+85 | f = tarfile.TarFile("foo.tar").open()
+86 | f = tarfile.TarFile().open()
+87 | f = zipfile.ZipFile("foo.zip").open("foo.txt")
+   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ SIM115
+88 | f = io.open("foo.txt")
+89 | f = io.open_code("foo.txt")
+   |
+
+SIM115.py:88:5: SIM115 Use a context manager for opening files
+   |
+86 | f = tarfile.TarFile().open()
+87 | f = zipfile.ZipFile("foo.zip").open("foo.txt")
+88 | f = io.open("foo.txt")
+   |     ^^^^^^^ SIM115
+89 | f = io.open_code("foo.txt")
+90 | f = codecs.open("foo.txt")
+   |
+
+SIM115.py:89:5: SIM115 Use a context manager for opening files
+   |
+87 | f = zipfile.ZipFile("foo.zip").open("foo.txt")
+88 | f = io.open("foo.txt")
+89 | f = io.open_code("foo.txt")
+   |     ^^^^^^^^^^^^ SIM115
+90 | f = codecs.open("foo.txt")
+91 | f = bz2.open("foo.txt")
+   |
+
+SIM115.py:90:5: SIM115 Use a context manager for opening files
+   |
+88 | f = io.open("foo.txt")
+89 | f = io.open_code("foo.txt")
+90 | f = codecs.open("foo.txt")
+   |     ^^^^^^^^^^^ SIM115
+91 | f = bz2.open("foo.txt")
+92 | f = gzip.open("foo.txt")
+   |
+
+SIM115.py:91:5: SIM115 Use a context manager for opening files
+   |
+89 | f = io.open_code("foo.txt")
+90 | f = codecs.open("foo.txt")
+91 | f = bz2.open("foo.txt")
+   |     ^^^^^^^^ SIM115
+92 | f = gzip.open("foo.txt")
+93 | f = dbm.open("foo.db")
+   |
+
+SIM115.py:92:5: SIM115 Use a context manager for opening files
+   |
+90 | f = codecs.open("foo.txt")
+91 | f = bz2.open("foo.txt")
+92 | f = gzip.open("foo.txt")
+   |     ^^^^^^^^^ SIM115
+93 | f = dbm.open("foo.db")
+94 | f = dbm.gnu.open("foo.db")
+   |
+
+SIM115.py:93:5: SIM115 Use a context manager for opening files
+   |
+91 | f = bz2.open("foo.txt")
+92 | f = gzip.open("foo.txt")
+93 | f = dbm.open("foo.db")
+   |     ^^^^^^^^ SIM115
+94 | f = dbm.gnu.open("foo.db")
+95 | f = dbm.ndbm.open("foo.db")
+   |
+
+SIM115.py:94:5: SIM115 Use a context manager for opening files
+   |
+92 | f = gzip.open("foo.txt")
+93 | f = dbm.open("foo.db")
+94 | f = dbm.gnu.open("foo.db")
+   |     ^^^^^^^^^^^^ SIM115
+95 | f = dbm.ndbm.open("foo.db")
+96 | f = dbm.dumb.open("foo.db")
+   |
+
+SIM115.py:95:5: SIM115 Use a context manager for opening files
+   |
+93 | f = dbm.open("foo.db")
+94 | f = dbm.gnu.open("foo.db")
+95 | f = dbm.ndbm.open("foo.db")
+   |     ^^^^^^^^^^^^^ SIM115
+96 | f = dbm.dumb.open("foo.db")
+97 | f = lzma.open("foo.xz")
+   |
+
+SIM115.py:96:5: SIM115 Use a context manager for opening files
+   |
+94 | f = dbm.gnu.open("foo.db")
+95 | f = dbm.ndbm.open("foo.db")
+96 | f = dbm.dumb.open("foo.db")
+   |     ^^^^^^^^^^^^^ SIM115
+97 | f = lzma.open("foo.xz")
+98 | f = lzma.LZMAFile("foo.xz")
+   |
+
+SIM115.py:97:5: SIM115 Use a context manager for opening files
+   |
+95 | f = dbm.ndbm.open("foo.db")
+96 | f = dbm.dumb.open("foo.db")
+97 | f = lzma.open("foo.xz")
+   |     ^^^^^^^^^ SIM115
+98 | f = lzma.LZMAFile("foo.xz")
+99 | f = shelve.open("foo.db")
+   |
+
+SIM115.py:98:5: SIM115 Use a context manager for opening files
+    |
+ 96 | f = dbm.dumb.open("foo.db")
+ 97 | f = lzma.open("foo.xz")
+ 98 | f = lzma.LZMAFile("foo.xz")
+    |     ^^^^^^^^^^^^^ SIM115
+ 99 | f = shelve.open("foo.db")
+100 | f = tokenize.open("foo.py")
+    |
+
+SIM115.py:99:5: SIM115 Use a context manager for opening files
+    |
+ 97 | f = lzma.open("foo.xz")
+ 98 | f = lzma.LZMAFile("foo.xz")
+ 99 | f = shelve.open("foo.db")
+    |     ^^^^^^^^^^^ SIM115
+100 | f = tokenize.open("foo.py")
+101 | f = wave.open("foo.wav")
+    |
+
+SIM115.py:100:5: SIM115 Use a context manager for opening files
+    |
+ 98 | f = lzma.LZMAFile("foo.xz")
+ 99 | f = shelve.open("foo.db")
+100 | f = tokenize.open("foo.py")
+    |     ^^^^^^^^^^^^^ SIM115
+101 | f = wave.open("foo.wav")
+102 | f = tarfile.TarFile.taropen("foo.tar")
+    |
+
+SIM115.py:101:5: SIM115 Use a context manager for opening files
+    |
+ 99 | f = shelve.open("foo.db")
+100 | f = tokenize.open("foo.py")
+101 | f = wave.open("foo.wav")
+    |     ^^^^^^^^^ SIM115
+102 | f = tarfile.TarFile.taropen("foo.tar")
+103 | f = fileinput.input("foo.txt")
+    |
+
+SIM115.py:102:5: SIM115 Use a context manager for opening files
+    |
+100 | f = tokenize.open("foo.py")
+101 | f = wave.open("foo.wav")
+102 | f = tarfile.TarFile.taropen("foo.tar")
+    |     ^^^^^^^^^^^^^^^^^^^^^^^ SIM115
+103 | f = fileinput.input("foo.txt")
+104 | f = fileinput.FileInput("foo.txt")
+    |
+
+SIM115.py:103:5: SIM115 Use a context manager for opening files
+    |
+101 | f = wave.open("foo.wav")
+102 | f = tarfile.TarFile.taropen("foo.tar")
+103 | f = fileinput.input("foo.txt")
+    |     ^^^^^^^^^^^^^^^ SIM115
+104 | f = fileinput.FileInput("foo.txt")
+    |
+
+SIM115.py:104:5: SIM115 Use a context manager for opening files
+    |
+102 | f = tarfile.TarFile.taropen("foo.tar")
+103 | f = fileinput.input("foo.txt")
+104 | f = fileinput.FileInput("foo.txt")
+    |     ^^^^^^^^^^^^^^^^^^^ SIM115
+105 | 
+106 | with contextlib.suppress(Exception):
+    |
+
+SIM115.py:240:9: SIM115 Use a context manager for opening files
+    |
+238 | def aliased():
+239 |     from shelve import open as open_shelf
+240 |     x = open_shelf("foo.dbm")
+    |         ^^^^^^^^^^ SIM115
+241 |     x.close()
+    |
+
+SIM115.py:244:9: SIM115 Use a context manager for opening files
+    |
+243 |     from tarfile import TarFile as TF
+244 |     f = TF("foo").open()
+    |         ^^^^^^^^^^^^^^ SIM115
+245 |     f.close()
+    |

crates/ruff_linter/src/rules/ruff/rules/missing_fstring_syntax.rs

@@ -4,7 +4,7 @@ use ruff_python_ast as ast;
 use ruff_python_literal::format::FormatSpec;
 use ruff_python_parser::parse_expression;
 use ruff_python_semantic::analyze::logging::is_logger_candidate;
-use ruff_python_semantic::{Modules, SemanticModel};
+use ruff_python_semantic::{Binding, Modules, ScopeId, SemanticModel};
 use ruff_source_file::Locator;
 use ruff_text_size::{Ranged, TextRange};
 
@@ -36,6 +36,8 @@ use crate::rules::fastapi::rules::is_fastapi_route_call;
 /// 6. Any format specifiers in the potential f-string are invalid.
 /// 7. The string is part of a function call that is known to expect a template string rather than an
 ///    evaluated f-string: for example, a [`logging`] call, a [`gettext`] call, or a [`fastAPI` path].
+/// 8. The string is assigned to a symbol where any of the *references* to that symbol are part of a
+///    function call that is known to expect a template string rather than an evaluated f-string.
 ///
 /// ## Example
 ///
@@ -61,16 +63,27 @@ pub struct MissingFStringSyntax;
 impl AlwaysFixableViolation for MissingFStringSyntax {
     #[derive_message_formats]
     fn message(&self) -> String {
-        format!(r#"Possible f-string without an `f` prefix"#)
+        format!("Possible f-string without an `f` prefix")
     }
 
     fn fix_title(&self) -> String {
-        "Add `f` prefix".into()
+        "Add `f` prefix".to_string()
     }
 }
 
 /// RUF027
-pub(crate) fn missing_fstring_syntax(checker: &mut Checker, literal: &ast::StringLiteral) {
+///
+/// Analyze a [`ast::StringLiteral`] node to see if it looks like it could be a string that was
+/// meant to be an f-string, but is missing an `f` prefix. If so, emit a diagnostic.
+///
+/// This routine skips any string literals that are part of "simple assignments", e.g. `x = "foo"`
+/// or `x: str = "foo"`. These are checked by the [`missing_fstring_syntax_binding`] function
+/// below, which is part of the same check.
+pub(crate) fn missing_fstring_syntax_expr(checker: &mut Checker, literal: &ast::StringLiteral) {
+    if !has_brackets(&literal.value) {
+        return;
+    }
+
     let semantic = checker.semantic();
 
     // we want to avoid statement expressions that are just a string literal.
@@ -82,35 +95,168 @@ pub(crate) fn missing_fstring_syntax(checker: &mut Checker, literal: &ast::Strin
         }
     }
 
+    // Simple assignments will be dealt with by `missing_fstring_syntax_binding`
+    if is_simple_assignment_to_literal(literal, semantic) {
+        return;
+    }
+
     let logger_objects = &checker.settings.logger_objects;
     let fastapi_seen = semantic.seen_module(Modules::FASTAPI);
 
+    let mut arg_names = FxHashSet::default();
+
     // We also want to avoid:
     // - Expressions inside `gettext()` calls
     // - Expressions passed to logging calls (since the `logging` module evaluates them lazily:
     //   https://docs.python.org/3/howto/logging-cookbook.html#using-particular-formatting-styles-throughout-your-application)
     // - `fastAPI` paths: https://fastapi.tiangolo.com/tutorial/path-params/
     // - Expressions where a method is immediately called on the string literal
-    if semantic
+    for call_expr in semantic
         .current_expressions()
         .filter_map(ast::Expr::as_call_expr)
-        .any(|call_expr| {
-            is_method_call_on_literal(call_expr, literal)
-                || is_gettext(call_expr, semantic)
-                || is_logger_candidate(&call_expr.func, semantic, logger_objects)
-                || (fastapi_seen && is_fastapi_route_call(call_expr, semantic))
-        })
     {
-        return;
+        if is_method_call_on_literal(call_expr, literal) {
+            return;
+        }
+        if is_gettext(call_expr, semantic) {
+            return;
+        }
+        if is_logger_candidate(&call_expr.func, semantic, logger_objects) {
+            return;
+        }
+        if fastapi_seen && is_fastapi_route_call(call_expr, semantic) {
+            return;
+        }
+        let ast::Arguments { keywords, args, .. } = &call_expr.arguments;
+        for keyword in &**keywords {
+            if let Some(ident) = keyword.arg.as_ref() {
+                arg_names.insert(&ident.id);
+            }
+        }
+        for arg in &**args {
+            if let ast::Expr::Name(ast::ExprName { id, .. }) = arg {
+                arg_names.insert(id);
+            }
+        }
     }
 
-    if should_be_fstring(literal, checker.locator(), semantic) {
+    if should_be_fstring(
+        literal,
+        checker.locator(),
+        semantic,
+        semantic.scope_id,
+        &arg_names,
+    ) {
         let diagnostic = Diagnostic::new(MissingFStringSyntax, literal.range())
             .with_fix(fix_fstring_syntax(literal.range()));
         checker.diagnostics.push(diagnostic);
     }
 }
 
+/// RUF027
+///
+/// Analyze a [`Binding`] to see if the bound variable is a string literal
+/// that's part of a "simple assignment", e.g. `x = "foo"` or `x: str = "foo"`.
+/// If it is, check to see if the string looks like it's meant to be an f-string,
+/// but is missing an `f` prefix. False positives are minimized by analyzing the references
+/// to the binding, to see if the binding is ever used in a way that indicates that the
+/// string is clearly meant to be a string template rather than an f-string.
+pub(crate) fn missing_fstring_syntax_binding(
+    checker: &Checker,
+    binding: &Binding,
+) -> Option<Diagnostic> {
+    let semantic = checker.semantic();
+    let locator = checker.locator();
+
+    let stmt = binding.statement(semantic)?;
+    let string_literal = match stmt {
+        ast::Stmt::Assign(ast::StmtAssign { targets, value, .. }) => match targets.as_slice() {
+            [_] => value.as_string_literal_expr()?,
+            _ => return None,
+        },
+        ast::Stmt::AnnAssign(ast::StmtAnnAssign {
+            value: Some(value), ..
+        }) => value.as_string_literal_expr()?,
+        _ => return None,
+    };
+    let [string_literal] = string_literal.value.as_slice() else {
+        return None;
+    };
+    if !has_brackets(&string_literal.value) {
+        return None;
+    }
+
+    let logger_objects = &checker.settings.logger_objects;
+    let fastapi_seen = semantic.seen_module(Modules::FASTAPI);
+    let mut arg_names = FxHashSet::default();
+
+    for reference in binding.references().map(|id| semantic.reference(id)) {
+        let Some(expr_id) = reference.expression_id() else {
+            continue;
+        };
+        for call_expr in semantic
+            .expressions(expr_id)
+            .filter_map(ast::Expr::as_call_expr)
+        {
+            if is_method_call_on_name(call_expr, binding.name(locator)) {
+                return None;
+            }
+            if is_gettext(call_expr, semantic) {
+                return None;
+            }
+            if is_logger_candidate(&call_expr.func, semantic, logger_objects) {
+                return None;
+            }
+            if fastapi_seen && is_fastapi_route_call(call_expr, semantic) {
+                return None;
+            }
+            let ast::Arguments { keywords, args, .. } = &call_expr.arguments;
+            for keyword in &**keywords {
+                if let Some(ident) = keyword.arg.as_ref() {
+                    arg_names.insert(&ident.id);
+                }
+            }
+            for arg in &**args {
+                if let ast::Expr::Name(ast::ExprName { id, .. }) = arg {
+                    arg_names.insert(id);
+                }
+            }
+        }
+    }
+
+    should_be_fstring(string_literal, locator, semantic, binding.scope, &arg_names).then(|| {
+        Diagnostic::new(MissingFStringSyntax, string_literal.range())
+            .with_fix(fix_fstring_syntax(string_literal.range()))
+    })
+}
+
+/// Determine whether `stmt` represents a "simple assignment" to the string literal `literal`.
+///
+/// For our purposes here, a "simple assignment" is any assignment
+/// where `literal` appears on the right-hand side
+/// and `literal` is not part of an implicitly concatenated string.
+fn is_simple_assignment_to_literal(literal: &ast::StringLiteral, semantic: &SemanticModel) -> bool {
+    if semantic.current_expressions().any(|expr| match expr {
+        ast::Expr::Call(_) => true,
+        ast::Expr::StringLiteral(ast::ExprStringLiteral { value, .. }) => {
+            value.is_implicit_concatenated()
+        }
+        ast::Expr::FString(ast::ExprFString { value, .. }) => value.is_implicit_concatenated(),
+        _ => false,
+    }) {
+        return false;
+    }
+    match semantic.current_statement() {
+        ast::Stmt::Assign(ast::StmtAssign { value, .. }) => {
+            value.range().contains_range(literal.range())
+        }
+        ast::Stmt::AnnAssign(ast::StmtAnnAssign {
+            value: Some(value), ..
+        }) => value.range().contains_range(literal.range()),
+        _ => false,
+    }
+}
+
 /// Returns `true` if an expression appears to be a `gettext` call.
 ///
 /// We want to avoid statement expressions and assignments related to aliases
@@ -162,17 +308,28 @@ fn is_method_call_on_literal(call_expr: &ast::ExprCall, literal: &ast::StringLit
     value.as_slice().contains(literal)
 }
 
+/// Determine whether `call_expr` represents a method call on a symbol bound to the name `name`.
+///
+/// E.g., `foo.format(bar=3)`, where `name == "foo"`.
+fn is_method_call_on_name(call_expr: &ast::ExprCall, name: &str) -> bool {
+    let ast::Expr::Attribute(ast::ExprAttribute { value, .. }) = &*call_expr.func else {
+        return false;
+    };
+    let ast::Expr::Name(ast::ExprName { id, .. }) = &**value else {
+        return false;
+    };
+    id == name
+}
+
 /// Returns `true` if `literal` is likely an f-string with a missing `f` prefix.
 /// See [`MissingFStringSyntax`] for the validation criteria.
 fn should_be_fstring(
     literal: &ast::StringLiteral,
     locator: &Locator,
     semantic: &SemanticModel,
+    scope: ScopeId,
+    relevant_argument_names: &FxHashSet<&ast::name::Name>,
 ) -> bool {
-    if !has_brackets(&literal.value) {
-        return false;
-    }
-
     let fstring_expr = format!("f{}", locator.slice(literal));
     let Ok(parsed) = parse_expression(&fstring_expr) else {
         return false;
@@ -183,35 +340,21 @@ fn should_be_fstring(
         return false;
     };
 
-    let mut arg_names = FxHashSet::default();
-    for expr in semantic
-        .current_expressions()
-        .filter_map(ast::Expr::as_call_expr)
-    {
-        let ast::Arguments { keywords, args, .. } = &expr.arguments;
-        for keyword in &**keywords {
-            if let Some(ident) = keyword.arg.as_ref() {
-                arg_names.insert(&ident.id);
-            }
-        }
-        for arg in &**args {
-            if let ast::Expr::Name(ast::ExprName { id, .. }) = arg {
-                arg_names.insert(id);
-            }
-        }
-    }
-
     for f_string in value.f_strings() {
         let mut has_name = false;
         for element in f_string.elements.expressions() {
             if let ast::Expr::Name(ast::ExprName { id, .. }) = element.expression.as_ref() {
-                if arg_names.contains(id) {
+                if relevant_argument_names.contains(id) {
                     return false;
                 }
-                if semantic
-                    .lookup_symbol(id)
-                    .map_or(true, |id| semantic.binding(id).kind.is_builtin())
-                {
+                let Some(binding_id) = semantic.lookup_symbol_in_scope(id, scope, false) else {
+                    return false;
+                };
+                let binding = semantic.binding(binding_id);
+                if binding.kind.is_builtin() {
+                    return false;
+                }
+                if binding.start() > literal.end() {
                     return false;
                 }
                 has_name = true;

crates/ruff_linter/src/rules/ruff/snapshots/ruff_linter__rules__ruff__tests__RUF027_RUF027_0.py.snap

@@ -163,7 +163,7 @@ RUF027_0.py:39:19: RUF027 [*] Possible f-string without an `f` prefix
 39 |     single_line = """ {a} """  # RUF027
    |                   ^^^^^^^^^^^ RUF027
 40 |     # RUF027
-41 |     multi_line = a = """b { # comment
+41 |     multi_line = """b { # comment
    |
    = help: Add `f` prefix
 
@@ -174,15 +174,15 @@ RUF027_0.py:39:19: RUF027 [*] Possible f-string without an `f` prefix
 39    |-    single_line = """ {a} """  # RUF027
    39 |+    single_line = f""" {a} """  # RUF027
 40 40 |     # RUF027
-41 41 |     multi_line = a = """b { # comment
+41 41 |     multi_line = """b { # comment
 42 42 |     c}  d
 
-RUF027_0.py:41:22: RUF027 [*] Possible f-string without an `f` prefix
+RUF027_0.py:41:18: RUF027 [*] Possible f-string without an `f` prefix
    |
 39 |       single_line = """ {a} """  # RUF027
 40 |       # RUF027
-41 |       multi_line = a = """b { # comment
-   |  ______________________^
+41 |       multi_line = """b { # comment
+   |  __________________^
 42 | |     c}  d
 43 | |     """
    | |_______^ RUF027
@@ -193,8 +193,8 @@ RUF027_0.py:41:22: RUF027 [*] Possible f-string without an `f` prefix
 38 38 |     c = a
 39 39 |     single_line = """ {a} """  # RUF027
 40 40 |     # RUF027
-41    |-    multi_line = a = """b { # comment
-   41 |+    multi_line = a = f"""b { # comment
+41    |-    multi_line = """b { # comment
+   41 |+    multi_line = f"""b { # comment
 42 42 |     c}  d
 43 43 |     """
 44 44 | 
@@ -315,5 +315,64 @@ RUF027_0.py:74:9: RUF027 [*] Possible f-string without an `f` prefix
 73 73 |     a = 4
 74    |-    b = "{a:b} {a:^5}"
    74 |+    b = f"{a:b} {a:^5}"
+75 75 | 
+76 76 | 
+77 77 | def implicitly_concatenated_fstring(x: int):
 
+RUF027_0.py:78:16: RUF027 [*] Possible f-string without an `f` prefix
+   |
+77 | def implicitly_concatenated_fstring(x: int):
+78 |     y = f"{x}" "{x}"  # RUF027
+   |                ^^^^^ RUF027
+79 |     z = "{x}" f"{x}"  # RUF027
+   |
+   = help: Add `f` prefix
 
+ℹ Unsafe fix
+75 75 | 
+76 76 | 
+77 77 | def implicitly_concatenated_fstring(x: int):
+78    |-    y = f"{x}" "{x}"  # RUF027
+   78 |+    y = f"{x}" f"{x}"  # RUF027
+79 79 |     z = "{x}" f"{x}"  # RUF027
+80 80 | 
+81 81 | 
+
+RUF027_0.py:79:9: RUF027 [*] Possible f-string without an `f` prefix
+   |
+77 | def implicitly_concatenated_fstring(x: int):
+78 |     y = f"{x}" "{x}"  # RUF027
+79 |     z = "{x}" f"{x}"  # RUF027
+   |         ^^^^^ RUF027
+   |
+   = help: Add `f` prefix
+
+ℹ Unsafe fix
+76 76 | 
+77 77 | def implicitly_concatenated_fstring(x: int):
+78 78 |     y = f"{x}" "{x}"  # RUF027
+79    |-    z = "{x}" f"{x}"  # RUF027
+   79 |+    z = f"{x}" f"{x}"  # RUF027
+80 80 | 
+81 81 | 
+82 82 | def method_call_passed_to_logging(x: int):
+
+RUF027_0.py:85:13: RUF027 [*] Possible f-string without an `f` prefix
+   |
+83 |     # We special-case bindings that are passed to logging calls later on,
+84 |     # but not bindings that involve arbitrary call expressions!
+85 |     y = foo("{x}")  # RUF027
+   |             ^^^^^ RUF027
+86 |     import logging
+87 |     logging.log(0, y)
+   |
+   = help: Add `f` prefix
+
+ℹ Unsafe fix
+82 82 | def method_call_passed_to_logging(x: int):
+83 83 |     # We special-case bindings that are passed to logging calls later on,
+84 84 |     # but not bindings that involve arbitrary call expressions!
+85    |-    y = foo("{x}")  # RUF027
+   85 |+    y = foo(f"{x}")  # RUF027
+86 86 |     import logging
+87 87 |     logging.log(0, y)

crates/ruff_python_ast/src/nodes.rs

@@ -2152,7 +2152,7 @@ impl BytesLiteralValue {
     }
 
     /// Returns an iterator over the bytes of the concatenated bytes.
-    fn bytes(&self) -> impl Iterator<Item = u8> + '_ {
+    pub fn bytes(&self) -> impl Iterator<Item = u8> + '_ {
         self.iter().flat_map(|part| part.as_slice().iter().copied())
     }
 }

crates/ruff_python_semantic/src/model.rs

@@ -621,10 +621,22 @@ impl<'a> SemanticModel<'a> {
         }
     }
 
-    /// Lookup a symbol in the current scope. This is a carbon copy of [`Self::resolve_load`], but
-    /// doesn't add any read references to the resolved symbol.
+    /// Lookup a symbol in the current scope
     pub fn lookup_symbol(&self, symbol: &str) -> Option<BindingId> {
-        if self.in_forward_reference() {
+        self.lookup_symbol_in_scope(symbol, self.scope_id, self.in_forward_reference())
+    }
+
+    /// Lookup a symbol in a certain scope
+    ///
+    /// This is a carbon copy of [`Self::resolve_load`], but
+    /// doesn't add any read references to the resolved symbol.
+    pub fn lookup_symbol_in_scope(
+        &self,
+        symbol: &str,
+        scope: ScopeId,
+        in_forward_reference: bool,
+    ) -> Option<BindingId> {
+        if in_forward_reference {
             if let Some(binding_id) = self.scopes.global().get(symbol) {
                 if !self.bindings[binding_id].is_unbound() {
                     return Some(binding_id);
@@ -634,7 +646,7 @@ impl<'a> SemanticModel<'a> {
 
         let mut seen_function = false;
         let mut class_variables_visible = true;
-        for (index, scope_id) in self.scopes.ancestor_ids(self.scope_id).enumerate() {
+        for (index, scope_id) in self.scopes.ancestor_ids(scope).enumerate() {
             let scope = &self.scopes[scope_id];
             if scope.kind.is_class() {
                 if seen_function && matches!(symbol, "__class__") {

crates/ruff_wasm/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "ruff_wasm"
-version = "0.6.1"
+version = "0.6.2"
 publish = false
 authors = { workspace = true }
 edition = { workspace = true }

crates/ruff_workspace/src/resolver.rs

@@ -253,7 +253,7 @@ fn is_package_with_cache<'a>(
 /// Applies a transformation to a [`Configuration`].
 ///
 /// Used to override options with the values provided by the CLI.
-pub trait ConfigurationTransformer: Sync {
+pub trait ConfigurationTransformer {
     fn transform(&self, config: Configuration) -> Configuration;
 }
 
@@ -334,7 +334,7 @@ pub fn resolve_root_settings(
 pub fn python_files_in_path<'a>(
     paths: &[PathBuf],
     pyproject_config: &'a PyprojectConfig,
-    transformer: &dyn ConfigurationTransformer,
+    transformer: &(dyn ConfigurationTransformer + Sync),
 ) -> Result<(Vec<Result<ResolvedFile, ignore::Error>>, Resolver<'a>)> {
     // Normalize every path (e.g., convert from relative to absolute).
     let mut paths: Vec<PathBuf> = paths.iter().map(fs::normalize_path).unique().collect();
@@ -430,12 +430,12 @@ impl<'config> WalkPythonFilesState<'config> {
 
 struct PythonFilesVisitorBuilder<'s, 'config> {
     state: &'s WalkPythonFilesState<'config>,
-    transformer: &'s dyn ConfigurationTransformer,
+    transformer: &'s (dyn ConfigurationTransformer + Sync),
 }
 
 impl<'s, 'config> PythonFilesVisitorBuilder<'s, 'config> {
     fn new(
-        transformer: &'s dyn ConfigurationTransformer,
+        transformer: &'s (dyn ConfigurationTransformer + Sync),
         state: &'s WalkPythonFilesState<'config>,
     ) -> Self {
         Self { state, transformer }
@@ -446,7 +446,7 @@ struct PythonFilesVisitor<'s, 'config> {
     local_files: Vec<Result<ResolvedFile, ignore::Error>>,
     local_error: Result<()>,
     global: &'s WalkPythonFilesState<'config>,
-    transformer: &'s dyn ConfigurationTransformer,
+    transformer: &'s (dyn ConfigurationTransformer + Sync),
 }
 
 impl<'config, 's> ignore::ParallelVisitorBuilder<'s> for PythonFilesVisitorBuilder<'s, 'config>

docs/integrations.md

@@ -78,7 +78,7 @@ Ruff can be used as a [pre-commit](https://pre-commit.com) hook via [`ruff-pre-c
 ```yaml
 - repo: https://github.com/astral-sh/ruff-pre-commit
   # Ruff version.
-  rev: v0.6.1
+  rev: v0.6.2
   hooks:
     # Run the linter.
     - id: ruff
@@ -91,7 +91,7 @@ To enable lint fixes, add the `--fix` argument to the lint hook:
 ```yaml
 - repo: https://github.com/astral-sh/ruff-pre-commit
   # Ruff version.
-  rev: v0.6.1
+  rev: v0.6.2
   hooks:
     # Run the linter.
     - id: ruff
@@ -105,7 +105,7 @@ To run the hooks over Jupyter Notebooks too, add `jupyter` to the list of allowe
 ```yaml
 - repo: https://github.com/astral-sh/ruff-pre-commit
   # Ruff version.
-  rev: v0.6.1
+  rev: v0.6.2
   hooks:
     # Run the linter.
     - id: ruff

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "maturin"
 
 [project]
 name = "ruff"
-version = "0.6.1"
+version = "0.6.2"
 description = "An extremely fast Python linter and code formatter, written in Rust."
 authors = [{ name = "Astral Software Inc.", email = "hey@astral.sh" }]
 readme = "README.md"

scripts/benchmarks/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "scripts"
-version = "0.6.1"
+version = "0.6.2"
 description = ""
 authors = ["Charles Marsh <charlie.r.marsh@gmail.com>"]
 

scripts/knot_benchmark/README.md

@@ -0,0 +1,21 @@
+## Getting started
+
+1. [Install `uv`](https://docs.astral.sh/uv/getting-started/installation/)
+
+- Unix: `curl -LsSf https://astral.sh/uv/install.sh | sh`
+- Windows: `powershell -c "irm https://astral.sh/uv/install.ps1 | iex"`
+
+1. Build red_knot: `cargo build --bin red_knot --release`
+1. `cd` into the benchmark directory: `cd scripts/knot_benchmark`
+1. Run benchmarks: `uv run benchmark`
+
+## Known limitations
+
+Red Knot only implements a tiny fraction of Mypy's and Pyright's functionality,
+so the benchmarks aren't in any way a fair comparison today. However,
+they'll become more meaningful as we build out more type checking features in Red Knot.
+
+### Windows support
+
+The script should work on Windows, but we haven't tested it yet.
+We do make use of `shlex` which has known limitations when using non-POSIX shells.

scripts/knot_benchmark/pyproject.toml

@@ -0,0 +1,21 @@
+[project]
+name = "knot_benchmark"
+version = "0.0.1"
+description = "Package for running end-to-end Red Knot benchmarks"
+requires-python = ">=3.12"
+dependencies = ["mypy", "pyright"]
+
+[project.scripts]
+benchmark = "benchmark.run:main"
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/benchmark"]
+
+[tool.ruff.lint]
+ignore = [
+    "E501", # We use ruff format
+]

scripts/knot_benchmark/src/benchmark/__init__.py

@@ -0,0 +1,76 @@
+from __future__ import annotations
+
+import logging
+import shlex
+import subprocess
+import typing
+from pathlib import Path
+
+
+class Command(typing.NamedTuple):
+    name: str
+    """The name of the command to benchmark."""
+
+    command: list[str]
+    """The command to benchmark."""
+
+    prepare: str | None = None
+    """The command to run before each benchmark run."""
+
+
+class Hyperfine(typing.NamedTuple):
+    name: str
+    """The benchmark to run."""
+
+    commands: list[Command]
+    """The commands to benchmark."""
+
+    warmup: int
+    """The number of warmup runs to perform."""
+
+    min_runs: int
+    """The minimum number of runs to perform."""
+
+    verbose: bool
+    """Whether to print verbose output."""
+
+    json: bool
+    """Whether to export results to JSON."""
+
+    def run(self, *, cwd: Path | None = None) -> None:
+        """Run the benchmark using `hyperfine`."""
+        args = [
+            "hyperfine",
+            # Most repositories have some typing errors.
+            # This is annoying because it prevents us from capturing "real" errors.
+            "-i",
+        ]
+
+        # Export to JSON.
+        if self.json:
+            args.extend(["--export-json", f"{self.name}.json"])
+
+        # Preamble: benchmark-wide setup.
+        if self.verbose:
+            args.append("--show-output")
+
+        args.extend(["--warmup", str(self.warmup), "--min-runs", str(self.min_runs)])
+
+        # Add all command names,
+        for command in self.commands:
+            args.extend(["--command-name", command.name])
+
+        # Add all prepare statements.
+        for command in self.commands:
+            args.extend(["--prepare", command.prepare or ""])
+
+        # Add all commands.
+        for command in self.commands:
+            args.append(shlex.join(command.command))
+
+        logging.info(f"Running {args}")
+
+        subprocess.run(
+            args,
+            cwd=cwd,
+        )

scripts/knot_benchmark/src/benchmark/cases.py

@@ -0,0 +1,212 @@
+from __future__ import annotations
+
+import abc
+import enum
+import logging
+import os
+import shutil
+import subprocess
+import sys
+from pathlib import Path
+
+from benchmark import Command
+from benchmark.projects import Project
+
+
+class Benchmark(enum.Enum):
+    """Enumeration of the benchmarks to run."""
+
+    COLD = "cold"
+    """Cold check of an entire project without a cache present."""
+
+    WARM = "warm"
+    """Re-checking the entire project without any changes"."""
+
+
+def which_tool(name: str) -> Path:
+    tool = shutil.which(name)
+
+    assert (
+        tool is not None
+    ), f"Tool {name} not found. Run the script with `uv run <script>`."
+
+    return Path(tool)
+
+
+class Tool(abc.ABC):
+    def command(
+        self, benchmark: Benchmark, project: Project, venv: Venv
+    ) -> Command | None:
+        """Generate a command to benchmark a given tool."""
+        match benchmark:
+            case Benchmark.COLD:
+                return self.cold_command(project, venv)
+            case Benchmark.WARM:
+                return self.warm_command(project, venv)
+            case _:
+                raise ValueError(f"Invalid benchmark: {benchmark}")
+
+    @abc.abstractmethod
+    def cold_command(self, project: Project, venv: Venv) -> Command: ...
+
+    def warm_command(self, project: Project, venv: Venv) -> Command | None:
+        return None
+
+
+class Knot(Tool):
+    path: Path
+    name: str
+
+    def __init__(self, *, path: Path | None = None):
+        self.name = str(path) or "knot"
+        self.path = path or (
+            (Path(__file__) / "../../../../../target/release/red_knot").resolve()
+        )
+
+        assert self.path.is_file(), f"Red Knot not found at '{self.path}'. Run `cargo build --release --bin red_knot`."
+
+    def cold_command(self, project: Project, venv: Venv) -> Command:
+        command = [str(self.path), "-v"]
+
+        assert len(project.include) < 2, "Knot doesn't support multiple source folders"
+
+        if project.include:
+            command.extend(["--current-directory", project.include[0]])
+
+        command.extend(["--venv-path", str(venv.path)])
+
+        return Command(
+            name="knot",
+            command=command,
+        )
+
+
+class Mypy(Tool):
+    path: Path
+
+    def __init__(self, *, path: Path | None = None):
+        self.path = path or which_tool(
+            "mypy",
+        )
+
+    def cold_command(self, project: Project, venv: Venv) -> Command:
+        command = [
+            *self._base_command(project, venv),
+            "--no-incremental",
+            "--cache-dir",
+            os.devnull,
+        ]
+
+        return Command(
+            name="mypy",
+            command=command,
+        )
+
+    def warm_command(self, project: Project, venv: Venv) -> Command | None:
+        command = [
+            str(self.path),
+            *(project.mypy_arguments or project.include),
+            "--python-executable",
+            str(venv.python),
+        ]
+
+        return Command(
+            name="mypy",
+            command=command,
+        )
+
+    def _base_command(self, project: Project, venv: Venv) -> list[str]:
+        return [
+            str(self.path),
+            "--python-executable",
+            str(venv.python),
+            *(project.mypy_arguments or project.include),
+        ]
+
+
+class Pyright(Tool):
+    path: Path
+
+    def __init__(self, *, path: Path | None = None):
+        self.path = path or which_tool("pyright")
+
+    def cold_command(self, project: Project, venv: Venv) -> Command:
+        command = [
+            str(self.path),
+            "--venvpath",
+            str(
+                venv.path.parent
+            ),  # This is not the path to the venv folder, but the folder that contains the venv...
+            *(project.pyright_arguments or project.include),
+        ]
+
+        return Command(
+            name="Pyright",
+            command=command,
+        )
+
+
+class Venv:
+    path: Path
+
+    def __init__(self, path: Path):
+        self.path = path
+
+    @property
+    def name(self):
+        """The name of the virtual environment directory."""
+        return self.path.name
+
+    @property
+    def python(self):
+        """Returns the path to the python executable"""
+        return self.script("python")
+
+    @property
+    def bin(self) -> Path:
+        bin_dir = "scripts" if sys.platform == "win32" else "bin"
+        return self.path / bin_dir
+
+    def script(self, name: str) -> Path:
+        extension = ".exe" if sys.platform == "win32" else ""
+        return self.bin / f"{name}{extension}"
+
+    @staticmethod
+    def create(parent: Path) -> Venv:
+        """Creates a new, empty virtual environment."""
+
+        command = [
+            "uv",
+            "venv",
+            "--quiet",
+            "venv",
+        ]
+
+        try:
+            subprocess.run(
+                command, cwd=parent, check=True, capture_output=True, text=True
+            )
+        except subprocess.CalledProcessError as e:
+            raise RuntimeError(f"Failed to create venv: {e.stderr}")
+
+        root = parent / "venv"
+        return Venv(root)
+
+    def install(self, dependencies: list[str]):
+        """Installs the dependencies required to type check the project."""
+
+        logging.debug(f"Installing dependencies: {', '.join(dependencies)}")
+        command = [
+            "uv",
+            "pip",
+            "install",
+            "--quiet",
+            *dependencies,
+        ]
+
+        try:
+            subprocess.run(
+                command, cwd=self.path, check=True, capture_output=True, text=True
+            )
+        except subprocess.CalledProcessError as e:
+            raise RuntimeError(f"Failed to install dependencies: {e.stderr}")

scripts/knot_benchmark/src/benchmark/projects.py

@@ -0,0 +1,142 @@
+import logging
+import os
+import subprocess
+import typing
+from pathlib import Path
+
+
+class Project(typing.NamedTuple):
+    name: str
+    """The name of the project to benchmark."""
+
+    repository: str
+    """The git repository to clone."""
+
+    revision: str
+
+    dependencies: list[str]
+    """List of type checking dependencies"""
+
+    include: list[str] = []
+    """The directories and files to check. If empty, checks the current directory"""
+
+    pyright_arguments: list[str] | None = None
+    """The arguments passed to pyright. Overrides `include` if set."""
+
+    mypy_arguments: list[str] | None = None
+    """The arguments passed to mypy. Overrides `include` if set."""
+
+    def clone(self, checkout_dir: Path):
+        # Skip cloning if the project has already been cloned (the script doesn't yet support updating)
+        if os.path.exists(os.path.join(checkout_dir, ".git")):
+            return
+
+        logging.debug(f"Cloning {self.repository} to {checkout_dir}")
+
+        try:
+            # git doesn't support cloning a specific revision.
+            # This is the closest that I found to a "shallow clone with a specific revision"
+            subprocess.run(
+                [
+                    "git",
+                    "init",
+                    "--quiet",
+                ],
+                env={"GIT_TERMINAL_PROMPT": "0"},
+                cwd=checkout_dir,
+                check=True,
+                capture_output=True,
+                text=True,
+            )
+
+            subprocess.run(
+                ["git", "remote", "add", "origin", str(self.repository), "--no-fetch"],
+                env={"GIT_TERMINAL_PROMPT": "0"},
+                cwd=checkout_dir,
+                check=True,
+                capture_output=True,
+                text=True,
+            )
+
+            subprocess.run(
+                [
+                    "git",
+                    "fetch",
+                    "origin",
+                    self.revision,
+                    "--quiet",
+                    "--depth",
+                    "1",
+                    "--no-tags",
+                ],
+                check=True,
+                cwd=checkout_dir,
+                capture_output=True,
+                text=True,
+            )
+
+            subprocess.run(
+                ["git", "reset", "--hard", "FETCH_HEAD", "--quiet"],
+                check=True,
+                cwd=checkout_dir,
+                capture_output=True,
+                text=True,
+            )
+
+        except subprocess.CalledProcessError as e:
+            raise RuntimeError(f"Failed to clone {self.name}: {e.stderr}")
+
+        logging.info(f"Cloned {self.name} to {checkout_dir}.")
+
+
+# Selection of projects taken from
+# [mypy-primer](https://github.com/hauntsaninja/mypy_primer/blob/0ea6cc614b3e91084059b9a3acc58f94c066a211/mypy_primer/projects.py#L71).
+# May require frequent updating, especially the dependencies list
+ALL = [
+    Project(
+        name="black",
+        repository="https://github.com/psf/black",
+        revision="c20423249e9d8dfb8581eebbfc67a13984ee45e9",
+        include=["src"],
+        dependencies=[
+            "aiohttp",
+            "click",
+            "pathspec",
+            "tomli",
+            "platformdirs",
+            "packaging",
+        ],
+    ),
+    Project(
+        name="jinja",
+        repository="https://github.com/pallets/jinja",
+        revision="b490da6b23b7ad25dc969976f64dc4ffb0a2c182",
+        include=[],
+        dependencies=["markupsafe"],
+    ),
+    Project(
+        name="pandas",
+        repository="https://github.com/pandas-dev/pandas",
+        revision="7945e563d36bcf4694ccc44698829a6221905839",
+        include=["pandas"],
+        dependencies=[
+            "numpy",
+            "types-python-dateutil",
+            "types-pytz",
+            "types-PyMySQL",
+            "types-setuptools",
+            "pytest",
+        ],
+    ),
+    Project(
+        name="isort",
+        repository="https://github.com/pycqa/isort",
+        revision="7de182933fd50e04a7c47cc8be75a6547754b19c",
+        mypy_arguments=["--ignore-missing-imports", "isort"],
+        include=["isort"],
+        dependencies=["types-setuptools"],
+    ),
+]
+
+
+DEFAULT: list[str] = ["black", "jinja", "isort"]

scripts/knot_benchmark/src/benchmark/run.py

@@ -0,0 +1,153 @@
+from __future__ import annotations
+
+import argparse
+import json
+import logging
+import tempfile
+import typing
+from pathlib import Path
+
+from benchmark import Hyperfine
+from benchmark.cases import Benchmark, Knot, Mypy, Pyright, Tool, Venv
+from benchmark.projects import ALL as all_projects
+from benchmark.projects import DEFAULT as default_projects
+
+if typing.TYPE_CHECKING:
+    from benchmark.cases import Tool
+
+
+def main():
+    """Run the benchmark."""
+    parser = argparse.ArgumentParser(
+        description="Benchmark knot against other packaging tools."
+    )
+    parser.add_argument(
+        "--verbose", "-v", action="store_true", help="Print verbose output."
+    )
+    parser.add_argument(
+        "--warmup",
+        type=int,
+        help="The number of warmup runs to perform.",
+        default=3,
+    )
+    parser.add_argument(
+        "--min-runs",
+        type=int,
+        help="The minimum number of runs to perform.",
+        default=10,
+    )
+    parser.add_argument(
+        "--benchmark",
+        "-b",
+        type=str,
+        help="The benchmark(s) to run.",
+        choices=[benchmark.value for benchmark in Benchmark],
+        action="append",
+    )
+    parser.add_argument(
+        "--project",
+        "-p",
+        type=str,
+        help="The project(s) to run.",
+        choices=[project.name for project in all_projects],
+        action="append",
+    )
+    parser.add_argument(
+        "--mypy",
+        help="Whether to benchmark `mypy`.",
+        action="store_true",
+    )
+    parser.add_argument(
+        "--pyright",
+        help="Whether to benchmark `pyright`.",
+        action="store_true",
+    )
+    parser.add_argument(
+        "--knot",
+        help="Whether to benchmark knot (assumes a red_knot binary exists at `./target/release/red_knot`).",
+        action="store_true",
+    )
+    parser.add_argument(
+        "--knot-path",
+        type=str,
+        help="Path(s) to the red_knot binary to benchmark.",
+        action="append",
+    )
+
+    args = parser.parse_args()
+    logging.basicConfig(
+        level=logging.INFO if args.verbose else logging.WARN,
+        format="%(asctime)s %(levelname)s %(message)s",
+        datefmt="%Y-%m-%d %H:%M:%S",
+    )
+
+    verbose = args.verbose
+    warmup = args.warmup
+    min_runs = args.min_runs
+
+    # Determine the tools to benchmark, based on the user-provided arguments.
+    suites: list[Tool] = []
+    if args.pyright:
+        suites.append(Pyright())
+
+    if args.knot:
+        suites.append(Knot())
+
+    for path in args.knot_path or []:
+        suites.append(Knot(path=path))
+
+    if args.mypy:
+        suites.append(Mypy())
+
+    # If no tools were specified, default to benchmarking all tools.
+    suites = suites or [Knot(), Pyright(), Mypy()]
+
+    # Determine the benchmarks to run, based on user input.
+    benchmarks = (
+        [Benchmark(benchmark) for benchmark in args.benchmark]
+        if args.benchmark is not None
+        else list(Benchmark)
+    )
+
+    projects = [
+        project
+        for project in all_projects
+        if project.name in (args.project or default_projects)
+    ]
+
+    for project in projects:
+        with tempfile.TemporaryDirectory() as cwd:
+            cwd = Path(cwd)
+            project.clone(cwd)
+
+            venv = Venv.create(cwd)
+            venv.install(project.dependencies)
+
+            # Set the `venv` config for pyright. Pyright only respects the `--venvpath`
+            # CLI option when `venv` is set in the configuration... 🤷‍♂️
+            with open(cwd / "pyrightconfig.json", "w") as f:
+                f.write(json.dumps(dict(venv=venv.name)))
+
+            for benchmark in benchmarks:
+                # Generate the benchmark command for each tool.
+                commands = [
+                    command
+                    for suite in suites
+                    if (command := suite.command(benchmark, project, venv))
+                ]
+
+                # not all tools support all benchmarks.
+                if not commands:
+                    continue
+
+                print(f"{project.name} ({benchmark.value})")
+
+                hyperfine = Hyperfine(
+                    name=f"{project.name}-{benchmark.value}",
+                    commands=commands,
+                    warmup=warmup,
+                    min_runs=min_runs,
+                    verbose=verbose,
+                    json=False,
+                )
+                hyperfine.run(cwd=cwd)

scripts/knot_benchmark/uv.lock

@@ -0,0 +1,74 @@
+version = 1
+requires-python = ">=3.12"
+
+[[package]]
+name = "knot-benchmark"
+version = "0.0.1"
+source = { editable = "." }
+dependencies = [
+    { name = "mypy" },
+    { name = "pyright" },
+]
+
+[package.metadata]
+requires-dist = [
+    { name = "mypy" },
+    { name = "pyright" },
+]
+
+[[package]]
+name = "mypy"
+version = "1.11.1"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "mypy-extensions" },
+    { name = "typing-extensions" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/b6/9c/a4b3bda53823439cf395db8ecdda6229a83f9bf201714a68a15190bb2919/mypy-1.11.1.tar.gz", hash = "sha256:f404a0b069709f18bbdb702eb3dcfe51910602995de00bd39cea3050b5772d08", size = 3078369 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/3a/34/69638cee2e87303f19a0c35e80d42757e14d9aba328f272fdcdc0bf3c9b8/mypy-1.11.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:f39918a50f74dc5969807dcfaecafa804fa7f90c9d60506835036cc1bc891dc8", size = 10995789 },
+    { url = "https://files.pythonhosted.org/packages/c4/3c/3e0611348fc53a4a7c80485959478b4f6eae706baf3b7c03cafa22639216/mypy-1.11.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0bc71d1fb27a428139dd78621953effe0d208aed9857cb08d002280b0422003a", size = 10002696 },
+    { url = "https://files.pythonhosted.org/packages/1c/21/a6b46c91b4c9d1918ee59c305f46850cde7cbea748635a352e7c3c8ed204/mypy-1.11.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b868d3bcff720dd7217c383474008ddabaf048fad8d78ed948bb4b624870a417", size = 12505772 },
+    { url = "https://files.pythonhosted.org/packages/c4/55/07904d4c8f408e70308015edcbff067eaa77514475938a9dd81b063de2a8/mypy-1.11.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:a707ec1527ffcdd1c784d0924bf5cb15cd7f22683b919668a04d2b9c34549d2e", size = 12954190 },
+    { url = "https://files.pythonhosted.org/packages/1e/b7/3a50f318979c8c541428c2f1ee973cda813bcc89614de982dafdd0df2b3e/mypy-1.11.1-cp312-cp312-win_amd64.whl", hash = "sha256:64f4a90e3ea07f590c5bcf9029035cf0efeae5ba8be511a8caada1a4893f5525", size = 9663138 },
+    { url = "https://files.pythonhosted.org/packages/f8/d4/4960d0df55f30a7625d9c3c9414dfd42f779caabae137ef73ffaed0c97b9/mypy-1.11.1-py3-none-any.whl", hash = "sha256:0624bdb940255d2dd24e829d99a13cfeb72e4e9031f9492148f410ed30bcab54", size = 2619257 },
+]
+
+[[package]]
+name = "mypy-extensions"
+version = "1.0.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/98/a4/1ab47638b92648243faf97a5aeb6ea83059cc3624972ab6b8d2316078d3f/mypy_extensions-1.0.0.tar.gz", hash = "sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782", size = 4433 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/2a/e2/5d3f6ada4297caebe1a2add3b126fe800c96f56dbe5d1988a2cbe0b267aa/mypy_extensions-1.0.0-py3-none-any.whl", hash = "sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d", size = 4695 },
+]
+
+[[package]]
+name = "nodeenv"
+version = "1.9.1"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/43/16/fc88b08840de0e0a72a2f9d8c6bae36be573e475a6326ae854bcc549fc45/nodeenv-1.9.1.tar.gz", hash = "sha256:6ec12890a2dab7946721edbfbcd91f3319c6ccc9aec47be7c7e6b7011ee6645f", size = 47437 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/d2/1d/1b658dbd2b9fa9c4c9f32accbfc0205d532c8c6194dc0f2a4c0428e7128a/nodeenv-1.9.1-py2.py3-none-any.whl", hash = "sha256:ba11c9782d29c27c70ffbdda2d7415098754709be8a7056d79a737cd901155c9", size = 22314 },
+]
+
+[[package]]
+name = "pyright"
+version = "1.1.377"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "nodeenv" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/49/f0/25b0db363d6888164adb7c828b877bbf2c30936955fb9513922ae03e70e4/pyright-1.1.377.tar.gz", hash = "sha256:aabc30fedce0ded34baa0c49b24f10e68f4bfc8f68ae7f3d175c4b0f256b4fcf", size = 17484 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/34/c9/89c40c4de44fe9463e77dddd0c4e2d2dd7a93e8ddc6858dfe7d5f75d263d/pyright-1.1.377-py3-none-any.whl", hash = "sha256:af0dd2b6b636c383a6569a083f8c5a8748ae4dcde5df7914b3f3f267e14dd162", size = 18223 },
+]
+
+[[package]]
+name = "typing-extensions"
+version = "4.12.2"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/df/db/f35a00659bc03fec321ba8bce9420de607a1d37f8342eee1863174c69557/typing_extensions-4.12.2.tar.gz", hash = "sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8", size = 85321 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/26/9f/ad63fc0248c5379346306f8668cda6e2e2e9c95e01216d2b8ffd9ff037d0/typing_extensions-4.12.2-py3-none-any.whl", hash = "sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d", size = 37438 },
+]