serde_json is no longer optional

## Summary

By taking a purely syntactic approach to the problem of trivial
initializer calls we can supress `x: T = T()`, `x: T = x.y.T()` and `x:
MyNewType = MyNewType(0)` but still display `x: T[U] = T()`.

The place where we drop a ball is this does not compose with our
analysis for supressing `x = (0, "hello")` as `x = (0, T())` and `x =
(T(), T())` will still get inlay hints (I don't think this is a huge
deal).

* fixes https://github.com/astral-sh/ty/issues/1516

## Test Plan

Existing snapshots cover this well.

.github/renovate.json5

@@ -75,14 +75,6 @@
       matchManagers: ["cargo"],
       enabled: false,
     },
-    {
-      // `mkdocs-material` requires a manual update to keep the version in sync
-      // with `mkdocs-material-insider`.
-      // See: https://squidfunk.github.io/mkdocs-material/insiders/upgrade/
-      matchManagers: ["pip_requirements"],
-      matchPackageNames: ["mkdocs-material"],
-      enabled: false,
-    },
     {
       groupName: "pre-commit dependencies",
       matchManagers: ["pre-commit"],

.github/workflows/ci.yaml

@@ -24,6 +24,8 @@ env:
   PACKAGE_NAME: ruff
   PYTHON_VERSION: "3.14"
   NEXTEST_PROFILE: ci
+  # Enable mdtests that require external dependencies
+  MDTEST_EXTERNAL: "1"
 
 jobs:
   determine_changes:
@@ -779,8 +781,6 @@ jobs:
     name: "mkdocs"
     runs-on: ubuntu-latest
     timeout-minutes: 10
-    env:
-      MKDOCS_INSIDERS_SSH_KEY_EXISTS: ${{ secrets.MKDOCS_INSIDERS_SSH_KEY != '' }}
     steps:
       - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
         with:
@@ -788,11 +788,6 @@ jobs:
       - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2.8.2
         with:
           save-if: ${{ github.ref == 'refs/heads/main' }}
-      - name: "Add SSH key"
-        if: ${{ env.MKDOCS_INSIDERS_SSH_KEY_EXISTS == 'true' }}
-        uses: webfactory/ssh-agent@a6f90b1f127823b31d4d4a8d96047790581349bd # v0.9.1
-        with:
-          ssh-private-key: ${{ secrets.MKDOCS_INSIDERS_SSH_KEY }}
       - name: "Install Rust toolchain"
         run: rustup show
       - name: Install uv
@@ -800,11 +795,7 @@ jobs:
         with:
           python-version: 3.13
           activate-environment: true
-      - name: "Install Insiders dependencies"
-        if: ${{ env.MKDOCS_INSIDERS_SSH_KEY_EXISTS == 'true' }}
-        run: uv pip install -r docs/requirements-insiders.txt
       - name: "Install dependencies"
-        if: ${{ env.MKDOCS_INSIDERS_SSH_KEY_EXISTS != 'true' }}
         run: uv pip install -r docs/requirements.txt
       - name: "Update README File"
         run: python scripts/transform_readme.py --target mkdocs
@@ -812,12 +803,8 @@ jobs:
         run: python scripts/generate_mkdocs.py
       - name: "Check docs formatting"
         run: python scripts/check_docs_formatted.py
-      - name: "Build Insiders docs"
-        if: ${{ env.MKDOCS_INSIDERS_SSH_KEY_EXISTS == 'true' }}
-        run: mkdocs build --strict -f mkdocs.insiders.yml
       - name: "Build docs"
-        if: ${{ env.MKDOCS_INSIDERS_SSH_KEY_EXISTS != 'true' }}
-        run: mkdocs build --strict -f mkdocs.public.yml
+        run: mkdocs build --strict -f mkdocs.yml
 
   check-formatter-instability-and-black-similarity:
     name: "formatter instabilities and black similarity"

.github/workflows/publish-docs.yml

@@ -20,8 +20,6 @@ on:
 jobs:
   mkdocs:
     runs-on: ubuntu-latest
-    env:
-      MKDOCS_INSIDERS_SSH_KEY_EXISTS: ${{ secrets.MKDOCS_INSIDERS_SSH_KEY != '' }}
     steps:
       - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
         with:
@@ -59,23 +57,12 @@ jobs:
           echo "branch_name=update-docs-$branch_display_name-$timestamp" >> "$GITHUB_ENV"
           echo "timestamp=$timestamp" >> "$GITHUB_ENV"
 
-      - name: "Add SSH key"
-        if: ${{ env.MKDOCS_INSIDERS_SSH_KEY_EXISTS == 'true' }}
-        uses: webfactory/ssh-agent@a6f90b1f127823b31d4d4a8d96047790581349bd # v0.9.1
-        with:
-          ssh-private-key: ${{ secrets.MKDOCS_INSIDERS_SSH_KEY }}
-
       - name: "Install Rust toolchain"
         run: rustup show
 
       - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2.8.2
 
-      - name: "Install Insiders dependencies"
-        if: ${{ env.MKDOCS_INSIDERS_SSH_KEY_EXISTS == 'true' }}
-        run: pip install -r docs/requirements-insiders.txt
-
       - name: "Install dependencies"
-        if: ${{ env.MKDOCS_INSIDERS_SSH_KEY_EXISTS != 'true' }}
         run: pip install -r docs/requirements.txt
 
       - name: "Copy README File"
@@ -83,13 +70,8 @@ jobs:
           python scripts/transform_readme.py --target mkdocs
           python scripts/generate_mkdocs.py
 
-      - name: "Build Insiders docs"
-        if: ${{ env.MKDOCS_INSIDERS_SSH_KEY_EXISTS == 'true' }}
-        run: mkdocs build --strict -f mkdocs.insiders.yml
-
       - name: "Build docs"
-        if: ${{ env.MKDOCS_INSIDERS_SSH_KEY_EXISTS != 'true' }}
-        run: mkdocs build --strict -f mkdocs.public.yml
+        run: mkdocs build --strict -f mkdocs.yml
 
       - name: "Clone docs repo"
         run: git clone https://${{ secrets.ASTRAL_DOCS_PAT }}@github.com/astral-sh/docs.git astral-docs

.github/workflows/publish-pypi.yml

@@ -18,7 +18,8 @@ jobs:
     environment:
       name: release
     permissions:
-      id-token: write # For PyPI's trusted publishing + PEP 740 attestations
+      # For PyPI's trusted publishing.
+      id-token: write
     steps:
       - name: "Install uv"
         uses: astral-sh/setup-uv@1e862dfacbd1d6d858c55d9b792c756523627244 # v7.1.4
@@ -27,8 +28,5 @@ jobs:
           pattern: wheels-*
           path: wheels
           merge-multiple: true
-      - uses: astral-sh/attest-action@2c727738cea36d6c97dd85eb133ea0e0e8fe754b # v0.0.4
-        with:
-          paths: wheels/*
       - name: Publish to PyPi
         run: uv publish -v wheels/*

CHANGELOG.md

@@ -1,5 +1,34 @@
 # Changelog
 
+## 0.14.8
+
+Released on 2025-12-04.
+
+### Preview features
+
+- \[`flake8-bugbear`\] Catch `yield` expressions within other statements (`B901`) ([#21200](https://github.com/astral-sh/ruff/pull/21200))
+- \[`flake8-use-pathlib`\] Mark fixes unsafe for return type changes (`PTH104`, `PTH105`, `PTH109`, `PTH115`) ([#21440](https://github.com/astral-sh/ruff/pull/21440))
+
+### Bug fixes
+
+- Fix syntax error false positives for `await` outside functions ([#21763](https://github.com/astral-sh/ruff/pull/21763))
+- \[`flake8-simplify`\] Fix truthiness assumption for non-iterable arguments in tuple/list/set calls (`SIM222`, `SIM223`) ([#21479](https://github.com/astral-sh/ruff/pull/21479))
+
+### Documentation
+
+- Suggest using `--output-file` option in GitLab integration ([#21706](https://github.com/astral-sh/ruff/pull/21706))
+
+### Other changes
+
+- [syntax-error] Default type parameter followed by non-default type parameter ([#21657](https://github.com/astral-sh/ruff/pull/21657))
+
+### Contributors
+
+- [@kieran-ryan](https://github.com/kieran-ryan)
+- [@11happy](https://github.com/11happy)
+- [@danparizher](https://github.com/danparizher)
+- [@ntBre](https://github.com/ntBre)
+
 ## 0.14.7
 
 Released on 2025-11-28.

CONTRIBUTING.md

@@ -331,13 +331,6 @@ you addressed them.
 
 ## MkDocs
 
-> [!NOTE]
->
-> The documentation uses Material for MkDocs Insiders, which is closed-source software.
-> This means only members of the Astral organization can preview the documentation exactly as it
-> will appear in production.
-> Outside contributors can still preview the documentation, but there will be some differences. Consult [the Material for MkDocs documentation](https://squidfunk.github.io/mkdocs-material/insiders/benefits/#features) for which features are exclusively available in the insiders version.
-
 To preview any changes to the documentation locally:
 
 1. Install the [Rust toolchain](https://www.rust-lang.org/tools/install).
@@ -351,11 +344,7 @@ To preview any changes to the documentation locally:
 1. Run the development server with:
 
     ```shell
-    # For contributors.
-    uvx --with-requirements docs/requirements.txt -- mkdocs serve -f mkdocs.public.yml
-
-    # For members of the Astral org, which has access to MkDocs Insiders via sponsorship.
-    uvx --with-requirements docs/requirements-insiders.txt -- mkdocs serve -f mkdocs.insiders.yml
+    uvx --with-requirements docs/requirements.txt -- mkdocs serve -f mkdocs.yml
     ```
 
 The documentation should then be available locally at

Cargo.lock

@@ -2859,7 +2859,7 @@ dependencies = [
 
 [[package]]
 name = "ruff"
-version = "0.14.7"
+version = "0.14.8"
 dependencies = [
  "anyhow",
  "argfile",
@@ -3117,13 +3117,14 @@ dependencies = [
 
 [[package]]
 name = "ruff_linter"
-version = "0.14.7"
+version = "0.14.8"
 dependencies = [
  "aho-corasick",
  "anyhow",
  "bitflags 2.10.0",
  "clap",
  "colored 3.0.0",
+ "compact_str",
  "fern",
  "glob",
  "globset",
@@ -3472,7 +3473,7 @@ dependencies = [
 
 [[package]]
 name = "ruff_wasm"
-version = "0.14.7"
+version = "0.14.8"
 dependencies = [
  "console_error_panic_hook",
  "console_log",
@@ -4556,6 +4557,7 @@ dependencies = [
  "anyhow",
  "camino",
  "colored 3.0.0",
+ "dunce",
  "insta",
  "memchr",
  "path-slash",

Cargo.toml

@@ -272,6 +272,12 @@ large_stack_arrays = "allow"
 lto = "fat"
 codegen-units = 16
 
+# Profile to build a minimally sized binary for ruff/ty
+[profile.minimal-size]
+inherits = "release"
+opt-level = "z"
+codegen-units = 1
+
 # Some crates don't change as much but benefit more from
 # more expensive optimization passes, so we selectively
 # decrease codegen-units in some cases.

README.md

@@ -147,8 +147,8 @@ curl -LsSf https://astral.sh/ruff/install.sh | sh
 powershell -c "irm https://astral.sh/ruff/install.ps1 | iex"
 
 # For a specific version.
-curl -LsSf https://astral.sh/ruff/0.14.7/install.sh | sh
-powershell -c "irm https://astral.sh/ruff/0.14.7/install.ps1 | iex"
+curl -LsSf https://astral.sh/ruff/0.14.8/install.sh | sh
+powershell -c "irm https://astral.sh/ruff/0.14.8/install.ps1 | iex"
 ```
 
 You can also install Ruff via [Homebrew](https://formulae.brew.sh/formula/ruff), [Conda](https://anaconda.org/conda-forge/ruff),
@@ -181,7 +181,7 @@ Ruff can also be used as a [pre-commit](https://pre-commit.com/) hook via [`ruff
 ```yaml
 - repo: https://github.com/astral-sh/ruff-pre-commit
   # Ruff version.
-  rev: v0.14.7
+  rev: v0.14.8
   hooks:
     # Run the linter.
     - id: ruff-check

crates/ruff/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "ruff"
-version = "0.14.7"
+version = "0.14.8"
 publish = true
 authors = { workspace = true }
 edition = { workspace = true }

crates/ruff_benchmark/benches/lexer.rs

@@ -6,7 +6,8 @@ use criterion::{
 use ruff_benchmark::{
     LARGE_DATASET, NUMPY_CTYPESLIB, NUMPY_GLOBALS, PYDANTIC_TYPES, TestCase, UNICODE_PYPINYIN,
 };
-use ruff_python_parser::{Mode, TokenKind, lexer};
+use ruff_python_ast::token::TokenKind;
+use ruff_python_parser::{Mode, lexer};
 
 #[cfg(target_os = "windows")]
 #[global_allocator]

crates/ruff_db/src/diagnostic/mod.rs

@@ -166,28 +166,8 @@ impl Diagnostic {
     /// Returns the primary message for this diagnostic.
     ///
     /// A diagnostic always has a message, but it may be empty.
-    ///
-    /// NOTE: At present, this routine will return the first primary
-    /// annotation's message as the primary message when the main diagnostic
-    /// message is empty. This is meant to facilitate an incremental migration
-    /// in ty over to the new diagnostic data model. (The old data model
-    /// didn't distinguish between messages on the entire diagnostic and
-    /// messages attached to a particular span.)
     pub fn primary_message(&self) -> &str {
-        if !self.inner.message.as_str().is_empty() {
-            return self.inner.message.as_str();
-        }
-        // FIXME: As a special case, while we're migrating ty
-        // to the new diagnostic data model, we'll look for a primary
-        // message from the primary annotation. This is because most
-        // ty diagnostics are created with an empty diagnostic
-        // message and instead attach the message to the annotation.
-        // Fixing this will require touching basically every diagnostic
-        // in ty, so we do it this way for now to match the old
-        // semantics. ---AG
-        self.primary_annotation()
-            .and_then(|ann| ann.get_message())
-            .unwrap_or_default()
+        self.inner.message.as_str()
     }
 
     /// Introspects this diagnostic and returns what kind of "primary" message
@@ -199,18 +179,6 @@ impl Diagnostic {
     /// contains *essential* information or context for understanding the
     /// diagnostic.
     ///
-    /// The reason why we don't just always return both the main diagnostic
-    /// message and the primary annotation message is because this was written
-    /// in the midst of an incremental migration of ty over to the new
-    /// diagnostic data model. At time of writing, diagnostics were still
-    /// constructed in the old model where the main diagnostic message and the
-    /// primary annotation message were not distinguished from each other. So
-    /// for now, we carefully return what kind of messages this diagnostic
-    /// contains. In effect, if this diagnostic has a non-empty main message
-    /// *and* a non-empty primary annotation message, then the diagnostic is
-    /// 100% using the new diagnostic data model and we can format things
-    /// appropriately.
-    ///
     /// The type returned implements the `std::fmt::Display` trait. In most
     /// cases, just converting it to a string (or printing it) will do what
     /// you want.
@@ -224,11 +192,10 @@ impl Diagnostic {
             .primary_annotation()
             .and_then(|ann| ann.get_message())
             .unwrap_or_default();
-        match (main.is_empty(), annotation.is_empty()) {
-            (false, true) => ConciseMessage::MainDiagnostic(main),
-            (true, false) => ConciseMessage::PrimaryAnnotation(annotation),
-            (false, false) => ConciseMessage::Both { main, annotation },
-            (true, true) => ConciseMessage::Empty,
+        if annotation.is_empty() {
+            ConciseMessage::MainDiagnostic(main)
+        } else {
+            ConciseMessage::Both { main, annotation }
         }
     }
 
@@ -693,18 +660,6 @@ impl SubDiagnostic {
     /// contains *essential* information or context for understanding the
     /// diagnostic.
     ///
-    /// The reason why we don't just always return both the main diagnostic
-    /// message and the primary annotation message is because this was written
-    /// in the midst of an incremental migration of ty over to the new
-    /// diagnostic data model. At time of writing, diagnostics were still
-    /// constructed in the old model where the main diagnostic message and the
-    /// primary annotation message were not distinguished from each other. So
-    /// for now, we carefully return what kind of messages this diagnostic
-    /// contains. In effect, if this diagnostic has a non-empty main message
-    /// *and* a non-empty primary annotation message, then the diagnostic is
-    /// 100% using the new diagnostic data model and we can format things
-    /// appropriately.
-    ///
     /// The type returned implements the `std::fmt::Display` trait. In most
     /// cases, just converting it to a string (or printing it) will do what
     /// you want.
@@ -714,11 +669,10 @@ impl SubDiagnostic {
             .primary_annotation()
             .and_then(|ann| ann.get_message())
             .unwrap_or_default();
-        match (main.is_empty(), annotation.is_empty()) {
-            (false, true) => ConciseMessage::MainDiagnostic(main),
-            (true, false) => ConciseMessage::PrimaryAnnotation(annotation),
-            (false, false) => ConciseMessage::Both { main, annotation },
-            (true, true) => ConciseMessage::Empty,
+        if annotation.is_empty() {
+            ConciseMessage::MainDiagnostic(main)
+        } else {
+            ConciseMessage::Both { main, annotation }
         }
     }
 }
@@ -888,6 +842,10 @@ impl Annotation {
     pub fn hide_snippet(&mut self, yes: bool) {
         self.hide_snippet = yes;
     }
+
+    pub fn is_primary(&self) -> bool {
+        self.is_primary
+    }
 }
 
 /// Tags that can be associated with an annotation.
@@ -1508,28 +1466,10 @@ pub enum DiagnosticFormat {
 pub enum ConciseMessage<'a> {
     /// A diagnostic contains a non-empty main message and an empty
     /// primary annotation message.
-    ///
-    /// This strongly suggests that the diagnostic is using the
-    /// "new" data model.
     MainDiagnostic(&'a str),
-    /// A diagnostic contains an empty main message and a non-empty
-    /// primary annotation message.
-    ///
-    /// This strongly suggests that the diagnostic is using the
-    /// "old" data model.
-    PrimaryAnnotation(&'a str),
     /// A diagnostic contains a non-empty main message and a non-empty
     /// primary annotation message.
-    ///
-    /// This strongly suggests that the diagnostic is using the
-    /// "new" data model.
     Both { main: &'a str, annotation: &'a str },
-    /// A diagnostic contains an empty main message and an empty
-    /// primary annotation message.
-    ///
-    /// This indicates that the diagnostic is probably using the old
-    /// model.
-    Empty,
     /// A custom concise message has been provided.
     Custom(&'a str),
 }
@@ -1540,13 +1480,9 @@ impl std::fmt::Display for ConciseMessage<'_> {
             ConciseMessage::MainDiagnostic(main) => {
                 write!(f, "{main}")
             }
-            ConciseMessage::PrimaryAnnotation(annotation) => {
-                write!(f, "{annotation}")
-            }
             ConciseMessage::Both { main, annotation } => {
                 write!(f, "{main}: {annotation}")
             }
-            ConciseMessage::Empty => Ok(()),
             ConciseMessage::Custom(message) => {
                 write!(f, "{message}")
             }

crates/ruff_db/src/parsed.rs

@@ -21,7 +21,11 @@ use crate::source::source_text;
 /// reflected in the changed AST offsets.
 /// The other reason is that Ruff's AST doesn't implement `Eq` which Salsa requires
 /// for determining if a query result is unchanged.
-#[salsa::tracked(returns(ref), no_eq, heap_size=ruff_memory_usage::heap_size)]
+///
+/// The LRU capacity of 200 was picked without any empirical evidence that it's optimal,
+/// instead it's a wild guess that it should be unlikely that incremental changes involve
+/// more than 200 modules. Parsed ASTs within the same revision are never evicted by Salsa.
+#[salsa::tracked(returns(ref), no_eq, heap_size=ruff_memory_usage::heap_size, lru=200)]
 pub fn parsed_module(db: &dyn Db, file: File) -> ParsedModule {
     let _span = tracing::trace_span!("parsed_module", ?file).entered();
 
@@ -92,14 +96,9 @@ impl ParsedModule {
         self.inner.store(None);
     }
 
-    /// Returns the pointer address of this [`ParsedModule`].
-    ///
-    /// The pointer uniquely identifies the module within the current Salsa revision,
-    /// regardless of whether particular [`ParsedModuleRef`] instances are garbage collected.
-    pub fn addr(&self) -> usize {
-        // Note that the outer `Arc` in `inner` is stable across garbage collection, while the inner
-        // `Arc` within the `ArcSwap` may change.
-        Arc::as_ptr(&self.inner).addr()
+    /// Returns the file to which this module belongs.
+    pub fn file(&self) -> File {
+        self.file
     }
 }
 

crates/ruff_db/src/system/path.rs

@@ -667,6 +667,13 @@ impl Deref for SystemPathBuf {
     }
 }
 
+impl AsRef<Path> for SystemPathBuf {
+    #[inline]
+    fn as_ref(&self) -> &Path {
+        self.0.as_std_path()
+    }
+}
+
 impl<P: AsRef<SystemPath>> FromIterator<P> for SystemPathBuf {
     fn from_iter<I: IntoIterator<Item = P>>(iter: I) -> Self {
         let mut buf = SystemPathBuf::new();

crates/ruff_graph/src/lib.rs

@@ -49,7 +49,7 @@ impl ModuleImports {
         // Resolve the imports.
         let mut resolved_imports = ModuleImports::default();
         for import in imports {
-            for resolved in Resolver::new(db).resolve(import) {
+            for resolved in Resolver::new(db, path).resolve(import) {
                 if let Some(path) = resolved.as_system_path() {
                     resolved_imports.insert(path.to_path_buf());
                 }

crates/ruff_graph/src/resolver.rs

@@ -1,5 +1,9 @@
-use ruff_db::files::FilePath;
-use ty_python_semantic::{ModuleName, resolve_module, resolve_real_module};
+use ruff_db::files::{File, FilePath, system_path_to_file};
+use ruff_db::system::SystemPath;
+use ty_python_semantic::{
+    ModuleName, resolve_module, resolve_module_confident, resolve_real_module,
+    resolve_real_module_confident,
+};
 
 use crate::ModuleDb;
 use crate::collector::CollectedImport;
@@ -7,12 +11,15 @@ use crate::collector::CollectedImport;
 /// Collect all imports for a given Python file.
 pub(crate) struct Resolver<'a> {
     db: &'a ModuleDb,
+    file: Option<File>,
 }
 
 impl<'a> Resolver<'a> {
     /// Initialize a [`Resolver`] with a given [`ModuleDb`].
-    pub(crate) fn new(db: &'a ModuleDb) -> Self {
-        Self { db }
+    pub(crate) fn new(db: &'a ModuleDb, path: &SystemPath) -> Self {
+        // If we know the importing file we can potentially resolve more imports
+        let file = system_path_to_file(db, path).ok();
+        Self { db, file }
     }
 
     /// Resolve the [`CollectedImport`] into a [`FilePath`].
@@ -70,13 +77,21 @@ impl<'a> Resolver<'a> {
 
     /// Resolves a module name to a module.
     pub(crate) fn resolve_module(&self, module_name: &ModuleName) -> Option<&'a FilePath> {
-        let module = resolve_module(self.db, module_name)?;
+        let module = if let Some(file) = self.file {
+            resolve_module(self.db, file, module_name)?
+        } else {
+            resolve_module_confident(self.db, module_name)?
+        };
         Some(module.file(self.db)?.path(self.db))
     }
 
     /// Resolves a module name to a module (stubs not allowed).
     fn resolve_real_module(&self, module_name: &ModuleName) -> Option<&'a FilePath> {
-        let module = resolve_real_module(self.db, module_name)?;
+        let module = if let Some(file) = self.file {
+            resolve_real_module(self.db, file, module_name)?
+        } else {
+            resolve_real_module_confident(self.db, module_name)?
+        };
         Some(module.file(self.db)?.path(self.db))
     }
 }

crates/ruff_linter/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "ruff_linter"
-version = "0.14.7"
+version = "0.14.8"
 publish = false
 authors = { workspace = true }
 edition = { workspace = true }
@@ -35,6 +35,7 @@ anyhow = { workspace = true }
 bitflags = { workspace = true }
 clap = { workspace = true, features = ["derive", "string"], optional = true }
 colored = { workspace = true }
+compact_str = { workspace = true }
 fern = { workspace = true }
 glob = { workspace = true }
 globset = { workspace = true }

crates/ruff_linter/resources/test/fixtures/flake8_bandit/S506.py

@@ -28,9 +28,11 @@ yaml.load("{}", SafeLoader)
 yaml.load("{}", yaml.SafeLoader)
 yaml.load("{}", CSafeLoader)
 yaml.load("{}", yaml.CSafeLoader)
+yaml.load("{}", yaml.cyaml.CSafeLoader)
 yaml.load("{}", NewSafeLoader)
 yaml.load("{}", Loader=SafeLoader)
 yaml.load("{}", Loader=yaml.SafeLoader)
 yaml.load("{}", Loader=CSafeLoader)
 yaml.load("{}", Loader=yaml.CSafeLoader)
+yaml.load("{}", Loader=yaml.cyaml.CSafeLoader)
 yaml.load("{}", Loader=NewSafeLoader)

crates/ruff_linter/resources/test/fixtures/flake8_bugbear/B901.py

@@ -52,16 +52,16 @@ def not_broken5():
     yield inner()
 
 
-def not_broken6():
+def broken3():
     return (yield from [])
 
 
-def not_broken7():
+def broken4():
     x = yield from []
     return x
 
 
-def not_broken8():
+def broken5():
     x = None
 
     def inner(ex):
@@ -76,3 +76,13 @@ class NotBroken9(object):
     def __await__(self):
         yield from function()
         return 42
+
+
+async def broken6():
+    yield 1
+    return foo()
+
+
+async def broken7():
+    yield 1
+    return [1, 2, 3]

crates/ruff_linter/resources/test/fixtures/pyflakes/F704.py

@@ -17,3 +17,24 @@ def _():
 
     # Valid yield scope
     yield 3
+
+
+# await is valid in any generator, sync or async
+(await cor async for cor in f())  # ok
+(await cor for cor in f())  # ok
+
+# but not in comprehensions
+[await cor async for cor in f()]  # F704
+{await cor async for cor in f()}  # F704
+{await cor: 1 async for cor in f()}  # F704
+[await cor for cor in f()]  # F704
+{await cor for cor in f()}  # F704
+{await cor: 1 for cor in f()}  # F704
+
+# or in the iterator of an async generator, which is evaluated in the parent
+# scope
+(cor async for cor in await f())  # F704
+(await cor async for cor in [await c for c in f()])  # F704
+
+# this is also okay because the comprehension is within the generator scope
+([await c for c in cor] async for cor in f())  # ok

crates/ruff_linter/resources/test/fixtures/syntax_errors/await_outside_async_function.py

@@ -3,3 +3,5 @@ def func():
 
 # Top-level await
 await 1
+
+([await c for c in cor] async for cor in func())  # ok

crates/ruff_linter/resources/test/fixtures/syntax_errors/return_in_generator.py

@@ -0,0 +1,24 @@
+async def gen():
+    yield 1
+    return 42
+
+def gen(): # B901 but not a syntax error - not an async generator
+    yield 1
+    return 42
+
+async def gen(): # ok - no value in return
+    yield 1
+    return
+
+async def gen():
+    yield 1
+    return foo()
+
+async def gen():
+    yield 1
+    return [1, 2, 3]
+
+async def gen():
+    if True:
+        yield 1
+    return 10

crates/ruff_linter/src/checkers/ast/mod.rs

@@ -35,6 +35,7 @@ use ruff_python_ast::helpers::{collect_import_from_member, is_docstring_stmt, to
 use ruff_python_ast::identifier::Identifier;
 use ruff_python_ast::name::QualifiedName;
 use ruff_python_ast::str::Quote;
+use ruff_python_ast::token::Tokens;
 use ruff_python_ast::visitor::{Visitor, walk_except_handler, walk_pattern};
 use ruff_python_ast::{
     self as ast, AnyParameterRef, ArgOrKeyword, Comprehension, ElifElseClause, ExceptHandler, Expr,
@@ -48,7 +49,7 @@ use ruff_python_parser::semantic_errors::{
     SemanticSyntaxChecker, SemanticSyntaxContext, SemanticSyntaxError, SemanticSyntaxErrorKind,
 };
 use ruff_python_parser::typing::{AnnotationKind, ParsedAnnotation, parse_type_annotation};
-use ruff_python_parser::{ParseError, Parsed, Tokens};
+use ruff_python_parser::{ParseError, Parsed};
 use ruff_python_semantic::all::{DunderAllDefinition, DunderAllFlags};
 use ruff_python_semantic::analyze::{imports, typing};
 use ruff_python_semantic::{
@@ -68,6 +69,7 @@ use crate::noqa::NoqaMapping;
 use crate::package::PackageRoot;
 use crate::preview::is_undefined_export_in_dunder_init_enabled;
 use crate::registry::Rule;
+use crate::rules::flake8_bugbear::rules::ReturnInGenerator;
 use crate::rules::pyflakes::rules::{
     LateFutureImport, MultipleStarredExpressions, ReturnOutsideFunction,
     UndefinedLocalWithNestedImportStarUsage, YieldOutsideFunction,
@@ -728,6 +730,12 @@ impl SemanticSyntaxContext for Checker<'_> {
                     self.report_diagnostic(NonlocalWithoutBinding { name }, error.range);
                 }
             }
+            SemanticSyntaxErrorKind::ReturnInGenerator => {
+                // B901
+                if self.is_rule_enabled(Rule::ReturnInGenerator) {
+                    self.report_diagnostic(ReturnInGenerator, error.range);
+                }
+            }
             SemanticSyntaxErrorKind::ReboundComprehensionVariable
             | SemanticSyntaxErrorKind::DuplicateTypeParameter
             | SemanticSyntaxErrorKind::MultipleCaseAssignment(_)
@@ -746,6 +754,7 @@ impl SemanticSyntaxContext for Checker<'_> {
             | SemanticSyntaxErrorKind::LoadBeforeNonlocalDeclaration { .. }
             | SemanticSyntaxErrorKind::NonlocalAndGlobal(_)
             | SemanticSyntaxErrorKind::AnnotatedGlobal(_)
+            | SemanticSyntaxErrorKind::TypeParameterDefaultOrder(_)
             | SemanticSyntaxErrorKind::AnnotatedNonlocal(_) => {
                 self.semantic_errors.borrow_mut().push(error);
             }
@@ -779,6 +788,10 @@ impl SemanticSyntaxContext for Checker<'_> {
             match scope.kind {
                 ScopeKind::Class(_) => return false,
                 ScopeKind::Function(_) | ScopeKind::Lambda(_) => return true,
+                ScopeKind::Generator {
+                    kind: GeneratorKind::Generator,
+                    ..
+                } => return true,
                 ScopeKind::Generator { .. }
                 | ScopeKind::Module
                 | ScopeKind::Type
@@ -828,14 +841,19 @@ impl SemanticSyntaxContext for Checker<'_> {
         self.source_type.is_ipynb()
     }
 
-    fn in_generator_scope(&self) -> bool {
-        matches!(
-            &self.semantic.current_scope().kind,
-            ScopeKind::Generator {
-                kind: GeneratorKind::Generator,
-                ..
+    fn in_generator_context(&self) -> bool {
+        for scope in self.semantic.current_scopes() {
+            if matches!(
+                scope.kind,
+                ScopeKind::Generator {
+                    kind: GeneratorKind::Generator,
+                    ..
+                }
+            ) {
+                return true;
             }
-        )
+        }
+        false
     }
 
     fn in_loop_context(&self) -> bool {

crates/ruff_linter/src/checkers/logical_lines.rs

@@ -1,6 +1,6 @@
+use ruff_python_ast::token::{TokenKind, Tokens};
 use ruff_python_codegen::Stylist;
 use ruff_python_index::Indexer;
-use ruff_python_parser::{TokenKind, Tokens};
 use ruff_source_file::LineRanges;
 use ruff_text_size::{Ranged, TextRange};
 

crates/ruff_linter/src/checkers/tokens.rs

@@ -4,9 +4,9 @@ use std::path::Path;
 
 use ruff_notebook::CellOffsets;
 use ruff_python_ast::PySourceType;
+use ruff_python_ast::token::Tokens;
 use ruff_python_codegen::Stylist;
 use ruff_python_index::Indexer;
-use ruff_python_parser::Tokens;
 
 use crate::Locator;
 use crate::directives::TodoComment;

crates/ruff_linter/src/directives.rs

@@ -5,8 +5,8 @@ use std::str::FromStr;
 
 use bitflags::bitflags;
 
+use ruff_python_ast::token::{TokenKind, Tokens};
 use ruff_python_index::Indexer;
-use ruff_python_parser::{TokenKind, Tokens};
 use ruff_python_trivia::CommentRanges;
 use ruff_source_file::LineRanges;
 use ruff_text_size::{Ranged, TextLen, TextRange, TextSize};

crates/ruff_linter/src/doc_lines.rs

@@ -5,8 +5,8 @@ use std::iter::FusedIterator;
 use std::slice::Iter;
 
 use ruff_python_ast::statement_visitor::{StatementVisitor, walk_stmt};
+use ruff_python_ast::token::{Token, TokenKind, Tokens};
 use ruff_python_ast::{self as ast, Stmt, Suite};
-use ruff_python_parser::{Token, TokenKind, Tokens};
 use ruff_source_file::UniversalNewlineIterator;
 use ruff_text_size::{Ranged, TextSize};
 

crates/ruff_linter/src/importer/mod.rs

@@ -9,10 +9,11 @@ use anyhow::Result;
 use libcst_native as cst;
 
 use ruff_diagnostics::Edit;
+use ruff_python_ast::token::Tokens;
 use ruff_python_ast::{self as ast, Expr, ModModule, Stmt};
 use ruff_python_codegen::Stylist;
 use ruff_python_importer::Insertion;
-use ruff_python_parser::{Parsed, Tokens};
+use ruff_python_parser::Parsed;
 use ruff_python_semantic::{
     ImportedName, MemberNameImport, ModuleNameImport, NameImport, SemanticModel,
 };

crates/ruff_linter/src/lib.rs

@@ -46,6 +46,7 @@ pub mod rule_selector;
 pub mod rules;
 pub mod settings;
 pub mod source_kind;
+pub mod suppression;
 mod text_helpers;
 pub mod upstream_categories;
 mod violation;

crates/ruff_linter/src/linter.rs

@@ -1043,6 +1043,7 @@ mod tests {
         Rule::YieldFromInAsyncFunction,
         Path::new("yield_from_in_async_function.py")
     )]
+    #[test_case(Rule::ReturnInGenerator, Path::new("return_in_generator.py"))]
     fn test_syntax_errors(rule: Rule, path: &Path) -> Result<()> {
         let snapshot = path.to_string_lossy().to_string();
         let path = Path::new("resources/test/fixtures/syntax_errors").join(path);

crates/ruff_linter/src/rules/flake8_bandit/rules/unsafe_yaml_load.rs

@@ -75,6 +75,7 @@ pub(crate) fn unsafe_yaml_load(checker: &Checker, call: &ast::ExprCall) {
                         qualified_name.segments(),
                         ["yaml", "SafeLoader" | "CSafeLoader"]
                             | ["yaml", "loader", "SafeLoader" | "CSafeLoader"]
+                            | ["yaml", "cyaml", "CSafeLoader"]
                     )
                 })
             {

crates/ruff_linter/src/rules/flake8_bugbear/rules/return_in_generator.rs

@@ -1,6 +1,5 @@
 use ruff_macros::{ViolationMetadata, derive_message_formats};
-use ruff_python_ast::statement_visitor;
-use ruff_python_ast::statement_visitor::StatementVisitor;
+use ruff_python_ast::visitor::{Visitor, walk_expr, walk_stmt};
 use ruff_python_ast::{self as ast, Expr, Stmt, StmtFunctionDef};
 use ruff_text_size::TextRange;
 
@@ -96,6 +95,11 @@ pub(crate) fn return_in_generator(checker: &Checker, function_def: &StmtFunction
         return;
     }
 
+    // Async functions are flagged by the `ReturnInGenerator` semantic syntax error.
+    if function_def.is_async {
+        return;
+    }
+
     let mut visitor = ReturnInGeneratorVisitor::default();
     visitor.visit_body(&function_def.body);
 
@@ -112,15 +116,9 @@ struct ReturnInGeneratorVisitor {
     has_yield: bool,
 }
 
-impl StatementVisitor<'_> for ReturnInGeneratorVisitor {
+impl Visitor<'_> for ReturnInGeneratorVisitor {
     fn visit_stmt(&mut self, stmt: &Stmt) {
         match stmt {
-            Stmt::Expr(ast::StmtExpr { value, .. }) => match **value {
-                Expr::Yield(_) | Expr::YieldFrom(_) => {
-                    self.has_yield = true;
-                }
-                _ => {}
-            },
             Stmt::FunctionDef(_) => {
                 // Do not recurse into nested functions; they're evaluated separately.
             }
@@ -130,8 +128,19 @@ impl StatementVisitor<'_> for ReturnInGeneratorVisitor {
                 node_index: _,
             }) => {
                 self.return_ = Some(*range);
+                walk_stmt(self, stmt);
             }
-            _ => statement_visitor::walk_stmt(self, stmt),
+            _ => walk_stmt(self, stmt),
+        }
+    }
+
+    fn visit_expr(&mut self, expr: &Expr) {
+        match expr {
+            Expr::Lambda(_) => {}
+            Expr::Yield(_) | Expr::YieldFrom(_) => {
+                self.has_yield = true;
+            }
+            _ => walk_expr(self, expr),
         }
     }
 }

crates/ruff_linter/src/rules/flake8_bugbear/snapshots/ruff_linter__rules__flake8_bugbear__tests__B901_B901.py.snap

@@ -21,3 +21,46 @@ B901 Using `yield` and `return {value}` in a generator function can lead to conf
 37 |
 38 |     yield from not_broken()
    |
+
+B901 Using `yield` and `return {value}` in a generator function can lead to confusing behavior
+  --> B901.py:56:5
+   |
+55 | def broken3():
+56 |     return (yield from [])
+   |     ^^^^^^^^^^^^^^^^^^^^^^
+   |
+
+B901 Using `yield` and `return {value}` in a generator function can lead to confusing behavior
+  --> B901.py:61:5
+   |
+59 | def broken4():
+60 |     x = yield from []
+61 |     return x
+   |     ^^^^^^^^
+   |
+
+B901 Using `yield` and `return {value}` in a generator function can lead to confusing behavior
+  --> B901.py:72:5
+   |
+71 |     inner((yield from []))
+72 |     return x
+   |     ^^^^^^^^
+   |
+
+B901 Using `yield` and `return {value}` in a generator function can lead to confusing behavior
+  --> B901.py:83:5
+   |
+81 | async def broken6():
+82 |     yield 1
+83 |     return foo()
+   |     ^^^^^^^^^^^^
+   |
+
+B901 Using `yield` and `return {value}` in a generator function can lead to confusing behavior
+  --> B901.py:88:5
+   |
+86 | async def broken7():
+87 |     yield 1
+88 |     return [1, 2, 3]
+   |     ^^^^^^^^^^^^^^^^
+   |

crates/ruff_linter/src/rules/flake8_commas/rules/trailing_commas.rs

@@ -1,6 +1,6 @@
 use ruff_macros::{ViolationMetadata, derive_message_formats};
+use ruff_python_ast::token::{TokenKind, Tokens};
 use ruff_python_index::Indexer;
-use ruff_python_parser::{TokenKind, Tokens};
 use ruff_text_size::{Ranged, TextRange};
 
 use crate::Locator;

crates/ruff_linter/src/rules/flake8_comprehensions/rules/unnecessary_generator_list.rs

@@ -3,7 +3,7 @@ use ruff_python_ast as ast;
 use ruff_python_ast::ExprGenerator;
 use ruff_python_ast::comparable::ComparableExpr;
 use ruff_python_ast::parenthesize::parenthesized_range;
-use ruff_python_parser::TokenKind;
+use ruff_python_ast::token::TokenKind;
 use ruff_text_size::{Ranged, TextRange, TextSize};
 
 use crate::checkers::ast::Checker;

crates/ruff_linter/src/rules/flake8_comprehensions/rules/unnecessary_generator_set.rs

@@ -3,7 +3,7 @@ use ruff_python_ast as ast;
 use ruff_python_ast::ExprGenerator;
 use ruff_python_ast::comparable::ComparableExpr;
 use ruff_python_ast::parenthesize::parenthesized_range;
-use ruff_python_parser::TokenKind;
+use ruff_python_ast::token::TokenKind;
 use ruff_text_size::{Ranged, TextRange, TextSize};
 
 use crate::checkers::ast::Checker;

crates/ruff_linter/src/rules/flake8_comprehensions/rules/unnecessary_list_comprehension_set.rs

@@ -1,7 +1,7 @@
 use ruff_macros::{ViolationMetadata, derive_message_formats};
 use ruff_python_ast as ast;
 use ruff_python_ast::parenthesize::parenthesized_range;
-use ruff_python_parser::TokenKind;
+use ruff_python_ast::token::TokenKind;
 use ruff_text_size::{Ranged, TextRange, TextSize};
 
 use crate::checkers::ast::Checker;

crates/ruff_linter/src/rules/flake8_implicit_str_concat/rules/implicit.rs

@@ -3,8 +3,8 @@ use std::borrow::Cow;
 use itertools::Itertools;
 use ruff_macros::{ViolationMetadata, derive_message_formats};
 use ruff_python_ast::StringFlags;
+use ruff_python_ast::token::{Token, TokenKind, Tokens};
 use ruff_python_index::Indexer;
-use ruff_python_parser::{Token, TokenKind, Tokens};
 use ruff_source_file::LineRanges;
 use ruff_text_size::{Ranged, TextLen, TextRange};
 

crates/ruff_linter/src/rules/flake8_pie/rules/unnecessary_spread.rs

@@ -1,6 +1,6 @@
 use ruff_macros::{ViolationMetadata, derive_message_formats};
+use ruff_python_ast::token::{TokenKind, Tokens};
 use ruff_python_ast::{self as ast, Expr};
-use ruff_python_parser::{TokenKind, Tokens};
 use ruff_text_size::{Ranged, TextLen, TextSize};
 
 use crate::checkers::ast::Checker;

crates/ruff_linter/src/rules/flake8_return/rules/function.rs

@@ -4,10 +4,10 @@ use ruff_diagnostics::Applicability;
 use ruff_macros::{ViolationMetadata, derive_message_formats};
 use ruff_python_ast::helpers::{is_const_false, is_const_true};
 use ruff_python_ast::stmt_if::elif_else_range;
+use ruff_python_ast::token::TokenKind;
 use ruff_python_ast::visitor::Visitor;
 use ruff_python_ast::whitespace::indentation;
 use ruff_python_ast::{self as ast, Decorator, ElifElseClause, Expr, Stmt};
-use ruff_python_parser::TokenKind;
 use ruff_python_semantic::SemanticModel;
 use ruff_python_semantic::analyze::visibility::is_property;
 use ruff_python_trivia::{SimpleTokenKind, SimpleTokenizer, is_python_whitespace};

crates/ruff_linter/src/rules/isort/annotate.rs

@@ -1,5 +1,5 @@
+use ruff_python_ast::token::Tokens;
 use ruff_python_ast::{self as ast, Stmt};
-use ruff_python_parser::Tokens;
 use ruff_source_file::LineRanges;
 use ruff_text_size::{Ranged, TextRange};
 

crates/ruff_linter/src/rules/isort/helpers.rs

@@ -1,5 +1,5 @@
 use ruff_python_ast::Stmt;
-use ruff_python_parser::{TokenKind, Tokens};
+use ruff_python_ast::token::{TokenKind, Tokens};
 use ruff_python_trivia::PythonWhitespace;
 use ruff_source_file::UniversalNewlines;
 use ruff_text_size::Ranged;

crates/ruff_linter/src/rules/isort/mod.rs

@@ -11,8 +11,8 @@ use comments::Comment;
 use normalize::normalize_imports;
 use order::order_imports;
 use ruff_python_ast::PySourceType;
+use ruff_python_ast::token::Tokens;
 use ruff_python_codegen::Stylist;
-use ruff_python_parser::Tokens;
 use settings::Settings;
 use types::EitherImport::{Import, ImportFrom};
 use types::{AliasData, ImportBlock, TrailingComma};

crates/ruff_linter/src/rules/isort/rules/organize_imports.rs

@@ -1,11 +1,11 @@
 use itertools::{EitherOrBoth, Itertools};
 
 use ruff_macros::{ViolationMetadata, derive_message_formats};
+use ruff_python_ast::token::Tokens;
 use ruff_python_ast::whitespace::trailing_lines_end;
 use ruff_python_ast::{PySourceType, PythonVersion, Stmt};
 use ruff_python_codegen::Stylist;
 use ruff_python_index::Indexer;
-use ruff_python_parser::Tokens;
 use ruff_python_trivia::{PythonWhitespace, leading_indentation, textwrap::indent};
 use ruff_source_file::{LineRanges, UniversalNewlines};
 use ruff_text_size::{Ranged, TextRange};

crates/ruff_linter/src/rules/pycodestyle/helpers.rs

@@ -1,4 +1,4 @@
-use ruff_python_parser::TokenKind;
+use ruff_python_ast::token::TokenKind;
 
 /// Returns `true` if the name should be considered "ambiguous".
 pub(super) fn is_ambiguous_name(name: &str) -> bool {

crates/ruff_linter/src/rules/pycodestyle/rules/blank_lines.rs

@@ -8,10 +8,10 @@ use itertools::Itertools;
 use ruff_macros::{ViolationMetadata, derive_message_formats};
 use ruff_notebook::CellOffsets;
 use ruff_python_ast::PySourceType;
+use ruff_python_ast::token::TokenIterWithContext;
+use ruff_python_ast::token::TokenKind;
+use ruff_python_ast::token::Tokens;
 use ruff_python_codegen::Stylist;
-use ruff_python_parser::TokenIterWithContext;
-use ruff_python_parser::TokenKind;
-use ruff_python_parser::Tokens;
 use ruff_python_trivia::PythonWhitespace;
 use ruff_source_file::{LineRanges, UniversalNewlines};
 use ruff_text_size::TextRange;

crates/ruff_linter/src/rules/pycodestyle/rules/compound_statements.rs

@@ -1,8 +1,8 @@
 use ruff_macros::{ViolationMetadata, derive_message_formats};
 use ruff_notebook::CellOffsets;
 use ruff_python_ast::PySourceType;
+use ruff_python_ast::token::{TokenIterWithContext, TokenKind, Tokens};
 use ruff_python_index::Indexer;
-use ruff_python_parser::{TokenIterWithContext, TokenKind, Tokens};
 use ruff_text_size::{Ranged, TextSize};
 
 use crate::Locator;

crates/ruff_linter/src/rules/pycodestyle/rules/logical_lines/extraneous_whitespace.rs

@@ -1,5 +1,5 @@
 use ruff_macros::{ViolationMetadata, derive_message_formats};
-use ruff_python_parser::TokenKind;
+use ruff_python_ast::token::TokenKind;
 use ruff_text_size::{Ranged, TextRange};
 
 use crate::AlwaysFixableViolation;

crates/ruff_linter/src/rules/pycodestyle/rules/logical_lines/indentation.rs

@@ -1,5 +1,5 @@
 use ruff_macros::{ViolationMetadata, derive_message_formats};
-use ruff_python_parser::TokenKind;
+use ruff_python_ast::token::TokenKind;
 use ruff_text_size::TextRange;
 
 use crate::Violation;

crates/ruff_linter/src/rules/pycodestyle/rules/logical_lines/missing_whitespace.rs

@@ -1,5 +1,5 @@
 use ruff_macros::{ViolationMetadata, derive_message_formats};
-use ruff_python_parser::TokenKind;
+use ruff_python_ast::token::TokenKind;
 use ruff_text_size::Ranged;
 
 use crate::Edit;

crates/ruff_linter/src/rules/pycodestyle/rules/logical_lines/missing_whitespace_after_keyword.rs

@@ -1,5 +1,5 @@
 use ruff_macros::{ViolationMetadata, derive_message_formats};
-use ruff_python_parser::TokenKind;
+use ruff_python_ast::token::TokenKind;
 use ruff_text_size::Ranged;
 
 use crate::checkers::ast::LintContext;

crates/ruff_linter/src/rules/pycodestyle/rules/logical_lines/missing_whitespace_around_operator.rs

@@ -1,5 +1,5 @@
 use ruff_macros::{ViolationMetadata, derive_message_formats};
-use ruff_python_parser::TokenKind;
+use ruff_python_ast::token::TokenKind;
 use ruff_text_size::{Ranged, TextRange};
 
 use crate::checkers::ast::LintContext;

crates/ruff_linter/src/rules/pycodestyle/rules/logical_lines/mod.rs

@@ -9,7 +9,7 @@ pub(crate) use missing_whitespace::*;
 pub(crate) use missing_whitespace_after_keyword::*;
 pub(crate) use missing_whitespace_around_operator::*;
 pub(crate) use redundant_backslash::*;
-use ruff_python_parser::{TokenKind, Tokens};
+use ruff_python_ast::token::{TokenKind, Tokens};
 use ruff_python_trivia::is_python_whitespace;
 use ruff_text_size::{Ranged, TextLen, TextRange, TextSize};
 pub(crate) use space_around_operator::*;

crates/ruff_linter/src/rules/pycodestyle/rules/logical_lines/redundant_backslash.rs

@@ -1,6 +1,6 @@
 use ruff_macros::{ViolationMetadata, derive_message_formats};
+use ruff_python_ast::token::TokenKind;
 use ruff_python_index::Indexer;
-use ruff_python_parser::TokenKind;
 use ruff_source_file::LineRanges;
 use ruff_text_size::{Ranged, TextRange, TextSize};
 

crates/ruff_linter/src/rules/pycodestyle/rules/logical_lines/space_around_operator.rs

@@ -1,5 +1,5 @@
 use ruff_macros::{ViolationMetadata, derive_message_formats};
-use ruff_python_parser::TokenKind;
+use ruff_python_ast::token::TokenKind;
 use ruff_text_size::{Ranged, TextRange};
 
 use crate::checkers::ast::LintContext;

crates/ruff_linter/src/rules/pycodestyle/rules/logical_lines/whitespace_around_named_parameter_equals.rs

@@ -1,5 +1,5 @@
 use ruff_macros::{ViolationMetadata, derive_message_formats};
-use ruff_python_parser::TokenKind;
+use ruff_python_ast::token::TokenKind;
 use ruff_text_size::{Ranged, TextRange, TextSize};
 
 use crate::checkers::ast::LintContext;

crates/ruff_linter/src/rules/pycodestyle/rules/logical_lines/whitespace_before_comment.rs

@@ -1,5 +1,5 @@
 use ruff_macros::{ViolationMetadata, derive_message_formats};
-use ruff_python_parser::TokenKind;
+use ruff_python_ast::token::TokenKind;
 use ruff_python_trivia::PythonWhitespace;
 use ruff_source_file::LineRanges;
 use ruff_text_size::{Ranged, TextLen, TextRange, TextSize};

crates/ruff_linter/src/rules/pycodestyle/rules/logical_lines/whitespace_before_parameters.rs

@@ -1,5 +1,5 @@
 use ruff_macros::{ViolationMetadata, derive_message_formats};
-use ruff_python_parser::TokenKind;
+use ruff_python_ast::token::TokenKind;
 use ruff_text_size::{Ranged, TextRange, TextSize};
 
 use crate::checkers::ast::LintContext;

crates/ruff_linter/src/rules/pycodestyle/rules/too_many_newlines_at_end_of_file.rs

@@ -3,7 +3,7 @@ use std::iter::Peekable;
 use itertools::Itertools;
 use ruff_macros::{ViolationMetadata, derive_message_formats};
 use ruff_notebook::CellOffsets;
-use ruff_python_parser::{Token, TokenKind, Tokens};
+use ruff_python_ast::token::{Token, TokenKind, Tokens};
 use ruff_text_size::{Ranged, TextRange, TextSize};
 
 use crate::{AlwaysFixableViolation, Edit, Fix, checkers::ast::LintContext};

crates/ruff_linter/src/rules/pyflakes/rules/invalid_literal_comparisons.rs

@@ -2,8 +2,8 @@ use anyhow::{Error, bail};
 
 use ruff_macros::{ViolationMetadata, derive_message_formats};
 use ruff_python_ast::helpers;
+use ruff_python_ast::token::{TokenKind, Tokens};
 use ruff_python_ast::{CmpOp, Expr};
-use ruff_python_parser::{TokenKind, Tokens};
 use ruff_text_size::{Ranged, TextRange};
 
 use crate::checkers::ast::Checker;

crates/ruff_linter/src/rules/pyflakes/rules/unused_variable.rs

@@ -3,8 +3,8 @@ use itertools::Itertools;
 use ruff_macros::{ViolationMetadata, derive_message_formats};
 use ruff_python_ast::helpers::contains_effect;
 use ruff_python_ast::parenthesize::parenthesized_range;
+use ruff_python_ast::token::{TokenKind, Tokens};
 use ruff_python_ast::{self as ast, Stmt};
-use ruff_python_parser::{TokenKind, Tokens};
 use ruff_python_semantic::Binding;
 use ruff_text_size::{Ranged, TextRange, TextSize};
 

crates/ruff_linter/src/rules/pyflakes/snapshots/ruff_linter__rules__pyflakes__tests__F704_F704.py.snap

@@ -37,3 +37,88 @@ F704 `await` statement outside of a function
 12 |
 13 | def _():
    |
+
+F704 `await` statement outside of a function
+  --> F704.py:27:2
+   |
+26 | # but not in comprehensions
+27 | [await cor async for cor in f()]  # F704
+   |  ^^^^^^^^^
+28 | {await cor async for cor in f()}  # F704
+29 | {await cor: 1 async for cor in f()}  # F704
+   |
+
+F704 `await` statement outside of a function
+  --> F704.py:28:2
+   |
+26 | # but not in comprehensions
+27 | [await cor async for cor in f()]  # F704
+28 | {await cor async for cor in f()}  # F704
+   |  ^^^^^^^^^
+29 | {await cor: 1 async for cor in f()}  # F704
+30 | [await cor for cor in f()]  # F704
+   |
+
+F704 `await` statement outside of a function
+  --> F704.py:29:2
+   |
+27 | [await cor async for cor in f()]  # F704
+28 | {await cor async for cor in f()}  # F704
+29 | {await cor: 1 async for cor in f()}  # F704
+   |  ^^^^^^^^^
+30 | [await cor for cor in f()]  # F704
+31 | {await cor for cor in f()}  # F704
+   |
+
+F704 `await` statement outside of a function
+  --> F704.py:30:2
+   |
+28 | {await cor async for cor in f()}  # F704
+29 | {await cor: 1 async for cor in f()}  # F704
+30 | [await cor for cor in f()]  # F704
+   |  ^^^^^^^^^
+31 | {await cor for cor in f()}  # F704
+32 | {await cor: 1 for cor in f()}  # F704
+   |
+
+F704 `await` statement outside of a function
+  --> F704.py:31:2
+   |
+29 | {await cor: 1 async for cor in f()}  # F704
+30 | [await cor for cor in f()]  # F704
+31 | {await cor for cor in f()}  # F704
+   |  ^^^^^^^^^
+32 | {await cor: 1 for cor in f()}  # F704
+   |
+
+F704 `await` statement outside of a function
+  --> F704.py:32:2
+   |
+30 | [await cor for cor in f()]  # F704
+31 | {await cor for cor in f()}  # F704
+32 | {await cor: 1 for cor in f()}  # F704
+   |  ^^^^^^^^^
+33 |
+34 | # or in the iterator of an async generator, which is evaluated in the parent
+   |
+
+F704 `await` statement outside of a function
+  --> F704.py:36:23
+   |
+34 | # or in the iterator of an async generator, which is evaluated in the parent
+35 | # scope
+36 | (cor async for cor in await f())  # F704
+   |                       ^^^^^^^^^
+37 | (await cor async for cor in [await c for c in f()])  # F704
+   |
+
+F704 `await` statement outside of a function
+  --> F704.py:37:30
+   |
+35 | # scope
+36 | (cor async for cor in await f())  # F704
+37 | (await cor async for cor in [await c for c in f()])  # F704
+   |                              ^^^^^^^
+38 |
+39 | # this is also okay because the comprehension is within the generator scope
+   |

crates/ruff_linter/src/rules/pylint/rules/invalid_string_characters.rs

@@ -1,5 +1,5 @@
 use ruff_macros::{ViolationMetadata, derive_message_formats};
-use ruff_python_parser::{Token, TokenKind};
+use ruff_python_ast::token::{Token, TokenKind};
 use ruff_text_size::{Ranged, TextLen, TextRange, TextSize};
 
 use crate::Locator;

crates/ruff_linter/src/rules/pyupgrade/fixes.rs

@@ -1,5 +1,5 @@
 use ruff_python_ast::StmtImportFrom;
-use ruff_python_parser::{TokenKind, Tokens};
+use ruff_python_ast::token::{TokenKind, Tokens};
 use ruff_text_size::{Ranged, TextRange};
 
 use crate::Locator;

crates/ruff_linter/src/rules/pyupgrade/rules/deprecated_import.rs

@@ -1,10 +1,10 @@
 use itertools::Itertools;
 
 use ruff_macros::{ViolationMetadata, derive_message_formats};
+use ruff_python_ast::token::Tokens;
 use ruff_python_ast::whitespace::indentation;
 use ruff_python_ast::{Alias, StmtImportFrom, StmtRef};
 use ruff_python_codegen::Stylist;
-use ruff_python_parser::Tokens;
 use ruff_text_size::Ranged;
 
 use crate::Locator;

crates/ruff_linter/src/rules/pyupgrade/rules/extraneous_parentheses.rs

@@ -1,7 +1,7 @@
 use std::slice::Iter;
 
 use ruff_macros::{ViolationMetadata, derive_message_formats};
-use ruff_python_parser::{Token, TokenKind, Tokens};
+use ruff_python_ast::token::{Token, TokenKind, Tokens};
 use ruff_text_size::{Ranged, TextRange};
 
 use crate::Locator;

crates/ruff_linter/src/rules/pyupgrade/rules/f_strings.rs

@@ -6,11 +6,11 @@ use rustc_hash::{FxHashMap, FxHashSet};
 use ruff_macros::{ViolationMetadata, derive_message_formats};
 use ruff_python_ast::helpers::any_over_expr;
 use ruff_python_ast::str::{leading_quote, trailing_quote};
+use ruff_python_ast::token::TokenKind;
 use ruff_python_ast::{self as ast, Expr, Keyword, StringFlags};
 use ruff_python_literal::format::{
     FieldName, FieldNamePart, FieldType, FormatPart, FormatString, FromTemplate,
 };
-use ruff_python_parser::TokenKind;
 use ruff_source_file::LineRanges;
 use ruff_text_size::{Ranged, TextRange};
 

crates/ruff_linter/src/rules/pyupgrade/rules/printf_string_formatting.rs

@@ -3,12 +3,12 @@ use std::fmt::Write;
 use std::str::FromStr;
 
 use ruff_macros::{ViolationMetadata, derive_message_formats};
+use ruff_python_ast::token::TokenKind;
 use ruff_python_ast::{self as ast, AnyStringFlags, Expr, StringFlags, whitespace::indentation};
 use ruff_python_codegen::Stylist;
 use ruff_python_literal::cformat::{
     CConversionFlags, CFormatPart, CFormatPrecision, CFormatQuantity, CFormatString,
 };
-use ruff_python_parser::TokenKind;
 use ruff_python_stdlib::identifiers::is_identifier;
 use ruff_source_file::LineRanges;
 use ruff_text_size::{Ranged, TextRange};

crates/ruff_linter/src/rules/pyupgrade/rules/quoted_annotation.rs

@@ -1,6 +1,6 @@
 use ruff_macros::{ViolationMetadata, derive_message_formats};
 use ruff_python_ast::Stmt;
-use ruff_python_parser::TokenKind;
+use ruff_python_ast::token::TokenKind;
 use ruff_python_semantic::SemanticModel;
 use ruff_source_file::LineRanges;
 use ruff_text_size::{TextLen, TextRange, TextSize};

crates/ruff_linter/src/rules/pyupgrade/rules/redundant_open_modes.rs

@@ -1,7 +1,7 @@
 use anyhow::Result;
 use ruff_macros::{ViolationMetadata, derive_message_formats};
+use ruff_python_ast::token::{TokenKind, Tokens};
 use ruff_python_ast::{self as ast, Expr};
-use ruff_python_parser::{TokenKind, Tokens};
 use ruff_python_stdlib::open_mode::OpenMode;
 use ruff_text_size::{Ranged, TextSize};
 

crates/ruff_linter/src/rules/pyupgrade/rules/unnecessary_encode_utf8.rs

@@ -1,8 +1,8 @@
 use std::fmt::Write as _;
 
 use ruff_macros::{ViolationMetadata, derive_message_formats};
+use ruff_python_ast::token::{TokenKind, Tokens};
 use ruff_python_ast::{self as ast, Arguments, Expr, Keyword};
-use ruff_python_parser::{TokenKind, Tokens};
 use ruff_text_size::{Ranged, TextRange};
 
 use crate::Locator;

crates/ruff_linter/src/rules/ruff/rules/explicit_f_string_type_conversion.rs

@@ -4,8 +4,8 @@ use anyhow::Result;
 
 use libcst_native::{LeftParen, ParenthesizedNode, RightParen};
 use ruff_macros::{ViolationMetadata, derive_message_formats};
+use ruff_python_ast::token::TokenKind;
 use ruff_python_ast::{self as ast, Expr, OperatorPrecedence};
-use ruff_python_parser::TokenKind;
 use ruff_text_size::Ranged;
 
 use crate::checkers::ast::Checker;

crates/ruff_linter/src/rules/ruff/rules/needless_else.rs

@@ -2,9 +2,9 @@ use std::cmp::Ordering;
 
 use ruff_macros::{ViolationMetadata, derive_message_formats};
 use ruff_python_ast::helpers::comment_indentation_after;
+use ruff_python_ast::token::{TokenKind, Tokens};
 use ruff_python_ast::whitespace::indentation;
 use ruff_python_ast::{Stmt, StmtExpr, StmtFor, StmtIf, StmtTry, StmtWhile};
-use ruff_python_parser::{TokenKind, Tokens};
 use ruff_source_file::LineRanges;
 use ruff_text_size::{Ranged, TextLen, TextRange, TextSize};
 

crates/ruff_linter/src/rules/ruff/rules/sequence_sorting.rs

@@ -9,8 +9,8 @@ use std::cmp::Ordering;
 use itertools::Itertools;
 
 use ruff_python_ast as ast;
+use ruff_python_ast::token::{TokenKind, Tokens};
 use ruff_python_codegen::Stylist;
-use ruff_python_parser::{TokenKind, Tokens};
 use ruff_python_stdlib::str::is_cased_uppercase;
 use ruff_python_trivia::{SimpleTokenKind, first_non_trivia_token, leading_indentation};
 use ruff_source_file::LineRanges;

crates/ruff_linter/src/rules/ruff/rules/starmap_zip.rs

@@ -1,7 +1,7 @@
 use ruff_macros::{ViolationMetadata, derive_message_formats};
 use ruff_python_ast::PythonVersion;
+use ruff_python_ast::token::TokenKind;
 use ruff_python_ast::{Expr, ExprCall, parenthesize::parenthesized_range};
-use ruff_python_parser::TokenKind;
 use ruff_text_size::{Ranged, TextRange};
 
 use crate::checkers::ast::Checker;

crates/ruff_linter/src/snapshots/ruff_linter__linter__tests__await_outside_async_function.py.snap

@@ -17,4 +17,6 @@ PLE1142 `await` should be used within an async function
 4 | # Top-level await
 5 | await 1
   | ^^^^^^^
+6 |
+7 | ([await c for c in cor] async for cor in func())  # ok
   |

crates/ruff_linter/src/snapshots/ruff_linter__linter__tests__return_in_generator.py.snap

@@ -0,0 +1,55 @@
+---
+source: crates/ruff_linter/src/linter.rs
+---
+B901 Using `yield` and `return {value}` in a generator function can lead to confusing behavior
+ --> resources/test/fixtures/syntax_errors/return_in_generator.py:3:5
+  |
+1 | async def gen():
+2 |     yield 1
+3 |     return 42
+  |     ^^^^^^^^^
+4 |
+5 | def gen(): # B901 but not a syntax error - not an async generator
+  |
+
+B901 Using `yield` and `return {value}` in a generator function can lead to confusing behavior
+ --> resources/test/fixtures/syntax_errors/return_in_generator.py:7:5
+  |
+5 | def gen(): # B901 but not a syntax error - not an async generator
+6 |     yield 1
+7 |     return 42
+  |     ^^^^^^^^^
+8 |
+9 | async def gen(): # ok - no value in return
+  |
+
+B901 Using `yield` and `return {value}` in a generator function can lead to confusing behavior
+  --> resources/test/fixtures/syntax_errors/return_in_generator.py:15:5
+   |
+13 | async def gen():
+14 |     yield 1
+15 |     return foo()
+   |     ^^^^^^^^^^^^
+16 |
+17 | async def gen():
+   |
+
+B901 Using `yield` and `return {value}` in a generator function can lead to confusing behavior
+  --> resources/test/fixtures/syntax_errors/return_in_generator.py:19:5
+   |
+17 | async def gen():
+18 |     yield 1
+19 |     return [1, 2, 3]
+   |     ^^^^^^^^^^^^^^^^
+20 |
+21 | async def gen():
+   |
+
+B901 Using `yield` and `return {value}` in a generator function can lead to confusing behavior
+  --> resources/test/fixtures/syntax_errors/return_in_generator.py:24:5
+   |
+22 |     if True:
+23 |         yield 1
+24 |     return 10
+   |     ^^^^^^^^^
+   |

crates/ruff_memory_usage/src/lib.rs

@@ -1,17 +1,46 @@
-use std::sync::{LazyLock, Mutex};
+use std::cell::RefCell;
 
 use get_size2::{GetSize, StandardTracker};
 use ordermap::{OrderMap, OrderSet};
 
+thread_local! {
+    pub static TRACKER: RefCell<Option<StandardTracker>>= const { RefCell::new(None) };
+}
+
+struct TrackerGuard(Option<StandardTracker>);
+
+impl Drop for TrackerGuard {
+    fn drop(&mut self) {
+        TRACKER.set(self.0.take());
+    }
+}
+
+pub fn attach_tracker<R>(tracker: StandardTracker, f: impl FnOnce() -> R) -> R {
+    let prev = TRACKER.replace(Some(tracker));
+    let _guard = TrackerGuard(prev);
+    f()
+}
+
+fn with_tracker<F, R>(f: F) -> R
+where
+    F: FnOnce(Option<&mut StandardTracker>) -> R,
+{
+    TRACKER.with(|tracker| {
+        let mut tracker = tracker.borrow_mut();
+        f(tracker.as_mut())
+    })
+}
+
 /// Returns the memory usage of the provided object, using a global tracker to avoid
 /// double-counting shared objects.
 pub fn heap_size<T: GetSize>(value: &T) -> usize {
-    static TRACKER: LazyLock<Mutex<StandardTracker>> =
-        LazyLock::new(|| Mutex::new(StandardTracker::new()));
-
-    value
-        .get_heap_size_with_tracker(&mut *TRACKER.lock().unwrap())
-        .0
+    with_tracker(|tracker| {
+        if let Some(tracker) = tracker {
+            value.get_heap_size_with_tracker(tracker).0
+        } else {
+            value.get_heap_size()
+        }
+    })
 }
 
 /// An implementation of [`GetSize::get_heap_size`] for [`OrderSet`].

crates/ruff_python_ast/src/lib.rs

@@ -29,6 +29,7 @@ pub mod statement_visitor;
 pub mod stmt_if;
 pub mod str;
 pub mod str_prefix;
+pub mod token;
 pub mod traversal;
 pub mod types;
 pub mod visitor;

crates/ruff_python_ast/src/parenthesize.rs

@@ -11,6 +11,8 @@ use crate::ExprRef;
 /// Note that without a parent the range can be inaccurate, e.g. `f(a)` we falsely return a set of
 /// parentheses around `a` even if the parentheses actually belong to `f`. That is why you should
 /// generally prefer [`parenthesized_range`].
+///
+/// Prefer [`crate::token::parentheses_iterator`] if you have access to [`crate::token::Tokens`].
 pub fn parentheses_iterator<'a>(
     expr: ExprRef<'a>,
     parent: Option<AnyNodeRef>,
@@ -57,6 +59,8 @@ pub fn parentheses_iterator<'a>(
 
 /// Returns the [`TextRange`] of a given expression including parentheses, if the expression is
 /// parenthesized; or `None`, if the expression is not parenthesized.
+///
+/// Prefer [`crate::token::parenthesized_range`] if you have access to [`crate::token::Tokens`].
 pub fn parenthesized_range(
     expr: ExprRef,
     parent: AnyNodeRef,

crates/ruff_python_ast/src/script.rs

@@ -12,11 +12,11 @@ static FINDER: LazyLock<Finder> = LazyLock::new(|| Finder::new(b"# /// script"))
 #[derive(Debug, Clone, Eq, PartialEq)]
 pub struct ScriptTag {
     /// The content of the script before the metadata block.
-    prelude: String,
+    pub prelude: String,
     /// The metadata block.
-    metadata: String,
+    pub metadata: String,
     /// The content of the script after the metadata block.
-    postlude: String,
+    pub postlude: String,
 }
 
 impl ScriptTag {

crates/ruff_python_ast/src/token.rs

@@ -0,0 +1,853 @@
+//! Token kinds for Python source code created by the lexer and consumed by the `ruff_python_parser`.
+//!
+//! This module defines the tokens that the lexer recognizes. The tokens are
+//! loosely based on the token definitions found in the [CPython source].
+//!
+//! [CPython source]: https://github.com/python/cpython/blob/dfc2e065a2e71011017077e549cd2f9bf4944c54/Grammar/Tokens
+
+use std::fmt;
+
+use bitflags::bitflags;
+
+use crate::str::{Quote, TripleQuotes};
+use crate::str_prefix::{
+    AnyStringPrefix, ByteStringPrefix, FStringPrefix, StringLiteralPrefix, TStringPrefix,
+};
+use crate::{AnyStringFlags, BoolOp, Operator, StringFlags, UnaryOp};
+use ruff_text_size::{Ranged, TextRange};
+
+mod parentheses;
+mod tokens;
+
+pub use parentheses::{parentheses_iterator, parenthesized_range};
+pub use tokens::{TokenAt, TokenIterWithContext, Tokens};
+
+#[derive(Clone, Copy, PartialEq, Eq)]
+#[cfg_attr(feature = "get-size", derive(get_size2::GetSize))]
+pub struct Token {
+    /// The kind of the token.
+    kind: TokenKind,
+    /// The range of the token.
+    range: TextRange,
+    /// The set of flags describing this token.
+    flags: TokenFlags,
+}
+
+impl Token {
+    pub fn new(kind: TokenKind, range: TextRange, flags: TokenFlags) -> Token {
+        Self { kind, range, flags }
+    }
+
+    /// Returns the token kind.
+    #[inline]
+    pub const fn kind(&self) -> TokenKind {
+        self.kind
+    }
+
+    /// Returns the token as a tuple of (kind, range).
+    #[inline]
+    pub const fn as_tuple(&self) -> (TokenKind, TextRange) {
+        (self.kind, self.range)
+    }
+
+    /// Returns `true` if the current token is a triple-quoted string of any kind.
+    ///
+    /// # Panics
+    ///
+    /// If it isn't a string or any f/t-string tokens.
+    pub fn is_triple_quoted_string(self) -> bool {
+        self.unwrap_string_flags().is_triple_quoted()
+    }
+
+    /// Returns the [`Quote`] style for the current string token of any kind.
+    ///
+    /// # Panics
+    ///
+    /// If it isn't a string or any f/t-string tokens.
+    pub fn string_quote_style(self) -> Quote {
+        self.unwrap_string_flags().quote_style()
+    }
+
+    /// Returns the [`AnyStringFlags`] style for the current string token of any kind.
+    ///
+    /// # Panics
+    ///
+    /// If it isn't a string or any f/t-string tokens.
+    pub fn unwrap_string_flags(self) -> AnyStringFlags {
+        self.string_flags()
+            .unwrap_or_else(|| panic!("token to be a string"))
+    }
+
+    /// Returns true if the current token is a string and it is raw.
+    pub fn string_flags(self) -> Option<AnyStringFlags> {
+        if self.is_any_string() {
+            Some(self.flags.as_any_string_flags())
+        } else {
+            None
+        }
+    }
+
+    /// Returns `true` if this is any kind of string token - including
+    /// tokens in t-strings (which do not have type `str`).
+    const fn is_any_string(self) -> bool {
+        matches!(
+            self.kind,
+            TokenKind::String
+                | TokenKind::FStringStart
+                | TokenKind::FStringMiddle
+                | TokenKind::FStringEnd
+                | TokenKind::TStringStart
+                | TokenKind::TStringMiddle
+                | TokenKind::TStringEnd
+        )
+    }
+}
+
+impl Ranged for Token {
+    fn range(&self) -> TextRange {
+        self.range
+    }
+}
+
+impl fmt::Debug for Token {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "{:?} {:?}", self.kind, self.range)?;
+        if !self.flags.is_empty() {
+            f.write_str(" (flags = ")?;
+            let mut first = true;
+            for (name, _) in self.flags.iter_names() {
+                if first {
+                    first = false;
+                } else {
+                    f.write_str(" | ")?;
+                }
+                f.write_str(name)?;
+            }
+            f.write_str(")")?;
+        }
+        Ok(())
+    }
+}
+
+/// A kind of a token.
+#[derive(Copy, Clone, PartialEq, Eq, Hash, Debug, PartialOrd, Ord)]
+#[cfg_attr(feature = "get-size", derive(get_size2::GetSize))]
+pub enum TokenKind {
+    /// Token kind for a name, commonly known as an identifier.
+    Name,
+    /// Token kind for an integer.
+    Int,
+    /// Token kind for a floating point number.
+    Float,
+    /// Token kind for a complex number.
+    Complex,
+    /// Token kind for a string.
+    String,
+    /// Token kind for the start of an f-string. This includes the `f`/`F`/`fr` prefix
+    /// and the opening quote(s).
+    FStringStart,
+    /// Token kind that includes the portion of text inside the f-string that's not
+    /// part of the expression part and isn't an opening or closing brace.
+    FStringMiddle,
+    /// Token kind for the end of an f-string. This includes the closing quote.
+    FStringEnd,
+    /// Token kind for the start of a t-string. This includes the `t`/`T`/`tr` prefix
+    /// and the opening quote(s).
+    TStringStart,
+    /// Token kind that includes the portion of text inside the t-string that's not
+    /// part of the interpolation part and isn't an opening or closing brace.
+    TStringMiddle,
+    /// Token kind for the end of a t-string. This includes the closing quote.
+    TStringEnd,
+    /// Token kind for a IPython escape command.
+    IpyEscapeCommand,
+    /// Token kind for a comment. These are filtered out of the token stream prior to parsing.
+    Comment,
+    /// Token kind for a newline.
+    Newline,
+    /// Token kind for a newline that is not a logical line break. These are filtered out of
+    /// the token stream prior to parsing.
+    NonLogicalNewline,
+    /// Token kind for an indent.
+    Indent,
+    /// Token kind for a dedent.
+    Dedent,
+    EndOfFile,
+    /// Token kind for a question mark `?`.
+    Question,
+    /// Token kind for an exclamation mark `!`.
+    Exclamation,
+    /// Token kind for a left parenthesis `(`.
+    Lpar,
+    /// Token kind for a right parenthesis `)`.
+    Rpar,
+    /// Token kind for a left square bracket `[`.
+    Lsqb,
+    /// Token kind for a right square bracket `]`.
+    Rsqb,
+    /// Token kind for a colon `:`.
+    Colon,
+    /// Token kind for a comma `,`.
+    Comma,
+    /// Token kind for a semicolon `;`.
+    Semi,
+    /// Token kind for plus `+`.
+    Plus,
+    /// Token kind for minus `-`.
+    Minus,
+    /// Token kind for star `*`.
+    Star,
+    /// Token kind for slash `/`.
+    Slash,
+    /// Token kind for vertical bar `|`.
+    Vbar,
+    /// Token kind for ampersand `&`.
+    Amper,
+    /// Token kind for less than `<`.
+    Less,
+    /// Token kind for greater than `>`.
+    Greater,
+    /// Token kind for equal `=`.
+    Equal,
+    /// Token kind for dot `.`.
+    Dot,
+    /// Token kind for percent `%`.
+    Percent,
+    /// Token kind for left bracket `{`.
+    Lbrace,
+    /// Token kind for right bracket `}`.
+    Rbrace,
+    /// Token kind for double equal `==`.
+    EqEqual,
+    /// Token kind for not equal `!=`.
+    NotEqual,
+    /// Token kind for less than or equal `<=`.
+    LessEqual,
+    /// Token kind for greater than or equal `>=`.
+    GreaterEqual,
+    /// Token kind for tilde `~`.
+    Tilde,
+    /// Token kind for caret `^`.
+    CircumFlex,
+    /// Token kind for left shift `<<`.
+    LeftShift,
+    /// Token kind for right shift `>>`.
+    RightShift,
+    /// Token kind for double star `**`.
+    DoubleStar,
+    /// Token kind for double star equal `**=`.
+    DoubleStarEqual,
+    /// Token kind for plus equal `+=`.
+    PlusEqual,
+    /// Token kind for minus equal `-=`.
+    MinusEqual,
+    /// Token kind for star equal `*=`.
+    StarEqual,
+    /// Token kind for slash equal `/=`.
+    SlashEqual,
+    /// Token kind for percent equal `%=`.
+    PercentEqual,
+    /// Token kind for ampersand equal `&=`.
+    AmperEqual,
+    /// Token kind for vertical bar equal `|=`.
+    VbarEqual,
+    /// Token kind for caret equal `^=`.
+    CircumflexEqual,
+    /// Token kind for left shift equal `<<=`.
+    LeftShiftEqual,
+    /// Token kind for right shift equal `>>=`.
+    RightShiftEqual,
+    /// Token kind for double slash `//`.
+    DoubleSlash,
+    /// Token kind for double slash equal `//=`.
+    DoubleSlashEqual,
+    /// Token kind for colon equal `:=`.
+    ColonEqual,
+    /// Token kind for at `@`.
+    At,
+    /// Token kind for at equal `@=`.
+    AtEqual,
+    /// Token kind for arrow `->`.
+    Rarrow,
+    /// Token kind for ellipsis `...`.
+    Ellipsis,
+
+    // The keywords should be sorted in alphabetical order. If the boundary tokens for the
+    // "Keywords" and "Soft keywords" group change, update the related methods on `TokenKind`.
+
+    // Keywords
+    And,
+    As,
+    Assert,
+    Async,
+    Await,
+    Break,
+    Class,
+    Continue,
+    Def,
+    Del,
+    Elif,
+    Else,
+    Except,
+    False,
+    Finally,
+    For,
+    From,
+    Global,
+    If,
+    Import,
+    In,
+    Is,
+    Lambda,
+    None,
+    Nonlocal,
+    Not,
+    Or,
+    Pass,
+    Raise,
+    Return,
+    True,
+    Try,
+    While,
+    With,
+    Yield,
+
+    // Soft keywords
+    Case,
+    Match,
+    Type,
+
+    Unknown,
+}
+
+impl TokenKind {
+    /// Returns `true` if this is an end of file token.
+    #[inline]
+    pub const fn is_eof(self) -> bool {
+        matches!(self, TokenKind::EndOfFile)
+    }
+
+    /// Returns `true` if this is either a newline or non-logical newline token.
+    #[inline]
+    pub const fn is_any_newline(self) -> bool {
+        matches!(self, TokenKind::Newline | TokenKind::NonLogicalNewline)
+    }
+
+    /// Returns `true` if the token is a keyword (including soft keywords).
+    ///
+    /// See also [`is_soft_keyword`], [`is_non_soft_keyword`].
+    ///
+    /// [`is_soft_keyword`]: TokenKind::is_soft_keyword
+    /// [`is_non_soft_keyword`]: TokenKind::is_non_soft_keyword
+    #[inline]
+    pub fn is_keyword(self) -> bool {
+        TokenKind::And <= self && self <= TokenKind::Type
+    }
+
+    /// Returns `true` if the token is strictly a soft keyword.
+    ///
+    /// See also [`is_keyword`], [`is_non_soft_keyword`].
+    ///
+    /// [`is_keyword`]: TokenKind::is_keyword
+    /// [`is_non_soft_keyword`]: TokenKind::is_non_soft_keyword
+    #[inline]
+    pub fn is_soft_keyword(self) -> bool {
+        TokenKind::Case <= self && self <= TokenKind::Type
+    }
+
+    /// Returns `true` if the token is strictly a non-soft keyword.
+    ///
+    /// See also [`is_keyword`], [`is_soft_keyword`].
+    ///
+    /// [`is_keyword`]: TokenKind::is_keyword
+    /// [`is_soft_keyword`]: TokenKind::is_soft_keyword
+    #[inline]
+    pub fn is_non_soft_keyword(self) -> bool {
+        TokenKind::And <= self && self <= TokenKind::Yield
+    }
+
+    #[inline]
+    pub const fn is_operator(self) -> bool {
+        matches!(
+            self,
+            TokenKind::Lpar
+                | TokenKind::Rpar
+                | TokenKind::Lsqb
+                | TokenKind::Rsqb
+                | TokenKind::Comma
+                | TokenKind::Semi
+                | TokenKind::Plus
+                | TokenKind::Minus
+                | TokenKind::Star
+                | TokenKind::Slash
+                | TokenKind::Vbar
+                | TokenKind::Amper
+                | TokenKind::Less
+                | TokenKind::Greater
+                | TokenKind::Equal
+                | TokenKind::Dot
+                | TokenKind::Percent
+                | TokenKind::Lbrace
+                | TokenKind::Rbrace
+                | TokenKind::EqEqual
+                | TokenKind::NotEqual
+                | TokenKind::LessEqual
+                | TokenKind::GreaterEqual
+                | TokenKind::Tilde
+                | TokenKind::CircumFlex
+                | TokenKind::LeftShift
+                | TokenKind::RightShift
+                | TokenKind::DoubleStar
+                | TokenKind::PlusEqual
+                | TokenKind::MinusEqual
+                | TokenKind::StarEqual
+                | TokenKind::SlashEqual
+                | TokenKind::PercentEqual
+                | TokenKind::AmperEqual
+                | TokenKind::VbarEqual
+                | TokenKind::CircumflexEqual
+                | TokenKind::LeftShiftEqual
+                | TokenKind::RightShiftEqual
+                | TokenKind::DoubleStarEqual
+                | TokenKind::DoubleSlash
+                | TokenKind::DoubleSlashEqual
+                | TokenKind::At
+                | TokenKind::AtEqual
+                | TokenKind::Rarrow
+                | TokenKind::Ellipsis
+                | TokenKind::ColonEqual
+                | TokenKind::Colon
+                | TokenKind::And
+                | TokenKind::Or
+                | TokenKind::Not
+                | TokenKind::In
+                | TokenKind::Is
+        )
+    }
+
+    /// Returns `true` if this is a singleton token i.e., `True`, `False`, or `None`.
+    #[inline]
+    pub const fn is_singleton(self) -> bool {
+        matches!(self, TokenKind::False | TokenKind::True | TokenKind::None)
+    }
+
+    /// Returns `true` if this is a trivia token i.e., a comment or a non-logical newline.
+    #[inline]
+    pub const fn is_trivia(&self) -> bool {
+        matches!(self, TokenKind::Comment | TokenKind::NonLogicalNewline)
+    }
+
+    /// Returns `true` if this is a comment token.
+    #[inline]
+    pub const fn is_comment(&self) -> bool {
+        matches!(self, TokenKind::Comment)
+    }
+
+    #[inline]
+    pub const fn is_arithmetic(self) -> bool {
+        matches!(
+            self,
+            TokenKind::DoubleStar
+                | TokenKind::Star
+                | TokenKind::Plus
+                | TokenKind::Minus
+                | TokenKind::Slash
+                | TokenKind::DoubleSlash
+                | TokenKind::At
+        )
+    }
+
+    #[inline]
+    pub const fn is_bitwise_or_shift(self) -> bool {
+        matches!(
+            self,
+            TokenKind::LeftShift
+                | TokenKind::LeftShiftEqual
+                | TokenKind::RightShift
+                | TokenKind::RightShiftEqual
+                | TokenKind::Amper
+                | TokenKind::AmperEqual
+                | TokenKind::Vbar
+                | TokenKind::VbarEqual
+                | TokenKind::CircumFlex
+                | TokenKind::CircumflexEqual
+                | TokenKind::Tilde
+        )
+    }
+
+    /// Returns `true` if the current token is a unary arithmetic operator.
+    #[inline]
+    pub const fn is_unary_arithmetic_operator(self) -> bool {
+        matches!(self, TokenKind::Plus | TokenKind::Minus)
+    }
+
+    #[inline]
+    pub const fn is_interpolated_string_end(self) -> bool {
+        matches!(self, TokenKind::FStringEnd | TokenKind::TStringEnd)
+    }
+
+    /// Returns the [`UnaryOp`] that corresponds to this token kind, if it is a unary arithmetic
+    /// operator, otherwise return [None].
+    ///
+    /// Use [`as_unary_operator`] to match against any unary operator.
+    ///
+    /// [`as_unary_operator`]: TokenKind::as_unary_operator
+    #[inline]
+    pub const fn as_unary_arithmetic_operator(self) -> Option<UnaryOp> {
+        Some(match self {
+            TokenKind::Plus => UnaryOp::UAdd,
+            TokenKind::Minus => UnaryOp::USub,
+            _ => return None,
+        })
+    }
+
+    /// Returns the [`UnaryOp`] that corresponds to this token kind, if it is a unary operator,
+    /// otherwise return [None].
+    ///
+    /// Use [`as_unary_arithmetic_operator`] to match against only an arithmetic unary operator.
+    ///
+    /// [`as_unary_arithmetic_operator`]: TokenKind::as_unary_arithmetic_operator
+    #[inline]
+    pub const fn as_unary_operator(self) -> Option<UnaryOp> {
+        Some(match self {
+            TokenKind::Plus => UnaryOp::UAdd,
+            TokenKind::Minus => UnaryOp::USub,
+            TokenKind::Tilde => UnaryOp::Invert,
+            TokenKind::Not => UnaryOp::Not,
+            _ => return None,
+        })
+    }
+
+    /// Returns the [`BoolOp`] that corresponds to this token kind, if it is a boolean operator,
+    /// otherwise return [None].
+    #[inline]
+    pub const fn as_bool_operator(self) -> Option<BoolOp> {
+        Some(match self {
+            TokenKind::And => BoolOp::And,
+            TokenKind::Or => BoolOp::Or,
+            _ => return None,
+        })
+    }
+
+    /// Returns the binary [`Operator`] that corresponds to the current token, if it's a binary
+    /// operator, otherwise return [None].
+    ///
+    /// Use [`as_augmented_assign_operator`] to match against an augmented assignment token.
+    ///
+    /// [`as_augmented_assign_operator`]: TokenKind::as_augmented_assign_operator
+    pub const fn as_binary_operator(self) -> Option<Operator> {
+        Some(match self {
+            TokenKind::Plus => Operator::Add,
+            TokenKind::Minus => Operator::Sub,
+            TokenKind::Star => Operator::Mult,
+            TokenKind::At => Operator::MatMult,
+            TokenKind::DoubleStar => Operator::Pow,
+            TokenKind::Slash => Operator::Div,
+            TokenKind::DoubleSlash => Operator::FloorDiv,
+            TokenKind::Percent => Operator::Mod,
+            TokenKind::Amper => Operator::BitAnd,
+            TokenKind::Vbar => Operator::BitOr,
+            TokenKind::CircumFlex => Operator::BitXor,
+            TokenKind::LeftShift => Operator::LShift,
+            TokenKind::RightShift => Operator::RShift,
+            _ => return None,
+        })
+    }
+
+    /// Returns the [`Operator`] that corresponds to this token kind, if it is
+    /// an augmented assignment operator, or [`None`] otherwise.
+    #[inline]
+    pub const fn as_augmented_assign_operator(self) -> Option<Operator> {
+        Some(match self {
+            TokenKind::PlusEqual => Operator::Add,
+            TokenKind::MinusEqual => Operator::Sub,
+            TokenKind::StarEqual => Operator::Mult,
+            TokenKind::AtEqual => Operator::MatMult,
+            TokenKind::DoubleStarEqual => Operator::Pow,
+            TokenKind::SlashEqual => Operator::Div,
+            TokenKind::DoubleSlashEqual => Operator::FloorDiv,
+            TokenKind::PercentEqual => Operator::Mod,
+            TokenKind::AmperEqual => Operator::BitAnd,
+            TokenKind::VbarEqual => Operator::BitOr,
+            TokenKind::CircumflexEqual => Operator::BitXor,
+            TokenKind::LeftShiftEqual => Operator::LShift,
+            TokenKind::RightShiftEqual => Operator::RShift,
+            _ => return None,
+        })
+    }
+}
+
+impl From<BoolOp> for TokenKind {
+    #[inline]
+    fn from(op: BoolOp) -> Self {
+        match op {
+            BoolOp::And => TokenKind::And,
+            BoolOp::Or => TokenKind::Or,
+        }
+    }
+}
+
+impl From<UnaryOp> for TokenKind {
+    #[inline]
+    fn from(op: UnaryOp) -> Self {
+        match op {
+            UnaryOp::Invert => TokenKind::Tilde,
+            UnaryOp::Not => TokenKind::Not,
+            UnaryOp::UAdd => TokenKind::Plus,
+            UnaryOp::USub => TokenKind::Minus,
+        }
+    }
+}
+
+impl From<Operator> for TokenKind {
+    #[inline]
+    fn from(op: Operator) -> Self {
+        match op {
+            Operator::Add => TokenKind::Plus,
+            Operator::Sub => TokenKind::Minus,
+            Operator::Mult => TokenKind::Star,
+            Operator::MatMult => TokenKind::At,
+            Operator::Div => TokenKind::Slash,
+            Operator::Mod => TokenKind::Percent,
+            Operator::Pow => TokenKind::DoubleStar,
+            Operator::LShift => TokenKind::LeftShift,
+            Operator::RShift => TokenKind::RightShift,
+            Operator::BitOr => TokenKind::Vbar,
+            Operator::BitXor => TokenKind::CircumFlex,
+            Operator::BitAnd => TokenKind::Amper,
+            Operator::FloorDiv => TokenKind::DoubleSlash,
+        }
+    }
+}
+
+impl fmt::Display for TokenKind {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        let value = match self {
+            TokenKind::Unknown => "Unknown",
+            TokenKind::Newline => "newline",
+            TokenKind::NonLogicalNewline => "NonLogicalNewline",
+            TokenKind::Indent => "indent",
+            TokenKind::Dedent => "dedent",
+            TokenKind::EndOfFile => "end of file",
+            TokenKind::Name => "name",
+            TokenKind::Int => "int",
+            TokenKind::Float => "float",
+            TokenKind::Complex => "complex",
+            TokenKind::String => "string",
+            TokenKind::FStringStart => "FStringStart",
+            TokenKind::FStringMiddle => "FStringMiddle",
+            TokenKind::FStringEnd => "FStringEnd",
+            TokenKind::TStringStart => "TStringStart",
+            TokenKind::TStringMiddle => "TStringMiddle",
+            TokenKind::TStringEnd => "TStringEnd",
+            TokenKind::IpyEscapeCommand => "IPython escape command",
+            TokenKind::Comment => "comment",
+            TokenKind::Question => "`?`",
+            TokenKind::Exclamation => "`!`",
+            TokenKind::Lpar => "`(`",
+            TokenKind::Rpar => "`)`",
+            TokenKind::Lsqb => "`[`",
+            TokenKind::Rsqb => "`]`",
+            TokenKind::Lbrace => "`{`",
+            TokenKind::Rbrace => "`}`",
+            TokenKind::Equal => "`=`",
+            TokenKind::ColonEqual => "`:=`",
+            TokenKind::Dot => "`.`",
+            TokenKind::Colon => "`:`",
+            TokenKind::Semi => "`;`",
+            TokenKind::Comma => "`,`",
+            TokenKind::Rarrow => "`->`",
+            TokenKind::Plus => "`+`",
+            TokenKind::Minus => "`-`",
+            TokenKind::Star => "`*`",
+            TokenKind::DoubleStar => "`**`",
+            TokenKind::Slash => "`/`",
+            TokenKind::DoubleSlash => "`//`",
+            TokenKind::Percent => "`%`",
+            TokenKind::Vbar => "`|`",
+            TokenKind::Amper => "`&`",
+            TokenKind::CircumFlex => "`^`",
+            TokenKind::LeftShift => "`<<`",
+            TokenKind::RightShift => "`>>`",
+            TokenKind::Tilde => "`~`",
+            TokenKind::At => "`@`",
+            TokenKind::Less => "`<`",
+            TokenKind::Greater => "`>`",
+            TokenKind::EqEqual => "`==`",
+            TokenKind::NotEqual => "`!=`",
+            TokenKind::LessEqual => "`<=`",
+            TokenKind::GreaterEqual => "`>=`",
+            TokenKind::PlusEqual => "`+=`",
+            TokenKind::MinusEqual => "`-=`",
+            TokenKind::StarEqual => "`*=`",
+            TokenKind::DoubleStarEqual => "`**=`",
+            TokenKind::SlashEqual => "`/=`",
+            TokenKind::DoubleSlashEqual => "`//=`",
+            TokenKind::PercentEqual => "`%=`",
+            TokenKind::VbarEqual => "`|=`",
+            TokenKind::AmperEqual => "`&=`",
+            TokenKind::CircumflexEqual => "`^=`",
+            TokenKind::LeftShiftEqual => "`<<=`",
+            TokenKind::RightShiftEqual => "`>>=`",
+            TokenKind::AtEqual => "`@=`",
+            TokenKind::Ellipsis => "`...`",
+            TokenKind::False => "`False`",
+            TokenKind::None => "`None`",
+            TokenKind::True => "`True`",
+            TokenKind::And => "`and`",
+            TokenKind::As => "`as`",
+            TokenKind::Assert => "`assert`",
+            TokenKind::Async => "`async`",
+            TokenKind::Await => "`await`",
+            TokenKind::Break => "`break`",
+            TokenKind::Class => "`class`",
+            TokenKind::Continue => "`continue`",
+            TokenKind::Def => "`def`",
+            TokenKind::Del => "`del`",
+            TokenKind::Elif => "`elif`",
+            TokenKind::Else => "`else`",
+            TokenKind::Except => "`except`",
+            TokenKind::Finally => "`finally`",
+            TokenKind::For => "`for`",
+            TokenKind::From => "`from`",
+            TokenKind::Global => "`global`",
+            TokenKind::If => "`if`",
+            TokenKind::Import => "`import`",
+            TokenKind::In => "`in`",
+            TokenKind::Is => "`is`",
+            TokenKind::Lambda => "`lambda`",
+            TokenKind::Nonlocal => "`nonlocal`",
+            TokenKind::Not => "`not`",
+            TokenKind::Or => "`or`",
+            TokenKind::Pass => "`pass`",
+            TokenKind::Raise => "`raise`",
+            TokenKind::Return => "`return`",
+            TokenKind::Try => "`try`",
+            TokenKind::While => "`while`",
+            TokenKind::Match => "`match`",
+            TokenKind::Type => "`type`",
+            TokenKind::Case => "`case`",
+            TokenKind::With => "`with`",
+            TokenKind::Yield => "`yield`",
+        };
+        f.write_str(value)
+    }
+}
+
+bitflags! {
+    #[derive(Clone, Copy, Debug, PartialEq, Eq)]
+    pub struct TokenFlags: u16 {
+        /// The token is a string with double quotes (`"`).
+        const DOUBLE_QUOTES = 1 << 0;
+        /// The token is a triple-quoted string i.e., it starts and ends with three consecutive
+        /// quote characters (`"""` or `'''`).
+        const TRIPLE_QUOTED_STRING = 1 << 1;
+
+        /// The token is a unicode string i.e., prefixed with `u` or `U`
+        const UNICODE_STRING = 1 << 2;
+        /// The token is a byte string i.e., prefixed with `b` or `B`
+        const BYTE_STRING = 1 << 3;
+        /// The token is an f-string i.e., prefixed with `f` or `F`
+        const F_STRING = 1 << 4;
+        /// The token is a t-string i.e., prefixed with `t` or `T`
+        const T_STRING = 1 << 5;
+        /// The token is a raw string and the prefix character is in lowercase.
+        const RAW_STRING_LOWERCASE = 1 << 6;
+        /// The token is a raw string and the prefix character is in uppercase.
+        const RAW_STRING_UPPERCASE = 1 << 7;
+        /// String without matching closing quote(s)
+        const UNCLOSED_STRING = 1 << 8;
+
+        /// The token is a raw string i.e., prefixed with `r` or `R`
+        const RAW_STRING = Self::RAW_STRING_LOWERCASE.bits() | Self::RAW_STRING_UPPERCASE.bits();
+
+    }
+}
+
+#[cfg(feature = "get-size")]
+impl get_size2::GetSize for TokenFlags {}
+
+impl StringFlags for TokenFlags {
+    fn quote_style(self) -> Quote {
+        if self.intersects(TokenFlags::DOUBLE_QUOTES) {
+            Quote::Double
+        } else {
+            Quote::Single
+        }
+    }
+
+    fn triple_quotes(self) -> TripleQuotes {
+        if self.intersects(TokenFlags::TRIPLE_QUOTED_STRING) {
+            TripleQuotes::Yes
+        } else {
+            TripleQuotes::No
+        }
+    }
+
+    fn prefix(self) -> AnyStringPrefix {
+        if self.intersects(TokenFlags::F_STRING) {
+            if self.intersects(TokenFlags::RAW_STRING_LOWERCASE) {
+                AnyStringPrefix::Format(FStringPrefix::Raw { uppercase_r: false })
+            } else if self.intersects(TokenFlags::RAW_STRING_UPPERCASE) {
+                AnyStringPrefix::Format(FStringPrefix::Raw { uppercase_r: true })
+            } else {
+                AnyStringPrefix::Format(FStringPrefix::Regular)
+            }
+        } else if self.intersects(TokenFlags::T_STRING) {
+            if self.intersects(TokenFlags::RAW_STRING_LOWERCASE) {
+                AnyStringPrefix::Template(TStringPrefix::Raw { uppercase_r: false })
+            } else if self.intersects(TokenFlags::RAW_STRING_UPPERCASE) {
+                AnyStringPrefix::Template(TStringPrefix::Raw { uppercase_r: true })
+            } else {
+                AnyStringPrefix::Template(TStringPrefix::Regular)
+            }
+        } else if self.intersects(TokenFlags::BYTE_STRING) {
+            if self.intersects(TokenFlags::RAW_STRING_LOWERCASE) {
+                AnyStringPrefix::Bytes(ByteStringPrefix::Raw { uppercase_r: false })
+            } else if self.intersects(TokenFlags::RAW_STRING_UPPERCASE) {
+                AnyStringPrefix::Bytes(ByteStringPrefix::Raw { uppercase_r: true })
+            } else {
+                AnyStringPrefix::Bytes(ByteStringPrefix::Regular)
+            }
+        } else if self.intersects(TokenFlags::RAW_STRING_LOWERCASE) {
+            AnyStringPrefix::Regular(StringLiteralPrefix::Raw { uppercase: false })
+        } else if self.intersects(TokenFlags::RAW_STRING_UPPERCASE) {
+            AnyStringPrefix::Regular(StringLiteralPrefix::Raw { uppercase: true })
+        } else if self.intersects(TokenFlags::UNICODE_STRING) {
+            AnyStringPrefix::Regular(StringLiteralPrefix::Unicode)
+        } else {
+            AnyStringPrefix::Regular(StringLiteralPrefix::Empty)
+        }
+    }
+
+    fn is_unclosed(self) -> bool {
+        self.intersects(TokenFlags::UNCLOSED_STRING)
+    }
+}
+
+impl TokenFlags {
+    /// Returns `true` if the token is an f-string.
+    pub const fn is_f_string(self) -> bool {
+        self.intersects(TokenFlags::F_STRING)
+    }
+
+    /// Returns `true` if the token is a t-string.
+    pub const fn is_t_string(self) -> bool {
+        self.intersects(TokenFlags::T_STRING)
+    }
+
+    /// Returns `true` if the token is a t-string.
+    pub const fn is_interpolated_string(self) -> bool {
+        self.intersects(TokenFlags::T_STRING.union(TokenFlags::F_STRING))
+    }
+
+    /// Returns `true` if the token is a triple-quoted t-string.
+    pub fn is_triple_quoted_interpolated_string(self) -> bool {
+        self.intersects(TokenFlags::TRIPLE_QUOTED_STRING) && self.is_interpolated_string()
+    }
+
+    /// Returns `true` if the token is a raw string.
+    pub const fn is_raw_string(self) -> bool {
+        self.intersects(TokenFlags::RAW_STRING)
+    }
+}

crates/ruff_python_ast/src/token/parentheses.rs

@@ -0,0 +1,58 @@
+use ruff_text_size::{Ranged, TextLen, TextRange};
+
+use super::{TokenKind, Tokens};
+use crate::{AnyNodeRef, ExprRef};
+
+/// Returns an iterator over the ranges of the optional parentheses surrounding an expression.
+///
+/// E.g. for `((f()))` with `f()` as expression, the iterator returns the ranges (1, 6) and (0, 7).
+///
+/// Note that without a parent the range can be inaccurate, e.g. `f(a)` we falsely return a set of
+/// parentheses around `a` even if the parentheses actually belong to `f`. That is why you should
+/// generally prefer [`parenthesized_range`].
+pub fn parentheses_iterator<'a>(
+    expr: ExprRef<'a>,
+    parent: Option<AnyNodeRef>,
+    tokens: &'a Tokens,
+) -> impl Iterator<Item = TextRange> + 'a {
+    let after_tokens = if let Some(parent) = parent {
+        // If the parent is a node that brings its own parentheses, exclude the closing parenthesis
+        // from our search range. Otherwise, we risk matching on calls, like `func(x)`, for which
+        // the open and close parentheses are part of the `Arguments` node.
+        let exclusive_parent_end = if parent.is_arguments() {
+            parent.end() - ")".text_len()
+        } else {
+            parent.end()
+        };
+
+        tokens.in_range(TextRange::new(expr.end(), exclusive_parent_end))
+    } else {
+        tokens.after(expr.end())
+    };
+
+    let right_parens = after_tokens
+        .iter()
+        .filter(|token| !token.kind().is_trivia())
+        .take_while(move |token| token.kind() == TokenKind::Rpar);
+
+    let left_parens = tokens
+        .before(expr.start())
+        .iter()
+        .rev()
+        .filter(|token| !token.kind().is_trivia())
+        .take_while(|token| token.kind() == TokenKind::Lpar);
+
+    right_parens
+        .zip(left_parens)
+        .map(|(right, left)| TextRange::new(left.start(), right.end()))
+}
+
+/// Returns the [`TextRange`] of a given expression including parentheses, if the expression is
+/// parenthesized; or `None`, if the expression is not parenthesized.
+pub fn parenthesized_range(
+    expr: ExprRef,
+    parent: AnyNodeRef,
+    tokens: &Tokens,
+) -> Option<TextRange> {
+    parentheses_iterator(expr, Some(parent), tokens).last()
+}

crates/ruff_python_ast/src/token/tokens.rs

@@ -0,0 +1,520 @@
+use std::{iter::FusedIterator, ops::Deref};
+
+use super::{Token, TokenKind};
+use ruff_python_trivia::CommentRanges;
+use ruff_text_size::{Ranged as _, TextRange, TextSize};
+
+/// Tokens represents a vector of lexed [`Token`].
+#[derive(Debug, Clone, PartialEq, Eq)]
+#[cfg_attr(feature = "get-size", derive(get_size2::GetSize))]
+pub struct Tokens {
+    raw: Vec<Token>,
+}
+
+impl Tokens {
+    pub fn new(tokens: Vec<Token>) -> Tokens {
+        Tokens { raw: tokens }
+    }
+
+    /// Returns an iterator over all the tokens that provides context.
+    pub fn iter_with_context(&self) -> TokenIterWithContext<'_> {
+        TokenIterWithContext::new(&self.raw)
+    }
+
+    /// Performs a binary search to find the index of the **first** token that starts at the given `offset`.
+    ///
+    /// Unlike `binary_search_by_key`, this method ensures that if multiple tokens start at the same offset,
+    /// it returns the index of the first one. Multiple tokens can start at the same offset in cases where
+    /// zero-length tokens are involved (like `Dedent` or `Newline` at the end of the file).
+    pub fn binary_search_by_start(&self, offset: TextSize) -> Result<usize, usize> {
+        let partition_point = self.partition_point(|token| token.start() < offset);
+
+        let after = &self[partition_point..];
+
+        if after.first().is_some_and(|first| first.start() == offset) {
+            Ok(partition_point)
+        } else {
+            Err(partition_point)
+        }
+    }
+
+    /// Returns a slice of [`Token`] that are within the given `range`.
+    ///
+    /// The start and end offset of the given range should be either:
+    /// 1. Token boundary
+    /// 2. Gap between the tokens
+    ///
+    /// For example, considering the following tokens and their corresponding range:
+    ///
+    /// | Token               | Range     |
+    /// |---------------------|-----------|
+    /// | `Def`               | `0..3`    |
+    /// | `Name`              | `4..7`    |
+    /// | `Lpar`              | `7..8`    |
+    /// | `Rpar`              | `8..9`    |
+    /// | `Colon`             | `9..10`   |
+    /// | `Newline`           | `10..11`  |
+    /// | `Comment`           | `15..24`  |
+    /// | `NonLogicalNewline` | `24..25`  |
+    /// | `Indent`            | `25..29`  |
+    /// | `Pass`              | `29..33`  |
+    ///
+    /// Here, for (1) a token boundary is considered either the start or end offset of any of the
+    /// above tokens. For (2), the gap would be any offset between the `Newline` and `Comment`
+    /// token which are 12, 13, and 14.
+    ///
+    /// Examples:
+    /// 1) `4..10` would give `Name`, `Lpar`, `Rpar`, `Colon`
+    /// 2) `11..25` would give `Comment`, `NonLogicalNewline`
+    /// 3) `12..25` would give same as (2) and offset 12 is in the "gap"
+    /// 4) `9..12` would give `Colon`, `Newline` and offset 12 is in the "gap"
+    /// 5) `18..27` would panic because both the start and end offset is within a token
+    ///
+    /// ## Note
+    ///
+    /// The returned slice can contain the [`TokenKind::Unknown`] token if there was a lexical
+    /// error encountered within the given range.
+    ///
+    /// # Panics
+    ///
+    /// If either the start or end offset of the given range is within a token range.
+    pub fn in_range(&self, range: TextRange) -> &[Token] {
+        let tokens_after_start = self.after(range.start());
+
+        Self::before_impl(tokens_after_start, range.end())
+    }
+
+    /// Searches the token(s) at `offset`.
+    ///
+    /// Returns [`TokenAt::Between`] if `offset` points directly inbetween two tokens
+    /// (the left token ends at `offset` and the right token starts at `offset`).
+    pub fn at_offset(&self, offset: TextSize) -> TokenAt {
+        match self.binary_search_by_start(offset) {
+            // The token at `index` starts exactly at `offset.
+            // ```python
+            // object.attribute
+            //        ^ OFFSET
+            // ```
+            Ok(index) => {
+                let token = self[index];
+                // `token` starts exactly at `offset`. Test if the offset is right between
+                // `token` and the previous token (if there's any)
+                if let Some(previous) = index.checked_sub(1).map(|idx| self[idx]) {
+                    if previous.end() == offset {
+                        return TokenAt::Between(previous, token);
+                    }
+                }
+
+                TokenAt::Single(token)
+            }
+
+            // No token found that starts exactly at the given offset. But it's possible that
+            // the token starting before `offset` fully encloses `offset` (it's end range ends after `offset`).
+            // ```python
+            // object.attribute
+            //   ^ OFFSET
+            // # or
+            // if True:
+            //     print("test")
+            //  ^ OFFSET
+            // ```
+            Err(index) => {
+                if let Some(previous) = index.checked_sub(1).map(|idx| self[idx]) {
+                    if previous.range().contains_inclusive(offset) {
+                        return TokenAt::Single(previous);
+                    }
+                }
+
+                TokenAt::None
+            }
+        }
+    }
+
+    /// Returns a slice of tokens before the given [`TextSize`] offset.
+    ///
+    /// If the given offset is between two tokens, the returned slice will end just before the
+    /// following token. In other words, if the offset is between the end of previous token and
+    /// start of next token, the returned slice will end just before the next token.
+    ///
+    /// # Panics
+    ///
+    /// If the given offset is inside a token range at any point
+    /// other than the start of the range.
+    pub fn before(&self, offset: TextSize) -> &[Token] {
+        Self::before_impl(&self.raw, offset)
+    }
+
+    fn before_impl(tokens: &[Token], offset: TextSize) -> &[Token] {
+        let partition_point = tokens.partition_point(|token| token.start() < offset);
+        let before = &tokens[..partition_point];
+
+        if let Some(last) = before.last() {
+            // If it's equal to the end offset, then it's at a token boundary which is
+            // valid. If it's greater than the end offset, then it's in the gap between
+            // the tokens which is valid as well.
+            assert!(
+                offset >= last.end(),
+                "Offset {:?} is inside a token range {:?}",
+                offset,
+                last.range()
+            );
+        }
+        before
+    }
+
+    /// Returns a slice of tokens after the given [`TextSize`] offset.
+    ///
+    /// If the given offset is between two tokens, the returned slice will start from the following
+    /// token. In other words, if the offset is between the end of previous token and start of next
+    /// token, the returned slice will start from the next token.
+    ///
+    /// # Panics
+    ///
+    /// If the given offset is inside a token range at any point
+    /// other than the start of the range.
+    pub fn after(&self, offset: TextSize) -> &[Token] {
+        let partition_point = self.partition_point(|token| token.end() <= offset);
+        let after = &self[partition_point..];
+
+        if let Some(first) = after.first() {
+            // valid. If it's greater than the end offset, then it's in the gap between
+            // the tokens which is valid as well.
+            assert!(
+                offset <= first.start(),
+                "Offset {:?} is inside a token range {:?}",
+                offset,
+                first.range()
+            );
+        }
+
+        after
+    }
+}
+
+impl<'a> IntoIterator for &'a Tokens {
+    type Item = &'a Token;
+    type IntoIter = std::slice::Iter<'a, Token>;
+
+    fn into_iter(self) -> Self::IntoIter {
+        self.iter()
+    }
+}
+
+impl Deref for Tokens {
+    type Target = [Token];
+
+    fn deref(&self) -> &Self::Target {
+        &self.raw
+    }
+}
+
+/// A token that encloses a given offset or ends exactly at it.
+#[derive(Debug, Clone)]
+pub enum TokenAt {
+    /// There's no token at the given offset
+    None,
+
+    /// There's a single token at the given offset.
+    Single(Token),
+
+    /// The offset falls exactly between two tokens. E.g. `CURSOR` in `call<CURSOR>(arguments)` is
+    /// positioned exactly between the `call` and `(` tokens.
+    Between(Token, Token),
+}
+
+impl Iterator for TokenAt {
+    type Item = Token;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        match *self {
+            TokenAt::None => None,
+            TokenAt::Single(token) => {
+                *self = TokenAt::None;
+                Some(token)
+            }
+            TokenAt::Between(first, second) => {
+                *self = TokenAt::Single(second);
+                Some(first)
+            }
+        }
+    }
+}
+
+impl FusedIterator for TokenAt {}
+
+impl From<&Tokens> for CommentRanges {
+    fn from(tokens: &Tokens) -> Self {
+        let mut ranges = vec![];
+        for token in tokens {
+            if token.kind() == TokenKind::Comment {
+                ranges.push(token.range());
+            }
+        }
+        CommentRanges::new(ranges)
+    }
+}
+
+/// An iterator over the [`Token`]s with context.
+///
+/// This struct is created by the [`iter_with_context`] method on [`Tokens`]. Refer to its
+/// documentation for more details.
+///
+/// [`iter_with_context`]: Tokens::iter_with_context
+#[derive(Debug, Clone)]
+pub struct TokenIterWithContext<'a> {
+    inner: std::slice::Iter<'a, Token>,
+    nesting: u32,
+}
+
+impl<'a> TokenIterWithContext<'a> {
+    fn new(tokens: &'a [Token]) -> TokenIterWithContext<'a> {
+        TokenIterWithContext {
+            inner: tokens.iter(),
+            nesting: 0,
+        }
+    }
+
+    /// Return the nesting level the iterator is currently in.
+    pub const fn nesting(&self) -> u32 {
+        self.nesting
+    }
+
+    /// Returns `true` if the iterator is within a parenthesized context.
+    pub const fn in_parenthesized_context(&self) -> bool {
+        self.nesting > 0
+    }
+
+    /// Returns the next [`Token`] in the iterator without consuming it.
+    pub fn peek(&self) -> Option<&'a Token> {
+        self.clone().next()
+    }
+}
+
+impl<'a> Iterator for TokenIterWithContext<'a> {
+    type Item = &'a Token;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        let token = self.inner.next()?;
+
+        match token.kind() {
+            TokenKind::Lpar | TokenKind::Lbrace | TokenKind::Lsqb => self.nesting += 1,
+            TokenKind::Rpar | TokenKind::Rbrace | TokenKind::Rsqb => {
+                self.nesting = self.nesting.saturating_sub(1);
+            }
+            // This mimics the behavior of re-lexing which reduces the nesting level on the lexer.
+            // We don't need to reduce it by 1 because unlike the lexer we see the final token
+            // after recovering from every unclosed parenthesis.
+            TokenKind::Newline if self.nesting > 0 => {
+                self.nesting = 0;
+            }
+            _ => {}
+        }
+
+        Some(token)
+    }
+}
+
+impl FusedIterator for TokenIterWithContext<'_> {}
+
+#[cfg(test)]
+mod tests {
+    use std::ops::Range;
+
+    use ruff_text_size::TextSize;
+
+    use crate::token::{Token, TokenFlags, TokenKind};
+
+    use super::*;
+
+    /// Test case containing a "gap" between two tokens.
+    ///
+    /// Code: <https://play.ruff.rs/a3658340-6df8-42c5-be80-178744bf1193>
+    const TEST_CASE_WITH_GAP: [(TokenKind, Range<u32>); 10] = [
+        (TokenKind::Def, 0..3),
+        (TokenKind::Name, 4..7),
+        (TokenKind::Lpar, 7..8),
+        (TokenKind::Rpar, 8..9),
+        (TokenKind::Colon, 9..10),
+        (TokenKind::Newline, 10..11),
+        // Gap               ||..||
+        (TokenKind::Comment, 15..24),
+        (TokenKind::NonLogicalNewline, 24..25),
+        (TokenKind::Indent, 25..29),
+        (TokenKind::Pass, 29..33),
+        // No newline at the end to keep the token set full of unique tokens
+    ];
+
+    /// Helper function to create [`Tokens`] from an iterator of (kind, range).
+    fn new_tokens(tokens: impl Iterator<Item = (TokenKind, Range<u32>)>) -> Tokens {
+        Tokens::new(
+            tokens
+                .map(|(kind, range)| {
+                    Token::new(
+                        kind,
+                        TextRange::new(TextSize::new(range.start), TextSize::new(range.end)),
+                        TokenFlags::empty(),
+                    )
+                })
+                .collect(),
+        )
+    }
+
+    #[test]
+    fn tokens_after_offset_at_token_start() {
+        let tokens = new_tokens(TEST_CASE_WITH_GAP.into_iter());
+        let after = tokens.after(TextSize::new(8));
+        assert_eq!(after.len(), 7);
+        assert_eq!(after.first().unwrap().kind(), TokenKind::Rpar);
+    }
+
+    #[test]
+    fn tokens_after_offset_at_token_end() {
+        let tokens = new_tokens(TEST_CASE_WITH_GAP.into_iter());
+        let after = tokens.after(TextSize::new(11));
+        assert_eq!(after.len(), 4);
+        assert_eq!(after.first().unwrap().kind(), TokenKind::Comment);
+    }
+
+    #[test]
+    fn tokens_after_offset_between_tokens() {
+        let tokens = new_tokens(TEST_CASE_WITH_GAP.into_iter());
+        let after = tokens.after(TextSize::new(13));
+        assert_eq!(after.len(), 4);
+        assert_eq!(after.first().unwrap().kind(), TokenKind::Comment);
+    }
+
+    #[test]
+    fn tokens_after_offset_at_last_token_end() {
+        let tokens = new_tokens(TEST_CASE_WITH_GAP.into_iter());
+        let after = tokens.after(TextSize::new(33));
+        assert_eq!(after.len(), 0);
+    }
+
+    #[test]
+    #[should_panic(expected = "Offset 5 is inside a token range 4..7")]
+    fn tokens_after_offset_inside_token() {
+        let tokens = new_tokens(TEST_CASE_WITH_GAP.into_iter());
+        tokens.after(TextSize::new(5));
+    }
+
+    #[test]
+    fn tokens_before_offset_at_first_token_start() {
+        let tokens = new_tokens(TEST_CASE_WITH_GAP.into_iter());
+        let before = tokens.before(TextSize::new(0));
+        assert_eq!(before.len(), 0);
+    }
+
+    #[test]
+    fn tokens_before_offset_after_first_token_gap() {
+        let tokens = new_tokens(TEST_CASE_WITH_GAP.into_iter());
+        let before = tokens.before(TextSize::new(3));
+        assert_eq!(before.len(), 1);
+        assert_eq!(before.last().unwrap().kind(), TokenKind::Def);
+    }
+
+    #[test]
+    fn tokens_before_offset_at_second_token_start() {
+        let tokens = new_tokens(TEST_CASE_WITH_GAP.into_iter());
+        let before = tokens.before(TextSize::new(4));
+        assert_eq!(before.len(), 1);
+        assert_eq!(before.last().unwrap().kind(), TokenKind::Def);
+    }
+
+    #[test]
+    fn tokens_before_offset_at_token_start() {
+        let tokens = new_tokens(TEST_CASE_WITH_GAP.into_iter());
+        let before = tokens.before(TextSize::new(8));
+        assert_eq!(before.len(), 3);
+        assert_eq!(before.last().unwrap().kind(), TokenKind::Lpar);
+    }
+
+    #[test]
+    fn tokens_before_offset_at_token_end() {
+        let tokens = new_tokens(TEST_CASE_WITH_GAP.into_iter());
+        let before = tokens.before(TextSize::new(11));
+        assert_eq!(before.len(), 6);
+        assert_eq!(before.last().unwrap().kind(), TokenKind::Newline);
+    }
+
+    #[test]
+    fn tokens_before_offset_between_tokens() {
+        let tokens = new_tokens(TEST_CASE_WITH_GAP.into_iter());
+        let before = tokens.before(TextSize::new(13));
+        assert_eq!(before.len(), 6);
+        assert_eq!(before.last().unwrap().kind(), TokenKind::Newline);
+    }
+
+    #[test]
+    fn tokens_before_offset_at_last_token_end() {
+        let tokens = new_tokens(TEST_CASE_WITH_GAP.into_iter());
+        let before = tokens.before(TextSize::new(33));
+        assert_eq!(before.len(), 10);
+        assert_eq!(before.last().unwrap().kind(), TokenKind::Pass);
+    }
+
+    #[test]
+    #[should_panic(expected = "Offset 5 is inside a token range 4..7")]
+    fn tokens_before_offset_inside_token() {
+        let tokens = new_tokens(TEST_CASE_WITH_GAP.into_iter());
+        tokens.before(TextSize::new(5));
+    }
+
+    #[test]
+    fn tokens_in_range_at_token_offset() {
+        let tokens = new_tokens(TEST_CASE_WITH_GAP.into_iter());
+        let in_range = tokens.in_range(TextRange::new(4.into(), 10.into()));
+        assert_eq!(in_range.len(), 4);
+        assert_eq!(in_range.first().unwrap().kind(), TokenKind::Name);
+        assert_eq!(in_range.last().unwrap().kind(), TokenKind::Colon);
+    }
+
+    #[test]
+    fn tokens_in_range_start_offset_at_token_end() {
+        let tokens = new_tokens(TEST_CASE_WITH_GAP.into_iter());
+        let in_range = tokens.in_range(TextRange::new(11.into(), 29.into()));
+        assert_eq!(in_range.len(), 3);
+        assert_eq!(in_range.first().unwrap().kind(), TokenKind::Comment);
+        assert_eq!(in_range.last().unwrap().kind(), TokenKind::Indent);
+    }
+
+    #[test]
+    fn tokens_in_range_end_offset_at_token_start() {
+        let tokens = new_tokens(TEST_CASE_WITH_GAP.into_iter());
+        let in_range = tokens.in_range(TextRange::new(8.into(), 15.into()));
+        assert_eq!(in_range.len(), 3);
+        assert_eq!(in_range.first().unwrap().kind(), TokenKind::Rpar);
+        assert_eq!(in_range.last().unwrap().kind(), TokenKind::Newline);
+    }
+
+    #[test]
+    fn tokens_in_range_start_offset_between_tokens() {
+        let tokens = new_tokens(TEST_CASE_WITH_GAP.into_iter());
+        let in_range = tokens.in_range(TextRange::new(13.into(), 29.into()));
+        assert_eq!(in_range.len(), 3);
+        assert_eq!(in_range.first().unwrap().kind(), TokenKind::Comment);
+        assert_eq!(in_range.last().unwrap().kind(), TokenKind::Indent);
+    }
+
+    #[test]
+    fn tokens_in_range_end_offset_between_tokens() {
+        let tokens = new_tokens(TEST_CASE_WITH_GAP.into_iter());
+        let in_range = tokens.in_range(TextRange::new(9.into(), 13.into()));
+        assert_eq!(in_range.len(), 2);
+        assert_eq!(in_range.first().unwrap().kind(), TokenKind::Colon);
+        assert_eq!(in_range.last().unwrap().kind(), TokenKind::Newline);
+    }
+
+    #[test]
+    #[should_panic(expected = "Offset 5 is inside a token range 4..7")]
+    fn tokens_in_range_start_offset_inside_token() {
+        let tokens = new_tokens(TEST_CASE_WITH_GAP.into_iter());
+        tokens.in_range(TextRange::new(5.into(), 10.into()));
+    }
+
+    #[test]
+    #[should_panic(expected = "Offset 6 is inside a token range 4..7")]
+    fn tokens_in_range_end_offset_inside_token() {
+        let tokens = new_tokens(TEST_CASE_WITH_GAP.into_iter());
+        tokens.in_range(TextRange::new(0.into(), 6.into()));
+    }
+}

crates/ruff_python_ast_integration_tests/tests/parentheses.rs

@@ -0,0 +1,199 @@
+//! Tests for [`ruff_python_ast::tokens::parentheses_iterator`] and
+//! [`ruff_python_ast::tokens::parenthesized_range`].
+
+use ruff_python_ast::{
+    self as ast, Expr,
+    token::{parentheses_iterator, parenthesized_range},
+};
+use ruff_python_parser::parse_module;
+
+#[test]
+fn test_no_parentheses() {
+    let source = "x = 2 + 2";
+    let parsed = parse_module(source).expect("should parse valid python");
+    let tokens = parsed.tokens();
+    let module = parsed.syntax();
+
+    let stmt = module.body.first().expect("module should have a statement");
+    let ast::Stmt::Assign(assign) = stmt else {
+        panic!("expected `Assign` statement, got {stmt:?}");
+    };
+
+    let result = parenthesized_range(assign.value.as_ref().into(), stmt.into(), tokens);
+    assert_eq!(result, None);
+}
+
+#[test]
+fn test_single_parentheses() {
+    let source = "x = (2 + 2)";
+    let parsed = parse_module(source).expect("should parse valid python");
+    let tokens = parsed.tokens();
+    let module = parsed.syntax();
+
+    let stmt = module.body.first().expect("module should have a statement");
+    let ast::Stmt::Assign(assign) = stmt else {
+        panic!("expected `Assign` statement, got {stmt:?}");
+    };
+
+    let result = parenthesized_range(assign.value.as_ref().into(), stmt.into(), tokens);
+    let range = result.expect("should find parentheses");
+    assert_eq!(&source[range], "(2 + 2)");
+}
+
+#[test]
+fn test_double_parentheses() {
+    let source = "x = ((2 + 2))";
+    let parsed = parse_module(source).expect("should parse valid python");
+    let tokens = parsed.tokens();
+    let module = parsed.syntax();
+
+    let stmt = module.body.first().expect("module should have a statement");
+    let ast::Stmt::Assign(assign) = stmt else {
+        panic!("expected `Assign` statement, got {stmt:?}");
+    };
+
+    let result = parenthesized_range(assign.value.as_ref().into(), stmt.into(), tokens);
+    let range = result.expect("should find parentheses");
+    assert_eq!(&source[range], "((2 + 2))");
+}
+
+#[test]
+fn test_parentheses_with_whitespace() {
+    let source = "x = (  2 + 2  )";
+    let parsed = parse_module(source).expect("should parse valid python");
+    let tokens = parsed.tokens();
+    let module = parsed.syntax();
+
+    let stmt = module.body.first().expect("module should have a statement");
+    let ast::Stmt::Assign(assign) = stmt else {
+        panic!("expected `Assign` statement, got {stmt:?}");
+    };
+
+    let result = parenthesized_range(assign.value.as_ref().into(), stmt.into(), tokens);
+    let range = result.expect("should find parentheses");
+    assert_eq!(&source[range], "(  2 + 2  )");
+}
+
+#[test]
+fn test_parentheses_with_comments() {
+    let source = "x = ( # comment\n    2 + 2\n)";
+    let parsed = parse_module(source).expect("should parse valid python");
+    let tokens = parsed.tokens();
+    let module = parsed.syntax();
+
+    let stmt = module.body.first().expect("module should have a statement");
+    let ast::Stmt::Assign(assign) = stmt else {
+        panic!("expected `Assign` statement, got {stmt:?}");
+    };
+
+    let result = parenthesized_range(assign.value.as_ref().into(), stmt.into(), tokens);
+    let range = result.expect("should find parentheses");
+    assert_eq!(&source[range], "( # comment\n    2 + 2\n)");
+}
+
+#[test]
+fn test_parenthesized_range_multiple() {
+    let source = "x = (((2 + 2)))";
+    let parsed = parse_module(source).expect("should parse valid python");
+    let tokens = parsed.tokens();
+    let module = parsed.syntax();
+
+    let stmt = module.body.first().expect("module should have a statement");
+    let ast::Stmt::Assign(assign) = stmt else {
+        panic!("expected `Assign` statement, got {stmt:?}");
+    };
+
+    let result = parenthesized_range(assign.value.as_ref().into(), stmt.into(), tokens);
+    let range = result.expect("should find parentheses");
+    assert_eq!(&source[range], "(((2 + 2)))");
+}
+
+#[test]
+fn test_parentheses_iterator_multiple() {
+    let source = "x = (((2 + 2)))";
+    let parsed = parse_module(source).expect("should parse valid python");
+    let tokens = parsed.tokens();
+    let module = parsed.syntax();
+
+    let stmt = module.body.first().expect("module should have a statement");
+    let ast::Stmt::Assign(assign) = stmt else {
+        panic!("expected `Assign` statement, got {stmt:?}");
+    };
+
+    let ranges: Vec<_> =
+        parentheses_iterator(assign.value.as_ref().into(), Some(stmt.into()), tokens).collect();
+    assert_eq!(ranges.len(), 3);
+    assert_eq!(&source[ranges[0]], "(2 + 2)");
+    assert_eq!(&source[ranges[1]], "((2 + 2))");
+    assert_eq!(&source[ranges[2]], "(((2 + 2)))");
+}
+
+#[test]
+fn test_call_arguments_not_counted() {
+    let source = "f(x)";
+    let parsed = parse_module(source).expect("should parse valid python");
+    let tokens = parsed.tokens();
+    let module = parsed.syntax();
+
+    let stmt = module.body.first().expect("module should have a statement");
+    let ast::Stmt::Expr(expr_stmt) = stmt else {
+        panic!("expected `Expr` statement, got {stmt:?}");
+    };
+
+    let Expr::Call(call) = expr_stmt.value.as_ref() else {
+        panic!("expected Call expression, got {:?}", expr_stmt.value);
+    };
+
+    let arg = call
+        .arguments
+        .args
+        .first()
+        .expect("call should have an argument");
+    let result = parenthesized_range(arg.into(), (&call.arguments).into(), tokens);
+    // The parentheses belong to the call, not the argument
+    assert_eq!(result, None);
+}
+
+#[test]
+fn test_call_with_parenthesized_argument() {
+    let source = "f((x))";
+    let parsed = parse_module(source).expect("should parse valid python");
+    let tokens = parsed.tokens();
+    let module = parsed.syntax();
+
+    let stmt = module.body.first().expect("module should have a statement");
+    let ast::Stmt::Expr(expr_stmt) = stmt else {
+        panic!("expected Expr statement, got {stmt:?}");
+    };
+
+    let Expr::Call(call) = expr_stmt.value.as_ref() else {
+        panic!("expected `Call` expression, got {:?}", expr_stmt.value);
+    };
+
+    let arg = call
+        .arguments
+        .args
+        .first()
+        .expect("call should have an argument");
+    let result = parenthesized_range(arg.into(), (&call.arguments).into(), tokens);
+
+    let range = result.expect("should find parentheses around argument");
+    assert_eq!(&source[range], "(x)");
+}
+
+#[test]
+fn test_multiline_with_parentheses() {
+    let source = "x = (\n    2 + 2 + 2\n)";
+    let parsed = parse_module(source).expect("should parse valid python");
+    let tokens = parsed.tokens();
+    let module = parsed.syntax();
+
+    let stmt = module.body.first().expect("module should have a statement");
+    let ast::Stmt::Assign(assign) = stmt else {
+        panic!("expected `Assign` statement, got {stmt:?}");
+    };
+
+    let result = parenthesized_range(assign.value.as_ref().into(), stmt.into(), tokens);
+    let range = result.expect("should find parentheses");
+    assert_eq!(&source[range], "(\n    2 + 2 + 2\n)");
+}

crates/ruff_python_codegen/src/stylist.rs

@@ -5,7 +5,7 @@ use std::cell::OnceCell;
 use std::ops::Deref;
 
 use ruff_python_ast::str::Quote;
-use ruff_python_parser::{Token, TokenKind, Tokens};
+use ruff_python_ast::token::{Token, TokenKind, Tokens};
 use ruff_source_file::{LineEnding, LineRanges, find_newline};
 use ruff_text_size::Ranged;
 

crates/ruff_python_formatter/src/context.rs

@@ -3,7 +3,7 @@ use std::ops::{Deref, DerefMut};
 
 use ruff_formatter::{Buffer, FormatContext, GroupId, IndentWidth, SourceCode};
 use ruff_python_ast::str::Quote;
-use ruff_python_parser::Tokens;
+use ruff_python_ast::token::Tokens;
 
 use crate::PyFormatOptions;
 use crate::comments::Comments;

crates/ruff_python_formatter/src/verbatim.rs

@@ -5,7 +5,7 @@ use std::slice::Iter;
 use ruff_formatter::{FormatError, write};
 use ruff_python_ast::AnyNodeRef;
 use ruff_python_ast::Stmt;
-use ruff_python_parser::{self as parser, TokenKind};
+use ruff_python_ast::token::{Token as AstToken, TokenKind};
 use ruff_python_trivia::lines_before;
 use ruff_source_file::LineRanges;
 use ruff_text_size::{Ranged, TextRange, TextSize};
@@ -770,7 +770,7 @@ impl Format<PyFormatContext<'_>> for FormatVerbatimStatementRange {
 }
 
 struct LogicalLinesIter<'a> {
-    tokens: Iter<'a, parser::Token>,
+    tokens: Iter<'a, AstToken>,
     // The end of the last logical line
     last_line_end: TextSize,
     // The position where the content to lex ends.
@@ -778,7 +778,7 @@ struct LogicalLinesIter<'a> {
 }
 
 impl<'a> LogicalLinesIter<'a> {
-    fn new(tokens: Iter<'a, parser::Token>, verbatim_range: TextRange) -> Self {
+    fn new(tokens: Iter<'a, AstToken>, verbatim_range: TextRange) -> Self {
         Self {
             tokens,
             last_line_end: verbatim_range.start(),

crates/ruff_python_importer/Cargo.toml

@@ -14,7 +14,6 @@ license = { workspace = true }
 ruff_diagnostics = { workspace = true }
 ruff_python_ast = { workspace = true }
 ruff_python_codegen = { workspace = true }
-ruff_python_parser = { workspace = true }
 ruff_python_trivia = { workspace = true }
 ruff_source_file = { workspace = true, features = ["serde"] }
 ruff_text_size = { workspace = true }
@@ -22,6 +21,8 @@ ruff_text_size = { workspace = true }
 anyhow = { workspace = true }
 
 [dev-dependencies]
+ruff_python_parser = { workspace = true }
+
 insta = { workspace = true }
 
 [features]

crates/ruff_python_importer/src/insertion.rs

@@ -5,8 +5,8 @@ use std::ops::Add;
 use ruff_diagnostics::Edit;
 use ruff_python_ast::Stmt;
 use ruff_python_ast::helpers::is_docstring_stmt;
+use ruff_python_ast::token::{TokenKind, Tokens};
 use ruff_python_codegen::Stylist;
-use ruff_python_parser::{TokenKind, Tokens};
 use ruff_python_trivia::is_python_whitespace;
 use ruff_python_trivia::{PythonWhitespace, textwrap::indent};
 use ruff_source_file::{LineRanges, UniversalNewlineIterator};
@@ -194,7 +194,7 @@ impl<'a> Insertion<'a> {
             tokens
                 .before(at)
                 .last()
-                .map(ruff_python_parser::Token::kind),
+                .map(ruff_python_ast::token::Token::kind),
             Some(TokenKind::Import)
         ) {
             return None;

crates/ruff_python_index/Cargo.toml

@@ -15,12 +15,12 @@ doctest = false
 
 [dependencies]
 ruff_python_ast = { workspace = true }
-ruff_python_parser = { workspace = true }
 ruff_python_trivia = { workspace = true }
 ruff_source_file = { workspace = true }
 ruff_text_size = { workspace = true }
 
 [dev-dependencies]
+ruff_python_parser = { workspace = true }
 
 [lints]
 workspace = true

crates/ruff_python_index/src/indexer.rs

@@ -2,7 +2,7 @@
 //! are omitted from the AST (e.g., commented lines).
 
 use ruff_python_ast::Stmt;
-use ruff_python_parser::{TokenKind, Tokens};
+use ruff_python_ast::token::{TokenKind, Tokens};
 use ruff_python_trivia::{
     CommentRanges, has_leading_content, has_trailing_content, is_python_whitespace,
 };

crates/ruff_python_index/src/interpolated_string_ranges.rs

@@ -1,6 +1,6 @@
 use std::collections::BTreeMap;
 
-use ruff_python_parser::{Token, TokenKind};
+use ruff_python_ast::token::{Token, TokenKind};
 use ruff_text_size::{Ranged, TextRange, TextSize};
 
 /// Stores the ranges of all interpolated strings in a file sorted by [`TextRange::start`].

crates/ruff_python_index/src/multiline_ranges.rs

@@ -1,4 +1,4 @@
-use ruff_python_parser::{Token, TokenKind};
+use ruff_python_ast::token::{Token, TokenKind};
 use ruff_text_size::{Ranged, TextRange};
 
 /// Stores the range of all multiline strings in a file sorted by

crates/ruff_python_parser/resources/inline/err/type_parameter_default_order.py

@@ -0,0 +1,3 @@
+class C[T = int, U]: ...
+class C[T1, T2 = int, T3, T4]: ...
+type Alias[T = int, U] = ...

crates/ruff_python_parser/src/error.rs

@@ -1,9 +1,10 @@
 use std::fmt::{self, Display};
 
 use ruff_python_ast::PythonVersion;
+use ruff_python_ast::token::TokenKind;
 use ruff_text_size::{Ranged, TextRange};
 
-use crate::{TokenKind, string::InterpolatedStringKind};
+use crate::string::InterpolatedStringKind;
 
 /// Represents represent errors that occur during parsing and are
 /// returned by the `parse_*` functions.

crates/ruff_python_parser/src/lexer.rs

@@ -14,6 +14,7 @@ use unicode_normalization::UnicodeNormalization;
 
 use ruff_python_ast::name::Name;
 use ruff_python_ast::str_prefix::{AnyStringPrefix, StringLiteralPrefix};
+use ruff_python_ast::token::{TokenFlags, TokenKind};
 use ruff_python_ast::{Int, IpyEscapeKind, StringFlags};
 use ruff_python_trivia::is_python_whitespace;
 use ruff_text_size::{TextLen, TextRange, TextSize};
@@ -26,7 +27,7 @@ use crate::lexer::interpolated_string::{
     InterpolatedStringContext, InterpolatedStrings, InterpolatedStringsCheckpoint,
 };
 use crate::string::InterpolatedStringKind;
-use crate::token::{TokenFlags, TokenKind, TokenValue};
+use crate::token::TokenValue;
 
 mod cursor;
 mod indentation;

crates/ruff_python_parser/src/lib.rs

@@ -63,23 +63,20 @@
 //! [lexical analysis]: https://en.wikipedia.org/wiki/Lexical_analysis
 //! [parsing]: https://en.wikipedia.org/wiki/Parsing
 //! [lexer]: crate::lexer
-use std::iter::FusedIterator;
-use std::ops::Deref;
 
 pub use crate::error::{
     InterpolatedStringErrorType, LexicalErrorType, ParseError, ParseErrorType,
     UnsupportedSyntaxError, UnsupportedSyntaxErrorKind,
 };
 pub use crate::parser::ParseOptions;
-pub use crate::token::{Token, TokenKind};
 
 use crate::parser::Parser;
 
+use ruff_python_ast::token::Tokens;
 use ruff_python_ast::{
     Expr, Mod, ModExpression, ModModule, PySourceType, StringFlags, StringLiteral, Suite,
 };
-use ruff_python_trivia::CommentRanges;
-use ruff_text_size::{Ranged, TextRange, TextSize};
+use ruff_text_size::{Ranged, TextRange};
 
 mod error;
 pub mod lexer;
@@ -473,351 +470,6 @@ impl Parsed<ModExpression> {
     }
 }
 
-/// Tokens represents a vector of lexed [`Token`].
-#[derive(Debug, Clone, PartialEq, Eq, get_size2::GetSize)]
-pub struct Tokens {
-    raw: Vec<Token>,
-}
-
-impl Tokens {
-    pub(crate) fn new(tokens: Vec<Token>) -> Tokens {
-        Tokens { raw: tokens }
-    }
-
-    /// Returns an iterator over all the tokens that provides context.
-    pub fn iter_with_context(&self) -> TokenIterWithContext<'_> {
-        TokenIterWithContext::new(&self.raw)
-    }
-
-    /// Performs a binary search to find the index of the **first** token that starts at the given `offset`.
-    ///
-    /// Unlike `binary_search_by_key`, this method ensures that if multiple tokens start at the same offset,
-    /// it returns the index of the first one. Multiple tokens can start at the same offset in cases where
-    /// zero-length tokens are involved (like `Dedent` or `Newline` at the end of the file).
-    pub fn binary_search_by_start(&self, offset: TextSize) -> Result<usize, usize> {
-        let partition_point = self.partition_point(|token| token.start() < offset);
-
-        let after = &self[partition_point..];
-
-        if after.first().is_some_and(|first| first.start() == offset) {
-            Ok(partition_point)
-        } else {
-            Err(partition_point)
-        }
-    }
-
-    /// Returns a slice of [`Token`] that are within the given `range`.
-    ///
-    /// The start and end offset of the given range should be either:
-    /// 1. Token boundary
-    /// 2. Gap between the tokens
-    ///
-    /// For example, considering the following tokens and their corresponding range:
-    ///
-    /// | Token               | Range     |
-    /// |---------------------|-----------|
-    /// | `Def`               | `0..3`    |
-    /// | `Name`              | `4..7`    |
-    /// | `Lpar`              | `7..8`    |
-    /// | `Rpar`              | `8..9`    |
-    /// | `Colon`             | `9..10`   |
-    /// | `Newline`           | `10..11`  |
-    /// | `Comment`           | `15..24`  |
-    /// | `NonLogicalNewline` | `24..25`  |
-    /// | `Indent`            | `25..29`  |
-    /// | `Pass`              | `29..33`  |
-    ///
-    /// Here, for (1) a token boundary is considered either the start or end offset of any of the
-    /// above tokens. For (2), the gap would be any offset between the `Newline` and `Comment`
-    /// token which are 12, 13, and 14.
-    ///
-    /// Examples:
-    /// 1) `4..10` would give `Name`, `Lpar`, `Rpar`, `Colon`
-    /// 2) `11..25` would give `Comment`, `NonLogicalNewline`
-    /// 3) `12..25` would give same as (2) and offset 12 is in the "gap"
-    /// 4) `9..12` would give `Colon`, `Newline` and offset 12 is in the "gap"
-    /// 5) `18..27` would panic because both the start and end offset is within a token
-    ///
-    /// ## Note
-    ///
-    /// The returned slice can contain the [`TokenKind::Unknown`] token if there was a lexical
-    /// error encountered within the given range.
-    ///
-    /// # Panics
-    ///
-    /// If either the start or end offset of the given range is within a token range.
-    pub fn in_range(&self, range: TextRange) -> &[Token] {
-        let tokens_after_start = self.after(range.start());
-
-        Self::before_impl(tokens_after_start, range.end())
-    }
-
-    /// Searches the token(s) at `offset`.
-    ///
-    /// Returns [`TokenAt::Between`] if `offset` points directly inbetween two tokens
-    /// (the left token ends at `offset` and the right token starts at `offset`).
-    ///
-    ///
-    /// ## Examples
-    ///
-    /// [Playground](https://play.ruff.rs/f3ad0a55-5931-4a13-96c7-b2b8bfdc9a2e?secondary=Tokens)
-    ///
-    /// ```
-    /// # use ruff_python_ast::PySourceType;
-    /// # use ruff_python_parser::{Token, TokenAt, TokenKind};
-    /// # use ruff_text_size::{Ranged, TextSize};
-    ///
-    /// let source = r#"
-    /// def test(arg):
-    ///     arg.call()
-    ///     if True:
-    ///         pass
-    ///     print("true")
-    /// "#.trim();
-    ///
-    /// let parsed = ruff_python_parser::parse_unchecked_source(source, PySourceType::Python);
-    /// let tokens = parsed.tokens();
-    ///
-    /// let collect_tokens = |offset: TextSize| {
-    ///     tokens.at_offset(offset).into_iter().map(|t| (t.kind(), &source[t.range()])).collect::<Vec<_>>()
-    /// };
-    ///
-    /// assert_eq!(collect_tokens(TextSize::new(4)), vec! [(TokenKind::Name, "test")]);
-    /// assert_eq!(collect_tokens(TextSize::new(6)), vec! [(TokenKind::Name, "test")]);
-    /// // between `arg` and `.`
-    /// assert_eq!(collect_tokens(TextSize::new(22)), vec! [(TokenKind::Name, "arg"), (TokenKind::Dot, ".")]);
-    /// assert_eq!(collect_tokens(TextSize::new(36)), vec! [(TokenKind::If, "if")]);
-    /// // Before the dedent token
-    /// assert_eq!(collect_tokens(TextSize::new(57)), vec! []);
-    /// ```
-    pub fn at_offset(&self, offset: TextSize) -> TokenAt {
-        match self.binary_search_by_start(offset) {
-            // The token at `index` starts exactly at `offset.
-            // ```python
-            // object.attribute
-            //        ^ OFFSET
-            // ```
-            Ok(index) => {
-                let token = self[index];
-                // `token` starts exactly at `offset`. Test if the offset is right between
-                // `token` and the previous token (if there's any)
-                if let Some(previous) = index.checked_sub(1).map(|idx| self[idx]) {
-                    if previous.end() == offset {
-                        return TokenAt::Between(previous, token);
-                    }
-                }
-
-                TokenAt::Single(token)
-            }
-
-            // No token found that starts exactly at the given offset. But it's possible that
-            // the token starting before `offset` fully encloses `offset` (it's end range ends after `offset`).
-            // ```python
-            // object.attribute
-            //   ^ OFFSET
-            // # or
-            // if True:
-            //     print("test")
-            //  ^ OFFSET
-            // ```
-            Err(index) => {
-                if let Some(previous) = index.checked_sub(1).map(|idx| self[idx]) {
-                    if previous.range().contains_inclusive(offset) {
-                        return TokenAt::Single(previous);
-                    }
-                }
-
-                TokenAt::None
-            }
-        }
-    }
-
-    /// Returns a slice of tokens before the given [`TextSize`] offset.
-    ///
-    /// If the given offset is between two tokens, the returned slice will end just before the
-    /// following token. In other words, if the offset is between the end of previous token and
-    /// start of next token, the returned slice will end just before the next token.
-    ///
-    /// # Panics
-    ///
-    /// If the given offset is inside a token range at any point
-    /// other than the start of the range.
-    pub fn before(&self, offset: TextSize) -> &[Token] {
-        Self::before_impl(&self.raw, offset)
-    }
-
-    fn before_impl(tokens: &[Token], offset: TextSize) -> &[Token] {
-        let partition_point = tokens.partition_point(|token| token.start() < offset);
-        let before = &tokens[..partition_point];
-
-        if let Some(last) = before.last() {
-            // If it's equal to the end offset, then it's at a token boundary which is
-            // valid. If it's greater than the end offset, then it's in the gap between
-            // the tokens which is valid as well.
-            assert!(
-                offset >= last.end(),
-                "Offset {:?} is inside a token range {:?}",
-                offset,
-                last.range()
-            );
-        }
-        before
-    }
-
-    /// Returns a slice of tokens after the given [`TextSize`] offset.
-    ///
-    /// If the given offset is between two tokens, the returned slice will start from the following
-    /// token. In other words, if the offset is between the end of previous token and start of next
-    /// token, the returned slice will start from the next token.
-    ///
-    /// # Panics
-    ///
-    /// If the given offset is inside a token range at any point
-    /// other than the start of the range.
-    pub fn after(&self, offset: TextSize) -> &[Token] {
-        let partition_point = self.partition_point(|token| token.end() <= offset);
-        let after = &self[partition_point..];
-
-        if let Some(first) = after.first() {
-            // valid. If it's greater than the end offset, then it's in the gap between
-            // the tokens which is valid as well.
-            assert!(
-                offset <= first.start(),
-                "Offset {:?} is inside a token range {:?}",
-                offset,
-                first.range()
-            );
-        }
-
-        after
-    }
-}
-
-impl<'a> IntoIterator for &'a Tokens {
-    type Item = &'a Token;
-    type IntoIter = std::slice::Iter<'a, Token>;
-
-    fn into_iter(self) -> Self::IntoIter {
-        self.iter()
-    }
-}
-
-impl Deref for Tokens {
-    type Target = [Token];
-
-    fn deref(&self) -> &Self::Target {
-        &self.raw
-    }
-}
-
-/// A token that encloses a given offset or ends exactly at it.
-#[derive(Debug, Clone)]
-pub enum TokenAt {
-    /// There's no token at the given offset
-    None,
-
-    /// There's a single token at the given offset.
-    Single(Token),
-
-    /// The offset falls exactly between two tokens. E.g. `CURSOR` in `call<CURSOR>(arguments)` is
-    /// positioned exactly between the `call` and `(` tokens.
-    Between(Token, Token),
-}
-
-impl Iterator for TokenAt {
-    type Item = Token;
-
-    fn next(&mut self) -> Option<Self::Item> {
-        match *self {
-            TokenAt::None => None,
-            TokenAt::Single(token) => {
-                *self = TokenAt::None;
-                Some(token)
-            }
-            TokenAt::Between(first, second) => {
-                *self = TokenAt::Single(second);
-                Some(first)
-            }
-        }
-    }
-}
-
-impl FusedIterator for TokenAt {}
-
-impl From<&Tokens> for CommentRanges {
-    fn from(tokens: &Tokens) -> Self {
-        let mut ranges = vec![];
-        for token in tokens {
-            if token.kind() == TokenKind::Comment {
-                ranges.push(token.range());
-            }
-        }
-        CommentRanges::new(ranges)
-    }
-}
-
-/// An iterator over the [`Token`]s with context.
-///
-/// This struct is created by the [`iter_with_context`] method on [`Tokens`]. Refer to its
-/// documentation for more details.
-///
-/// [`iter_with_context`]: Tokens::iter_with_context
-#[derive(Debug, Clone)]
-pub struct TokenIterWithContext<'a> {
-    inner: std::slice::Iter<'a, Token>,
-    nesting: u32,
-}
-
-impl<'a> TokenIterWithContext<'a> {
-    fn new(tokens: &'a [Token]) -> TokenIterWithContext<'a> {
-        TokenIterWithContext {
-            inner: tokens.iter(),
-            nesting: 0,
-        }
-    }
-
-    /// Return the nesting level the iterator is currently in.
-    pub const fn nesting(&self) -> u32 {
-        self.nesting
-    }
-
-    /// Returns `true` if the iterator is within a parenthesized context.
-    pub const fn in_parenthesized_context(&self) -> bool {
-        self.nesting > 0
-    }
-
-    /// Returns the next [`Token`] in the iterator without consuming it.
-    pub fn peek(&self) -> Option<&'a Token> {
-        self.clone().next()
-    }
-}
-
-impl<'a> Iterator for TokenIterWithContext<'a> {
-    type Item = &'a Token;
-
-    fn next(&mut self) -> Option<Self::Item> {
-        let token = self.inner.next()?;
-
-        match token.kind() {
-            TokenKind::Lpar | TokenKind::Lbrace | TokenKind::Lsqb => self.nesting += 1,
-            TokenKind::Rpar | TokenKind::Rbrace | TokenKind::Rsqb => {
-                self.nesting = self.nesting.saturating_sub(1);
-            }
-            // This mimics the behavior of re-lexing which reduces the nesting level on the lexer.
-            // We don't need to reduce it by 1 because unlike the lexer we see the final token
-            // after recovering from every unclosed parenthesis.
-            TokenKind::Newline if self.nesting > 0 => {
-                self.nesting = 0;
-            }
-            _ => {}
-        }
-
-        Some(token)
-    }
-}
-
-impl FusedIterator for TokenIterWithContext<'_> {}
-
 /// Control in the different modes by which a source file can be parsed.
 ///
 /// The mode argument specifies in what way code must be parsed.
@@ -888,204 +540,3 @@ impl std::fmt::Display for ModeParseError {
         write!(f, r#"mode must be "exec", "eval", "ipython", or "single""#)
     }
 }
-
-#[cfg(test)]
-mod tests {
-    use std::ops::Range;
-
-    use crate::token::TokenFlags;
-
-    use super::*;
-
-    /// Test case containing a "gap" between two tokens.
-    ///
-    /// Code: <https://play.ruff.rs/a3658340-6df8-42c5-be80-178744bf1193>
-    const TEST_CASE_WITH_GAP: [(TokenKind, Range<u32>); 10] = [
-        (TokenKind::Def, 0..3),
-        (TokenKind::Name, 4..7),
-        (TokenKind::Lpar, 7..8),
-        (TokenKind::Rpar, 8..9),
-        (TokenKind::Colon, 9..10),
-        (TokenKind::Newline, 10..11),
-        // Gap               ||..||
-        (TokenKind::Comment, 15..24),
-        (TokenKind::NonLogicalNewline, 24..25),
-        (TokenKind::Indent, 25..29),
-        (TokenKind::Pass, 29..33),
-        // No newline at the end to keep the token set full of unique tokens
-    ];
-
-    /// Helper function to create [`Tokens`] from an iterator of (kind, range).
-    fn new_tokens(tokens: impl Iterator<Item = (TokenKind, Range<u32>)>) -> Tokens {
-        Tokens::new(
-            tokens
-                .map(|(kind, range)| {
-                    Token::new(
-                        kind,
-                        TextRange::new(TextSize::new(range.start), TextSize::new(range.end)),
-                        TokenFlags::empty(),
-                    )
-                })
-                .collect(),
-        )
-    }
-
-    #[test]
-    fn tokens_after_offset_at_token_start() {
-        let tokens = new_tokens(TEST_CASE_WITH_GAP.into_iter());
-        let after = tokens.after(TextSize::new(8));
-        assert_eq!(after.len(), 7);
-        assert_eq!(after.first().unwrap().kind(), TokenKind::Rpar);
-    }
-
-    #[test]
-    fn tokens_after_offset_at_token_end() {
-        let tokens = new_tokens(TEST_CASE_WITH_GAP.into_iter());
-        let after = tokens.after(TextSize::new(11));
-        assert_eq!(after.len(), 4);
-        assert_eq!(after.first().unwrap().kind(), TokenKind::Comment);
-    }
-
-    #[test]
-    fn tokens_after_offset_between_tokens() {
-        let tokens = new_tokens(TEST_CASE_WITH_GAP.into_iter());
-        let after = tokens.after(TextSize::new(13));
-        assert_eq!(after.len(), 4);
-        assert_eq!(after.first().unwrap().kind(), TokenKind::Comment);
-    }
-
-    #[test]
-    fn tokens_after_offset_at_last_token_end() {
-        let tokens = new_tokens(TEST_CASE_WITH_GAP.into_iter());
-        let after = tokens.after(TextSize::new(33));
-        assert_eq!(after.len(), 0);
-    }
-
-    #[test]
-    #[should_panic(expected = "Offset 5 is inside a token range 4..7")]
-    fn tokens_after_offset_inside_token() {
-        let tokens = new_tokens(TEST_CASE_WITH_GAP.into_iter());
-        tokens.after(TextSize::new(5));
-    }
-
-    #[test]
-    fn tokens_before_offset_at_first_token_start() {
-        let tokens = new_tokens(TEST_CASE_WITH_GAP.into_iter());
-        let before = tokens.before(TextSize::new(0));
-        assert_eq!(before.len(), 0);
-    }
-
-    #[test]
-    fn tokens_before_offset_after_first_token_gap() {
-        let tokens = new_tokens(TEST_CASE_WITH_GAP.into_iter());
-        let before = tokens.before(TextSize::new(3));
-        assert_eq!(before.len(), 1);
-        assert_eq!(before.last().unwrap().kind(), TokenKind::Def);
-    }
-
-    #[test]
-    fn tokens_before_offset_at_second_token_start() {
-        let tokens = new_tokens(TEST_CASE_WITH_GAP.into_iter());
-        let before = tokens.before(TextSize::new(4));
-        assert_eq!(before.len(), 1);
-        assert_eq!(before.last().unwrap().kind(), TokenKind::Def);
-    }
-
-    #[test]
-    fn tokens_before_offset_at_token_start() {
-        let tokens = new_tokens(TEST_CASE_WITH_GAP.into_iter());
-        let before = tokens.before(TextSize::new(8));
-        assert_eq!(before.len(), 3);
-        assert_eq!(before.last().unwrap().kind(), TokenKind::Lpar);
-    }
-
-    #[test]
-    fn tokens_before_offset_at_token_end() {
-        let tokens = new_tokens(TEST_CASE_WITH_GAP.into_iter());
-        let before = tokens.before(TextSize::new(11));
-        assert_eq!(before.len(), 6);
-        assert_eq!(before.last().unwrap().kind(), TokenKind::Newline);
-    }
-
-    #[test]
-    fn tokens_before_offset_between_tokens() {
-        let tokens = new_tokens(TEST_CASE_WITH_GAP.into_iter());
-        let before = tokens.before(TextSize::new(13));
-        assert_eq!(before.len(), 6);
-        assert_eq!(before.last().unwrap().kind(), TokenKind::Newline);
-    }
-
-    #[test]
-    fn tokens_before_offset_at_last_token_end() {
-        let tokens = new_tokens(TEST_CASE_WITH_GAP.into_iter());
-        let before = tokens.before(TextSize::new(33));
-        assert_eq!(before.len(), 10);
-        assert_eq!(before.last().unwrap().kind(), TokenKind::Pass);
-    }
-
-    #[test]
-    #[should_panic(expected = "Offset 5 is inside a token range 4..7")]
-    fn tokens_before_offset_inside_token() {
-        let tokens = new_tokens(TEST_CASE_WITH_GAP.into_iter());
-        tokens.before(TextSize::new(5));
-    }
-
-    #[test]
-    fn tokens_in_range_at_token_offset() {
-        let tokens = new_tokens(TEST_CASE_WITH_GAP.into_iter());
-        let in_range = tokens.in_range(TextRange::new(4.into(), 10.into()));
-        assert_eq!(in_range.len(), 4);
-        assert_eq!(in_range.first().unwrap().kind(), TokenKind::Name);
-        assert_eq!(in_range.last().unwrap().kind(), TokenKind::Colon);
-    }
-
-    #[test]
-    fn tokens_in_range_start_offset_at_token_end() {
-        let tokens = new_tokens(TEST_CASE_WITH_GAP.into_iter());
-        let in_range = tokens.in_range(TextRange::new(11.into(), 29.into()));
-        assert_eq!(in_range.len(), 3);
-        assert_eq!(in_range.first().unwrap().kind(), TokenKind::Comment);
-        assert_eq!(in_range.last().unwrap().kind(), TokenKind::Indent);
-    }
-
-    #[test]
-    fn tokens_in_range_end_offset_at_token_start() {
-        let tokens = new_tokens(TEST_CASE_WITH_GAP.into_iter());
-        let in_range = tokens.in_range(TextRange::new(8.into(), 15.into()));
-        assert_eq!(in_range.len(), 3);
-        assert_eq!(in_range.first().unwrap().kind(), TokenKind::Rpar);
-        assert_eq!(in_range.last().unwrap().kind(), TokenKind::Newline);
-    }
-
-    #[test]
-    fn tokens_in_range_start_offset_between_tokens() {
-        let tokens = new_tokens(TEST_CASE_WITH_GAP.into_iter());
-        let in_range = tokens.in_range(TextRange::new(13.into(), 29.into()));
-        assert_eq!(in_range.len(), 3);
-        assert_eq!(in_range.first().unwrap().kind(), TokenKind::Comment);
-        assert_eq!(in_range.last().unwrap().kind(), TokenKind::Indent);
-    }
-
-    #[test]
-    fn tokens_in_range_end_offset_between_tokens() {
-        let tokens = new_tokens(TEST_CASE_WITH_GAP.into_iter());
-        let in_range = tokens.in_range(TextRange::new(9.into(), 13.into()));
-        assert_eq!(in_range.len(), 2);
-        assert_eq!(in_range.first().unwrap().kind(), TokenKind::Colon);
-        assert_eq!(in_range.last().unwrap().kind(), TokenKind::Newline);
-    }
-
-    #[test]
-    #[should_panic(expected = "Offset 5 is inside a token range 4..7")]
-    fn tokens_in_range_start_offset_inside_token() {
-        let tokens = new_tokens(TEST_CASE_WITH_GAP.into_iter());
-        tokens.in_range(TextRange::new(5.into(), 10.into()));
-    }
-
-    #[test]
-    #[should_panic(expected = "Offset 6 is inside a token range 4..7")]
-    fn tokens_in_range_end_offset_inside_token() {
-        let tokens = new_tokens(TEST_CASE_WITH_GAP.into_iter());
-        tokens.in_range(TextRange::new(0.into(), 6.into()));
-    }
-}

crates/ruff_python_parser/src/parser/expression.rs

@@ -4,6 +4,7 @@ use bitflags::bitflags;
 use rustc_hash::{FxBuildHasher, FxHashSet};
 
 use ruff_python_ast::name::Name;
+use ruff_python_ast::token::TokenKind;
 use ruff_python_ast::{
     self as ast, AnyStringFlags, AtomicNodeIndex, BoolOp, CmpOp, ConversionFlag, Expr, ExprContext,
     FString, InterpolatedStringElement, InterpolatedStringElements, IpyEscapeKind, Number,
@@ -18,7 +19,7 @@ use crate::string::{
     InterpolatedStringKind, StringType, parse_interpolated_string_literal_element,
     parse_string_literal,
 };
-use crate::token::{TokenKind, TokenValue};
+use crate::token::TokenValue;
 use crate::token_set::TokenSet;
 use crate::{
     InterpolatedStringErrorType, Mode, ParseErrorType, UnsupportedSyntaxError,