Add increment/decrement

## Summary

Ignoring all lines until the first logical line does not match the
behavior from pycodestyle. This PR therefore removes the `if
state.is_not_first_logical_line` skipping the line check before the
first logical line, and applies it only to `E302`.

For example, in the snippet below a rule violation should be detected on
the second comment and on the import.

```python
# first comment




# second comment




import foo
```

Fixes #10374

## Test Plan

Add test cases, update the snapshots and verify the ecosystem check output

Cargo.lock

@@ -2353,11 +2353,9 @@ dependencies = [
 name = "ruff_python_parser"
 version = "0.0.0"
 dependencies = [
- "annotate-snippets 0.9.2",
  "anyhow",
  "bitflags 2.4.2",
  "bstr",
- "drop_bomb",
  "insta",
  "is-macro",
  "itertools 0.12.1",
@@ -2365,7 +2363,6 @@ dependencies = [
  "lalrpop-util",
  "memchr",
  "ruff_python_ast",
- "ruff_source_file",
  "ruff_text_size",
  "rustc-hash",
  "static_assertions",

crates/ruff/tests/format.rs

@@ -523,7 +523,7 @@ from module import =
     ----- stdout -----
 
     ----- stderr -----
-    error: Failed to parse main.py:2:20: Unexpected token =
+    error: Failed to parse main.py:2:20: Unexpected token '='
     "###);
 
     Ok(())

crates/ruff/tests/integration_test.rs

@@ -731,11 +731,11 @@ fn stdin_parse_error() {
     success: false
     exit_code: 1
     ----- stdout -----
-    -:1:17: E999 SyntaxError: Unexpected token =
+    -:1:17: E999 SyntaxError: Unexpected token '='
     Found 1 error.
 
     ----- stderr -----
-    error: Failed to parse at 1:17: Unexpected token =
+    error: Failed to parse at 1:17: Unexpected token '='
     "###);
 }
 

crates/ruff/tests/snapshots/show_settings__display_default_settings.snap

@@ -52,7 +52,6 @@ file_resolver.exclude = [
 file_resolver.extend_exclude = [
 	"crates/ruff_linter/resources/",
 	"crates/ruff_python_formatter/resources/",
-	"crates/ruff_python_parser/resources/",
 ]
 file_resolver.force_exclude = false
 file_resolver.include = [

crates/ruff_benchmark/benches/formatter.rs

@@ -7,7 +7,7 @@ use ruff_benchmark::{TestCase, TestFile, TestFileDownloadError};
 use ruff_python_formatter::{format_module_ast, PreviewMode, PyFormatOptions};
 use ruff_python_index::CommentRangesBuilder;
 use ruff_python_parser::lexer::lex;
-use ruff_python_parser::{allocate_tokens_vec, parse_tokens, set_new_parser, Mode};
+use ruff_python_parser::{allocate_tokens_vec, parse_tokens, Mode};
 
 #[cfg(target_os = "windows")]
 #[global_allocator]
@@ -42,8 +42,6 @@ fn create_test_cases() -> Result<Vec<TestCase>, TestFileDownloadError> {
 }
 
 fn benchmark_formatter(criterion: &mut Criterion) {
-    set_new_parser(true);
-
     let mut group = criterion.benchmark_group("formatter");
     let test_cases = create_test_cases().unwrap();
 

crates/ruff_benchmark/benches/lexer.rs

@@ -2,7 +2,7 @@ use ruff_benchmark::criterion::{
     criterion_group, criterion_main, measurement::WallTime, BenchmarkId, Criterion, Throughput,
 };
 use ruff_benchmark::{TestCase, TestFile, TestFileDownloadError};
-use ruff_python_parser::{lexer, set_new_parser, Mode};
+use ruff_python_parser::{lexer, Mode};
 
 #[cfg(target_os = "windows")]
 #[global_allocator]
@@ -37,8 +37,6 @@ fn create_test_cases() -> Result<Vec<TestCase>, TestFileDownloadError> {
 }
 
 fn benchmark_lexer(criterion: &mut Criterion<WallTime>) {
-    set_new_parser(true);
-
     let test_cases = create_test_cases().unwrap();
     let mut group = criterion.benchmark_group("lexer");
 

crates/ruff_benchmark/benches/linter.rs

@@ -10,7 +10,7 @@ use ruff_linter::settings::{flags, LinterSettings};
 use ruff_linter::source_kind::SourceKind;
 use ruff_linter::{registry::Rule, RuleSelector};
 use ruff_python_ast::PySourceType;
-use ruff_python_parser::{lexer, parse_program_tokens, set_new_parser, Mode};
+use ruff_python_parser::{lexer, parse_program_tokens, Mode};
 
 #[cfg(target_os = "windows")]
 #[global_allocator]
@@ -45,8 +45,6 @@ fn create_test_cases() -> Result<Vec<TestCase>, TestFileDownloadError> {
 }
 
 fn benchmark_linter(mut group: BenchmarkGroup, settings: &LinterSettings) {
-    set_new_parser(true);
-
     let test_cases = create_test_cases().unwrap();
 
     for case in test_cases {

crates/ruff_benchmark/benches/parser.rs

@@ -4,7 +4,7 @@ use ruff_benchmark::criterion::{
 use ruff_benchmark::{TestCase, TestFile, TestFileDownloadError};
 use ruff_python_ast::statement_visitor::{walk_stmt, StatementVisitor};
 use ruff_python_ast::Stmt;
-use ruff_python_parser::{parse_suite, set_new_parser};
+use ruff_python_parser::parse_suite;
 
 #[cfg(target_os = "windows")]
 #[global_allocator]
@@ -50,8 +50,6 @@ impl<'a> StatementVisitor<'a> for CountVisitor {
 }
 
 fn benchmark_parser(criterion: &mut Criterion<WallTime>) {
-    set_new_parser(true);
-
     let test_cases = create_test_cases().unwrap();
     let mut group = criterion.benchmark_group("parser");
 

crates/ruff_linter/resources/test/fixtures/pycodestyle/E302_first_line_docstring.py

@@ -0,0 +1,4 @@
+"""Test where the error is after the module's docstring."""
+
+def fn():
+    pass

crates/ruff_linter/resources/test/fixtures/pycodestyle/E302_first_line_expression.py

@@ -0,0 +1,4 @@
+"Test where the first line is a comment, " + "and the rule violation follows it."
+
+def fn():
+    pass

crates/ruff_linter/resources/test/fixtures/pycodestyle/E302_first_line_function.py

@@ -0,0 +1,5 @@
+def fn1():
+    pass
+
+def fn2():
+    pass

crates/ruff_linter/resources/test/fixtures/pycodestyle/E302_first_line_statement.py

@@ -0,0 +1,4 @@
+print("Test where the first line is a statement, and the rule violation follows it.")
+
+def fn():
+    pass

crates/ruff_linter/resources/test/fixtures/pycodestyle/E303_first_line_comment.py

@@ -0,0 +1,6 @@
+# Test where the first line is a comment, and the rule violation follows it.
+
+
+
+def fn():
+    pass

crates/ruff_linter/resources/test/fixtures/pycodestyle/E303_first_line_docstring.py

@@ -0,0 +1,6 @@
+"""Test where the error is after the module's docstring."""
+
+
+
+def fn():
+    pass

crates/ruff_linter/resources/test/fixtures/pycodestyle/E303_first_line_expression.py

@@ -0,0 +1,6 @@
+"Test where the first line is a comment, " + "and the rule violation follows it."
+
+
+
+def fn():
+    pass

crates/ruff_linter/resources/test/fixtures/pycodestyle/E303_first_line_statement.py

@@ -0,0 +1,6 @@
+print("Test where the first line is a statement, and the rule violation follows it.")
+
+
+
+def fn():
+    pass

crates/ruff_linter/src/checkers/ast/analyze/expression.rs

@@ -254,7 +254,7 @@ pub(crate) fn expression(expr: &Expr, checker: &mut Checker) {
                         }
                     }
                 }
-                _ => {}
+                ExprContext::Del => {}
             }
             if checker.enabled(Rule::SixPY3) {
                 flake8_2020::rules::name_or_attribute(checker, expr);

crates/ruff_linter/src/checkers/ast/mod.rs

@@ -986,7 +986,6 @@ impl<'a> Visitor<'a> for Checker<'a> {
                 ExprContext::Load => self.handle_node_load(expr),
                 ExprContext::Store => self.handle_node_store(id, expr),
                 ExprContext::Del => self.handle_node_delete(expr),
-                ExprContext::Invalid => {}
             },
             _ => {}
         }

crates/ruff_linter/src/logging.rs

@@ -194,7 +194,7 @@ impl DisplayParseError {
         // Translate the byte offset to a location in the originating source.
         let location =
             if let Some(jupyter_index) = source_kind.as_ipy_notebook().map(Notebook::index) {
-                let source_location = source_code.source_location(error.location.start());
+                let source_location = source_code.source_location(error.offset);
 
                 ErrorLocation::Cell(
                     jupyter_index
@@ -208,7 +208,7 @@ impl DisplayParseError {
                     },
                 )
             } else {
-                ErrorLocation::File(source_code.source_location(error.location.start()))
+                ErrorLocation::File(source_code.source_location(error.offset))
             };
 
         Self {
@@ -275,7 +275,27 @@ impl<'a> DisplayParseErrorType<'a> {
 
 impl Display for DisplayParseErrorType<'_> {
     fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
-        write!(f, "{}", TruncateAtNewline(&self.0))
+        match self.0 {
+            ParseErrorType::Eof => write!(f, "Expected token but reached end of file."),
+            ParseErrorType::ExtraToken(ref tok) => write!(
+                f,
+                "Got extraneous token: {tok}",
+                tok = TruncateAtNewline(&tok)
+            ),
+            ParseErrorType::InvalidToken => write!(f, "Got invalid token"),
+            ParseErrorType::UnrecognizedToken(ref tok, ref expected) => {
+                if let Some(expected) = expected.as_ref() {
+                    write!(
+                        f,
+                        "Expected '{expected}', but got {tok}",
+                        tok = TruncateAtNewline(&tok)
+                    )
+                } else {
+                    write!(f, "Unexpected token {tok}", tok = TruncateAtNewline(&tok))
+                }
+            }
+            ParseErrorType::Lexical(ref error) => write!(f, "{error}"),
+        }
     }
 }
 

crates/ruff_linter/src/rules/flake8_bugbear/rules/function_uses_loop_variable.rs

@@ -67,7 +67,7 @@ impl<'a> Visitor<'a> for LoadedNamesVisitor<'a> {
             Expr::Name(name) => match &name.ctx {
                 ExprContext::Load => self.loaded.push(name),
                 ExprContext::Store => self.stored.push(name),
-                _ => {}
+                ExprContext::Del => {}
             },
             _ => visitor::walk_expr(self, expr),
         }

crates/ruff_linter/src/rules/isort/snapshots/ruff_linter__rules__isort__tests__bom_unsorted.py.snap

@@ -14,3 +14,5 @@ bom_unsorted.py:1:1: I001 [*] Import block is un-sorted or un-formatted
 2   |-import bar
   1 |+import bar
   2 |+import foo
+
+

crates/ruff_linter/src/rules/pycodestyle/mod.rs

@@ -171,6 +171,24 @@ mod tests {
         Ok(())
     }
 
+    #[test_case(Rule::BlankLinesTopLevel, Path::new("E302_first_line_docstring.py"))]
+    #[test_case(Rule::BlankLinesTopLevel, Path::new("E302_first_line_expression.py"))]
+    #[test_case(Rule::BlankLinesTopLevel, Path::new("E302_first_line_function.py"))]
+    #[test_case(Rule::BlankLinesTopLevel, Path::new("E302_first_line_statement.py"))]
+    #[test_case(Rule::TooManyBlankLines, Path::new("E303_first_line_comment.py"))]
+    #[test_case(Rule::TooManyBlankLines, Path::new("E303_first_line_docstring.py"))]
+    #[test_case(Rule::TooManyBlankLines, Path::new("E303_first_line_expression.py"))]
+    #[test_case(Rule::TooManyBlankLines, Path::new("E303_first_line_statement.py"))]
+    fn blank_lines_first_line(rule_code: Rule, path: &Path) -> Result<()> {
+        let snapshot = format!("{}_{}", rule_code.noqa_code(), path.to_string_lossy());
+        let diagnostics = test_path(
+            Path::new("pycodestyle").join(path).as_path(),
+            &settings::LinterSettings::for_rule(rule_code),
+        )?;
+        assert_messages!(snapshot, diagnostics);
+        Ok(())
+    }
+
     #[test_case(Rule::BlankLineBetweenMethods, Path::new("E30.py"))]
     #[test_case(Rule::BlankLinesTopLevel, Path::new("E30.py"))]
     #[test_case(Rule::TooManyBlankLines, Path::new("E30.py"))]

crates/ruff_linter/src/rules/pycodestyle/rules/blank_lines.rs

@@ -696,9 +696,7 @@ impl<'a> BlankLinesChecker<'a> {
             state.class_status.update(&logical_line);
             state.fn_status.update(&logical_line);
 
-            if state.is_not_first_logical_line {
-                self.check_line(&logical_line, &state, prev_indent_length, diagnostics);
-            }
+            self.check_line(&logical_line, &state, prev_indent_length, diagnostics);
 
             match logical_line.kind {
                 LogicalLineKind::Class => {
@@ -818,6 +816,8 @@ impl<'a> BlankLinesChecker<'a> {
             && line.kind.is_class_function_or_decorator()
             // Blank lines in stub files are used to group definitions. Don't enforce blank lines.
             && !self.source_type.is_stub()
+            // Do not expect blank lines before the first logical line.
+            && state.is_not_first_logical_line
         {
             // E302
             let mut diagnostic = Diagnostic::new(

crates/ruff_linter/src/rules/pycodestyle/rules/errors.rs

@@ -81,7 +81,7 @@ pub(crate) fn syntax_error(
     parse_error: &ParseError,
     locator: &Locator,
 ) {
-    let rest = locator.after(parse_error.location.start());
+    let rest = locator.after(parse_error.offset);
 
     // Try to create a non-empty range so that the diagnostic can print a caret at the
     // right position. This requires that we retrieve the next character, if any, and take its length
@@ -95,6 +95,6 @@ pub(crate) fn syntax_error(
         SyntaxError {
             message: format!("{}", DisplayParseErrorType::new(&parse_error.error)),
         },
-        TextRange::at(parse_error.location.start(), len),
+        TextRange::at(parse_error.offset, len),
     ));
 }

crates/ruff_linter/src/rules/pycodestyle/snapshots/ruff_linter__rules__pycodestyle__tests__E302_E302_first_line_docstring.py.snap

@@ -0,0 +1,19 @@
+---
+source: crates/ruff_linter/src/rules/pycodestyle/mod.rs
+---
+E302_first_line_docstring.py:3:1: E302 [*] Expected 2 blank lines, found 1
+  |
+1 | """Test where the error is after the module's docstring."""
+2 | 
+3 | def fn():
+  | ^^^ E302
+4 |     pass
+  |
+  = help: Add missing blank line(s)
+
+ℹ Safe fix
+1 1 | """Test where the error is after the module's docstring."""
+2 2 | 
+  3 |+
+3 4 | def fn():
+4 5 |     pass

crates/ruff_linter/src/rules/pycodestyle/snapshots/ruff_linter__rules__pycodestyle__tests__E302_E302_first_line_expression.py.snap

@@ -0,0 +1,19 @@
+---
+source: crates/ruff_linter/src/rules/pycodestyle/mod.rs
+---
+E302_first_line_expression.py:3:1: E302 [*] Expected 2 blank lines, found 1
+  |
+1 | "Test where the first line is a comment, " + "and the rule violation follows it."
+2 | 
+3 | def fn():
+  | ^^^ E302
+4 |     pass
+  |
+  = help: Add missing blank line(s)
+
+ℹ Safe fix
+1 1 | "Test where the first line is a comment, " + "and the rule violation follows it."
+2 2 | 
+  3 |+
+3 4 | def fn():
+4 5 |     pass

crates/ruff_linter/src/rules/pycodestyle/snapshots/ruff_linter__rules__pycodestyle__tests__E302_E302_first_line_function.py.snap

@@ -0,0 +1,20 @@
+---
+source: crates/ruff_linter/src/rules/pycodestyle/mod.rs
+---
+E302_first_line_function.py:4:1: E302 [*] Expected 2 blank lines, found 1
+  |
+2 |     pass
+3 | 
+4 | def fn2():
+  | ^^^ E302
+5 |     pass
+  |
+  = help: Add missing blank line(s)
+
+ℹ Safe fix
+1 1 | def fn1():
+2 2 |     pass
+3 3 | 
+  4 |+
+4 5 | def fn2():
+5 6 |     pass

crates/ruff_linter/src/rules/pycodestyle/snapshots/ruff_linter__rules__pycodestyle__tests__E302_E302_first_line_statement.py.snap

@@ -0,0 +1,19 @@
+---
+source: crates/ruff_linter/src/rules/pycodestyle/mod.rs
+---
+E302_first_line_statement.py:3:1: E302 [*] Expected 2 blank lines, found 1
+  |
+1 | print("Test where the first line is a statement, and the rule violation follows it.")
+2 | 
+3 | def fn():
+  | ^^^ E302
+4 |     pass
+  |
+  = help: Add missing blank line(s)
+
+ℹ Safe fix
+1 1 | print("Test where the first line is a statement, and the rule violation follows it.")
+2 2 | 
+  3 |+
+3 4 | def fn():
+4 5 |     pass

crates/ruff_linter/src/rules/pycodestyle/snapshots/ruff_linter__rules__pycodestyle__tests__E303_E303_first_line_comment.py.snap

@@ -0,0 +1,18 @@
+---
+source: crates/ruff_linter/src/rules/pycodestyle/mod.rs
+---
+E303_first_line_comment.py:5:1: E303 [*] Too many blank lines (3)
+  |
+5 | def fn():
+  | ^^^ E303
+6 |     pass
+  |
+  = help: Remove extraneous blank line(s)
+
+ℹ Safe fix
+1 1 | # Test where the first line is a comment, and the rule violation follows it.
+2 2 | 
+3 3 | 
+4   |-
+5 4 | def fn():
+6 5 |     pass

crates/ruff_linter/src/rules/pycodestyle/snapshots/ruff_linter__rules__pycodestyle__tests__E303_E303_first_line_docstring.py.snap

@@ -0,0 +1,18 @@
+---
+source: crates/ruff_linter/src/rules/pycodestyle/mod.rs
+---
+E303_first_line_docstring.py:5:1: E303 [*] Too many blank lines (3)
+  |
+5 | def fn():
+  | ^^^ E303
+6 |     pass
+  |
+  = help: Remove extraneous blank line(s)
+
+ℹ Safe fix
+1 1 | """Test where the error is after the module's docstring."""
+2 2 | 
+3 3 | 
+4   |-
+5 4 | def fn():
+6 5 |     pass

crates/ruff_linter/src/rules/pycodestyle/snapshots/ruff_linter__rules__pycodestyle__tests__E303_E303_first_line_expression.py.snap

@@ -0,0 +1,18 @@
+---
+source: crates/ruff_linter/src/rules/pycodestyle/mod.rs
+---
+E303_first_line_expression.py:5:1: E303 [*] Too many blank lines (3)
+  |
+5 | def fn():
+  | ^^^ E303
+6 |     pass
+  |
+  = help: Remove extraneous blank line(s)
+
+ℹ Safe fix
+1 1 | "Test where the first line is a comment, " + "and the rule violation follows it."
+2 2 | 
+3 3 | 
+4   |-
+5 4 | def fn():
+6 5 |     pass

crates/ruff_linter/src/rules/pycodestyle/snapshots/ruff_linter__rules__pycodestyle__tests__E303_E303_first_line_statement.py.snap

@@ -0,0 +1,18 @@
+---
+source: crates/ruff_linter/src/rules/pycodestyle/mod.rs
+---
+E303_first_line_statement.py:5:1: E303 [*] Too many blank lines (3)
+  |
+5 | def fn():
+  | ^^^ E303
+6 |     pass
+  |
+  = help: Remove extraneous blank line(s)
+
+ℹ Safe fix
+1 1 | print("Test where the first line is a statement, and the rule violation follows it.")
+2 2 | 
+3 3 | 
+4   |-
+5 4 | def fn():
+6 5 |     pass

crates/ruff_linter/src/rules/pycodestyle/snapshots/ruff_linter__rules__pycodestyle__tests__E999_E999.py.snap

@@ -8,3 +8,5 @@ E999.py:3:1: E999 SyntaxError: unindent does not match any outer indentation lev
   | ^ E999
 4 | 
   |
+
+

crates/ruff_linter/src/rules/pyupgrade/snapshots/ruff_linter__rules__pyupgrade__tests__UP027.py.snap

@@ -110,3 +110,5 @@ UP027.py:10:17: UP027 [*] Replace unpacked list comprehension with a generator e
 14 14 | 
 15 15 | # Should not change
 16 16 | foo = [fn(x) for x in items]
+
+

crates/ruff_linter/src/rules/ruff/rules/invalid_formatter_suppression_comment.rs

@@ -141,15 +141,15 @@ impl<'src, 'loc> UselessSuppressionComments<'src, 'loc> {
         if comment.kind == SuppressionKind::Off || comment.kind == SuppressionKind::On {
             if let Some(
                 AnyNodeRef::StmtClassDef(StmtClassDef {
-                    name,
-                    decorator_list,
-                    ..
-                })
+                                             name,
+                                             decorator_list,
+                                             ..
+                                         })
                 | AnyNodeRef::StmtFunctionDef(StmtFunctionDef {
-                    name,
-                    decorator_list,
-                    ..
-                }),
+                                                  name,
+                                                  decorator_list,
+                                                  ..
+                                              }),
             ) = comment.enclosing
             {
                 if comment.line_position.is_own_line() && comment.range.start() < name.start() {
@@ -196,7 +196,7 @@ impl<'src, 'loc> UselessSuppressionComments<'src, 'loc> {
         self.captured.sort_by_key(|(t, _)| t.start());
     }
 
-    fn ignored_comments(&self) -> impl Iterator<Item = (TextRange, IgnoredReason)> + '_ {
+    fn ignored_comments(&self) -> impl Iterator<Item=(TextRange, IgnoredReason)> + '_ {
         self.captured.iter().map(|(r, i)| (*r, *i))
     }
 }
@@ -276,7 +276,8 @@ const fn is_valid_enclosing_node(node: AnyNodeRef) -> bool {
         | AnyNodeRef::StmtIpyEscapeCommand(_)
         | AnyNodeRef::ExceptHandlerExceptHandler(_)
         | AnyNodeRef::MatchCase(_)
-        | AnyNodeRef::ElifElseClause(_) => true,
+        | AnyNodeRef::ElifElseClause(_)
+        | AnyNodeRef::StmtCrement(_) => true,
 
         AnyNodeRef::ExprBoolOp(_)
         | AnyNodeRef::ExprNamed(_)

crates/ruff_python_ast/src/comparable.rs

@@ -234,7 +234,6 @@ pub enum ComparablePattern<'a> {
     MatchStar(PatternMatchStar<'a>),
     MatchAs(PatternMatchAs<'a>),
     MatchOr(PatternMatchOr<'a>),
-    Invalid,
 }
 
 impl<'a> From<&'a ast::Pattern> for ComparablePattern<'a> {
@@ -865,7 +864,6 @@ pub enum ComparableExpr<'a> {
     Tuple(ExprTuple<'a>),
     Slice(ExprSlice<'a>),
     IpyEscapeCommand(ExprIpyEscapeCommand<'a>),
-    Invalid,
 }
 
 impl<'a> From<&'a Box<ast::Expr>> for Box<ComparableExpr<'a>> {
@@ -1545,6 +1543,9 @@ impl<'a> From<&'a ast::Stmt> for ComparableStmt<'a> {
             ast::Stmt::Pass(_) => Self::Pass,
             ast::Stmt::Break(_) => Self::Break,
             ast::Stmt::Continue(_) => Self::Continue,
+            ast::Stmt::Crement(_) => {
+                todo!()
+            }
         }
     }
 }

crates/ruff_python_ast/src/helpers.rs

@@ -432,6 +432,7 @@ pub fn any_over_stmt(stmt: &Stmt, func: &dyn Fn(&Expr) -> bool) -> bool {
         Stmt::AugAssign(ast::StmtAugAssign { target, value, .. }) => {
             any_over_expr(target, func) || any_over_expr(value, func)
         }
+        Stmt::Crement(ast::StmtCrement { target, .. }) => any_over_expr(target, func),
         Stmt::AnnAssign(ast::StmtAnnAssign {
             target,
             annotation,
@@ -1602,7 +1603,7 @@ mod tests {
     fn any_over_stmt_type_alias() {
         let seen = RefCell::new(Vec::new());
         let name = Expr::Name(ExprName {
-            id: "x".into(),
+            id: "x".to_string(),
             range: TextRange::default(),
             ctx: ExprContext::Load,
         });

crates/ruff_python_ast/src/node.rs

@@ -36,6 +36,7 @@ pub enum AnyNode {
     StmtTypeAlias(ast::StmtTypeAlias),
     StmtAssign(ast::StmtAssign),
     StmtAugAssign(ast::StmtAugAssign),
+    StmtCrement(ast::StmtCrement),
     StmtAnnAssign(ast::StmtAnnAssign),
     StmtFor(ast::StmtFor),
     StmtWhile(ast::StmtWhile),
@@ -130,6 +131,7 @@ impl AnyNode {
             AnyNode::StmtTypeAlias(node) => Some(Stmt::TypeAlias(node)),
             AnyNode::StmtAssign(node) => Some(Stmt::Assign(node)),
             AnyNode::StmtAugAssign(node) => Some(Stmt::AugAssign(node)),
+            AnyNode::StmtCrement(node) => Some(Stmt::Crement(node)),
             AnyNode::StmtAnnAssign(node) => Some(Stmt::AnnAssign(node)),
             AnyNode::StmtFor(node) => Some(Stmt::For(node)),
             AnyNode::StmtWhile(node) => Some(Stmt::While(node)),
@@ -262,6 +264,7 @@ impl AnyNode {
             | AnyNode::StmtTypeAlias(_)
             | AnyNode::StmtAssign(_)
             | AnyNode::StmtAugAssign(_)
+            | AnyNode::StmtCrement(_)
             | AnyNode::StmtAnnAssign(_)
             | AnyNode::StmtFor(_)
             | AnyNode::StmtWhile(_)
@@ -327,6 +330,7 @@ impl AnyNode {
             | AnyNode::StmtTypeAlias(_)
             | AnyNode::StmtAssign(_)
             | AnyNode::StmtAugAssign(_)
+            | AnyNode::StmtCrement(_)
             | AnyNode::StmtAnnAssign(_)
             | AnyNode::StmtFor(_)
             | AnyNode::StmtWhile(_)
@@ -432,6 +436,7 @@ impl AnyNode {
             | AnyNode::StmtTypeAlias(_)
             | AnyNode::StmtAssign(_)
             | AnyNode::StmtAugAssign(_)
+            | AnyNode::StmtCrement(_)
             | AnyNode::StmtAnnAssign(_)
             | AnyNode::StmtFor(_)
             | AnyNode::StmtWhile(_)
@@ -522,6 +527,7 @@ impl AnyNode {
             | AnyNode::StmtTypeAlias(_)
             | AnyNode::StmtAssign(_)
             | AnyNode::StmtAugAssign(_)
+            | AnyNode::StmtCrement(_)
             | AnyNode::StmtAnnAssign(_)
             | AnyNode::StmtFor(_)
             | AnyNode::StmtWhile(_)
@@ -637,6 +643,7 @@ impl AnyNode {
             Self::StmtTypeAlias(node) => AnyNodeRef::StmtTypeAlias(node),
             Self::StmtAssign(node) => AnyNodeRef::StmtAssign(node),
             Self::StmtAugAssign(node) => AnyNodeRef::StmtAugAssign(node),
+            Self::StmtCrement(node) => AnyNodeRef::StmtCrement(node),
             Self::StmtAnnAssign(node) => AnyNodeRef::StmtAnnAssign(node),
             Self::StmtFor(node) => AnyNodeRef::StmtFor(node),
             Self::StmtWhile(node) => AnyNodeRef::StmtWhile(node),
@@ -1125,6 +1132,48 @@ impl AstNode for ast::StmtAugAssign {
         visitor.visit_expr(value);
     }
 }
+impl AstNode for ast::StmtCrement {
+    fn cast(kind: AnyNode) -> Option<Self>
+    where
+        Self: Sized,
+    {
+        if let AnyNode::StmtCrement(node) = kind {
+            Some(node)
+        } else {
+            None
+        }
+    }
+
+    fn cast_ref(kind: AnyNodeRef) -> Option<&Self> {
+        if let AnyNodeRef::StmtCrement(node) = kind {
+            Some(node)
+        } else {
+            None
+        }
+    }
+
+    fn as_any_node_ref(&self) -> AnyNodeRef {
+        AnyNodeRef::from(self)
+    }
+
+    fn into_any_node(self) -> AnyNode {
+        AnyNode::from(self)
+    }
+
+    fn visit_preorder<'a, V>(&'a self, visitor: &mut V)
+    where
+        V: PreorderVisitor<'a> + ?Sized,
+    {
+        let ast::StmtCrement {
+            target,
+            op: _,
+            range: _,
+        } = self;
+
+        visitor.visit_expr(target);
+        // TODO(konstin): visitor.visit_operator(op);
+    }
+}
 impl AstNode for ast::StmtAnnAssign {
     fn cast(kind: AnyNode) -> Option<Self>
     where
@@ -4538,6 +4587,7 @@ impl From<Stmt> for AnyNode {
             Stmt::Break(node) => AnyNode::StmtBreak(node),
             Stmt::Continue(node) => AnyNode::StmtContinue(node),
             Stmt::IpyEscapeCommand(node) => AnyNode::StmtIpyEscapeCommand(node),
+            Stmt::Crement(node) => AnyNode::StmtCrement(node),
         }
     }
 }
@@ -4676,6 +4726,12 @@ impl From<ast::StmtAugAssign> for AnyNode {
     }
 }
 
+impl From<ast::StmtCrement> for AnyNode {
+    fn from(node: ast::StmtCrement) -> Self {
+        AnyNode::StmtCrement(node)
+    }
+}
+
 impl From<ast::StmtAnnAssign> for AnyNode {
     fn from(node: ast::StmtAnnAssign) -> Self {
         AnyNode::StmtAnnAssign(node)
@@ -5251,6 +5307,7 @@ impl Ranged for AnyNode {
             AnyNode::StringLiteral(node) => node.range(),
             AnyNode::BytesLiteral(node) => node.range(),
             AnyNode::ElifElseClause(node) => node.range(),
+            AnyNode::StmtCrement(node) => node.range(),
         }
     }
 }
@@ -5266,6 +5323,7 @@ pub enum AnyNodeRef<'a> {
     StmtTypeAlias(&'a ast::StmtTypeAlias),
     StmtAssign(&'a ast::StmtAssign),
     StmtAugAssign(&'a ast::StmtAugAssign),
+    StmtCrement(&'a ast::StmtCrement),
     StmtAnnAssign(&'a ast::StmtAnnAssign),
     StmtFor(&'a ast::StmtFor),
     StmtWhile(&'a ast::StmtWhile),
@@ -5444,6 +5502,7 @@ impl<'a> AnyNodeRef<'a> {
             AnyNodeRef::StringLiteral(node) => NonNull::from(*node).cast(),
             AnyNodeRef::BytesLiteral(node) => NonNull::from(*node).cast(),
             AnyNodeRef::ElifElseClause(node) => NonNull::from(*node).cast(),
+            AnyNodeRef::StmtCrement(node) => NonNull::from(*node).cast(),
         }
     }
 
@@ -5546,6 +5605,7 @@ impl<'a> AnyNodeRef<'a> {
             AnyNodeRef::StringLiteral(_) => NodeKind::StringLiteral,
             AnyNodeRef::BytesLiteral(_) => NodeKind::BytesLiteral,
             AnyNodeRef::ElifElseClause(_) => NodeKind::ElifElseClause,
+            AnyNodeRef::StmtCrement(_) => NodeKind::StmtCrement,
         }
     }
 
@@ -5558,6 +5618,7 @@ impl<'a> AnyNodeRef<'a> {
             | AnyNodeRef::StmtTypeAlias(_)
             | AnyNodeRef::StmtAssign(_)
             | AnyNodeRef::StmtAugAssign(_)
+            | AnyNodeRef::StmtCrement(_)
             | AnyNodeRef::StmtAnnAssign(_)
             | AnyNodeRef::StmtFor(_)
             | AnyNodeRef::StmtWhile(_)
@@ -5690,6 +5751,7 @@ impl<'a> AnyNodeRef<'a> {
             | AnyNodeRef::StmtTypeAlias(_)
             | AnyNodeRef::StmtAssign(_)
             | AnyNodeRef::StmtAugAssign(_)
+            | AnyNodeRef::StmtCrement(_)
             | AnyNodeRef::StmtAnnAssign(_)
             | AnyNodeRef::StmtFor(_)
             | AnyNodeRef::StmtWhile(_)
@@ -5754,6 +5816,7 @@ impl<'a> AnyNodeRef<'a> {
             | AnyNodeRef::StmtTypeAlias(_)
             | AnyNodeRef::StmtAssign(_)
             | AnyNodeRef::StmtAugAssign(_)
+            | AnyNodeRef::StmtCrement(_)
             | AnyNodeRef::StmtAnnAssign(_)
             | AnyNodeRef::StmtFor(_)
             | AnyNodeRef::StmtWhile(_)
@@ -5859,6 +5922,7 @@ impl<'a> AnyNodeRef<'a> {
             | AnyNodeRef::StmtTypeAlias(_)
             | AnyNodeRef::StmtAssign(_)
             | AnyNodeRef::StmtAugAssign(_)
+            | AnyNodeRef::StmtCrement(_)
             | AnyNodeRef::StmtAnnAssign(_)
             | AnyNodeRef::StmtFor(_)
             | AnyNodeRef::StmtWhile(_)
@@ -5949,6 +6013,7 @@ impl<'a> AnyNodeRef<'a> {
             | AnyNodeRef::StmtTypeAlias(_)
             | AnyNodeRef::StmtAssign(_)
             | AnyNodeRef::StmtAugAssign(_)
+            | AnyNodeRef::StmtCrement(_)
             | AnyNodeRef::StmtAnnAssign(_)
             | AnyNodeRef::StmtFor(_)
             | AnyNodeRef::StmtWhile(_)
@@ -6058,6 +6123,7 @@ impl<'a> AnyNodeRef<'a> {
             AnyNodeRef::StmtTypeAlias(node) => node.visit_preorder(visitor),
             AnyNodeRef::StmtAssign(node) => node.visit_preorder(visitor),
             AnyNodeRef::StmtAugAssign(node) => node.visit_preorder(visitor),
+            AnyNodeRef::StmtCrement(node) => node.visit_preorder(visitor),
             AnyNodeRef::StmtAnnAssign(node) => node.visit_preorder(visitor),
             AnyNodeRef::StmtFor(node) => node.visit_preorder(visitor),
             AnyNodeRef::StmtWhile(node) => node.visit_preorder(visitor),
@@ -6372,6 +6438,12 @@ impl<'a> From<&'a ast::StmtAugAssign> for AnyNodeRef<'a> {
     }
 }
 
+impl<'a> From<&'a ast::StmtCrement> for AnyNodeRef<'a> {
+    fn from(node: &'a ast::StmtCrement) -> Self {
+        AnyNodeRef::StmtCrement(node)
+    }
+}
+
 impl<'a> From<&'a ast::StmtAnnAssign> for AnyNodeRef<'a> {
     fn from(node: &'a ast::StmtAnnAssign) -> Self {
         AnyNodeRef::StmtAnnAssign(node)
@@ -6837,6 +6909,7 @@ impl<'a> From<&'a Stmt> for AnyNodeRef<'a> {
             Stmt::Break(node) => AnyNodeRef::StmtBreak(node),
             Stmt::Continue(node) => AnyNodeRef::StmtContinue(node),
             Stmt::IpyEscapeCommand(node) => AnyNodeRef::StmtIpyEscapeCommand(node),
+            Stmt::Crement(node) => AnyNodeRef::StmtCrement(node),
         }
     }
 }
@@ -7073,6 +7146,7 @@ impl Ranged for AnyNodeRef<'_> {
             AnyNodeRef::FString(node) => node.range(),
             AnyNodeRef::StringLiteral(node) => node.range(),
             AnyNodeRef::BytesLiteral(node) => node.range(),
+            AnyNodeRef::StmtCrement(node) => node.range(),
         }
     }
 }
@@ -7173,4 +7247,5 @@ pub enum NodeKind {
     FString,
     StringLiteral,
     BytesLiteral,
+    StmtCrement,
 }

crates/ruff_python_ast/src/nodes.rs

@@ -1,7 +1,6 @@
 #![allow(clippy::derive_partial_eq_without_eq)]
 
 use std::cell::OnceCell;
-
 use std::fmt;
 use std::fmt::Debug;
 use std::ops::Deref;
@@ -102,6 +101,8 @@ pub enum Stmt {
     // Jupyter notebook specific
     #[is(name = "ipy_escape_command_stmt")]
     IpyEscapeCommand(StmtIpyEscapeCommand),
+    #[is(name = "crement_stmt")]
+    Crement(StmtCrement),
 }
 
 /// An AST node used to represent a IPython escape command at the statement level.
@@ -298,6 +299,26 @@ impl From<StmtAugAssign> for Stmt {
     }
 }
 
+#[derive(Clone, Debug, PartialEq)]
+pub enum CrementKind {
+    Increment,
+    Decrement,
+}
+
+/// `++` or `--`
+#[derive(Clone, Debug, PartialEq)]
+pub struct StmtCrement {
+    pub range: TextRange,
+    pub target: Box<Expr>,
+    pub op: CrementKind,
+}
+
+impl From<StmtCrement> for Stmt {
+    fn from(payload: StmtCrement) -> Self {
+        Stmt::Crement(payload)
+    }
+}
+
 /// See also [AnnAssign](https://docs.python.org/3/library/ast.html#ast.AnnAssign)
 #[derive(Clone, Debug, PartialEq)]
 pub struct StmtAnnAssign {
@@ -948,19 +969,12 @@ impl Ranged for FStringExpressionElement {
     }
 }
 
-/// An `FStringLiteralElement` with an empty `value` is an invalid f-string element.
 #[derive(Clone, Debug, PartialEq)]
 pub struct FStringLiteralElement {
     pub range: TextRange,
     pub value: Box<str>,
 }
 
-impl FStringLiteralElement {
-    pub fn is_valid(&self) -> bool {
-        !self.value.is_empty()
-    }
-}
-
 impl Ranged for FStringLiteralElement {
     fn range(&self) -> TextRange {
         self.range
@@ -1509,9 +1523,6 @@ bitflags! {
         /// The string has an `r` or `R` prefix, meaning it is a raw string.
         /// It is invalid to set this flag if `U_PREFIX` is also set.
         const R_PREFIX = 1 << 3;
-
-        /// The string was deemed invalid by the parser.
-        const INVALID = 1 << 4;
     }
 }
 
@@ -1543,12 +1554,6 @@ impl StringLiteralFlags {
         self
     }
 
-    #[must_use]
-    pub fn with_invalid(mut self) -> Self {
-        self.0 |= StringLiteralFlagsInner::INVALID;
-        self
-    }
-
     pub const fn prefix(self) -> &'static str {
         if self.0.contains(StringLiteralFlagsInner::U_PREFIX) {
             debug_assert!(!self.0.contains(StringLiteralFlagsInner::R_PREFIX));
@@ -1643,15 +1648,6 @@ impl StringLiteral {
     pub fn as_str(&self) -> &str {
         self
     }
-
-    /// Creates an invalid string literal with the given range.
-    pub fn invalid(range: TextRange) -> Self {
-        Self {
-            range,
-            value: "".into(),
-            flags: StringLiteralFlags::default().with_invalid(),
-        }
-    }
 }
 
 impl From<StringLiteral> for Expr {
@@ -1860,9 +1856,6 @@ bitflags! {
 
         /// The bytestring has an `r` or `R` prefix, meaning it is a raw bytestring.
         const R_PREFIX = 1 << 3;
-
-        /// The bytestring was deemed invalid by the parser.
-        const INVALID = 1 << 4;
     }
 }
 
@@ -1890,12 +1883,6 @@ impl BytesLiteralFlags {
         self
     }
 
-    #[must_use]
-    pub fn with_invalid(mut self) -> Self {
-        self.0 |= BytesLiteralFlagsInner::INVALID;
-        self
-    }
-
     /// Does the bytestring have an `r` or `R` prefix?
     pub const fn is_raw(self) -> bool {
         self.0.contains(BytesLiteralFlagsInner::R_PREFIX)
@@ -1955,15 +1942,6 @@ impl BytesLiteral {
     pub fn as_slice(&self) -> &[u8] {
         self
     }
-
-    /// Creates a new invalid bytes literal with the given range.
-    pub fn invalid(range: TextRange) -> Self {
-        Self {
-            range,
-            value: Box::new([]),
-            flags: BytesLiteralFlags::default().with_invalid(),
-        }
-    }
 }
 
 impl From<BytesLiteral> for Expr {
@@ -2163,7 +2141,6 @@ pub enum ExprContext {
     Load,
     Store,
     Del,
-    Invalid,
 }
 impl ExprContext {
     #[inline]
@@ -3589,17 +3566,10 @@ impl IpyEscapeKind {
     }
 }
 
-/// An `Identifier` with an empty `id` is invalid.
-///
-/// For example, in the following code `id` will be empty.
-/// ```python
-/// def 1():
-///     ...
-/// ```
 #[derive(Clone, Debug, PartialEq, Eq, Hash)]
 pub struct Identifier {
-    pub id: String,
-    pub range: TextRange,
+    id: String,
+    range: TextRange,
 }
 
 impl Identifier {
@@ -3610,10 +3580,6 @@ impl Identifier {
             range,
         }
     }
-
-    pub fn is_valid(&self) -> bool {
-        !self.id.is_empty()
-    }
 }
 
 impl Identifier {
@@ -3749,6 +3715,11 @@ impl Ranged for crate::nodes::StmtAugAssign {
         self.range
     }
 }
+impl Ranged for crate::nodes::StmtCrement {
+    fn range(&self) -> TextRange {
+        self.range
+    }
+}
 impl Ranged for crate::nodes::StmtAnnAssign {
     fn range(&self) -> TextRange {
         self.range
@@ -3872,6 +3843,7 @@ impl Ranged for crate::Stmt {
             Self::Break(node) => node.range(),
             Self::Continue(node) => node.range(),
             Stmt::IpyEscapeCommand(node) => node.range(),
+            Stmt::Crement(node) => node.range(),
         }
     }
 }

crates/ruff_python_ast/src/visitor.rs

@@ -203,6 +203,14 @@ pub fn walk_stmt<'a, V: Visitor<'a> + ?Sized>(visitor: &mut V, stmt: &'a Stmt) {
             visitor.visit_operator(op);
             visitor.visit_expr(target);
         }
+        Stmt::Crement(ast::StmtCrement {
+            target,
+            op: _,
+            range: _,
+        }) => {
+            // TODO(konstin): visitor.visit_operator(op);
+            visitor.visit_expr(target);
+        }
         Stmt::AnnAssign(ast::StmtAnnAssign {
             target,
             annotation,

crates/ruff_python_ast/src/visitor/preorder.rs

@@ -209,6 +209,7 @@ where
             Stmt::TypeAlias(stmt) => stmt.visit_preorder(visitor),
             Stmt::Assign(stmt) => stmt.visit_preorder(visitor),
             Stmt::AugAssign(stmt) => stmt.visit_preorder(visitor),
+            Stmt::Crement(stmt) => stmt.visit_preorder(visitor),
             Stmt::AnnAssign(stmt) => stmt.visit_preorder(visitor),
             Stmt::For(stmt) => stmt.visit_preorder(visitor),
             Stmt::While(stmt) => stmt.visit_preorder(visitor),

crates/ruff_python_ast/src/visitor/transformer.rs

@@ -306,6 +306,7 @@ pub fn walk_stmt<V: Transformer + ?Sized>(visitor: &V, stmt: &mut Stmt) {
         Stmt::Nonlocal(_) => {}
         Stmt::Expr(ast::StmtExpr { value, range: _ }) => visitor.visit_expr(value),
         Stmt::Pass(_) | Stmt::Break(_) | Stmt::Continue(_) | Stmt::IpyEscapeCommand(_) => {}
+        Stmt::Crement(_) => {}
     }
 }
 

crates/ruff_python_codegen/src/generator.rs

@@ -4,10 +4,10 @@ use std::ops::Deref;
 
 use ruff_python_ast::str::Quote;
 use ruff_python_ast::{
-    self as ast, Alias, ArgOrKeyword, BoolOp, CmpOp, Comprehension, ConversionFlag, DebugText,
-    ExceptHandler, Expr, Identifier, MatchCase, Operator, Parameter, Parameters, Pattern,
-    Singleton, Stmt, Suite, TypeParam, TypeParamParamSpec, TypeParamTypeVar, TypeParamTypeVarTuple,
-    WithItem,
+    self as ast, Alias, ArgOrKeyword, BoolOp, CmpOp, Comprehension, ConversionFlag, CrementKind,
+    DebugText, ExceptHandler, Expr, Identifier, MatchCase, Operator, Parameter, Parameters,
+    Pattern, Singleton, Stmt, Suite, TypeParam, TypeParamParamSpec, TypeParamTypeVar,
+    TypeParamTypeVarTuple, WithItem,
 };
 use ruff_python_ast::{ParameterWithDefault, TypeParams};
 use ruff_python_literal::escape::{AsciiEscape, Escape, UnicodeEscape};
@@ -345,6 +345,20 @@ impl<'a> Generator<'a> {
                     self.unparse_expr(value, precedence::AUG_ASSIGN);
                 });
             }
+            Stmt::Crement(ast::StmtCrement {
+                target,
+                op,
+                range: _,
+            }) => {
+                statement!({
+                    self.unparse_expr(target, precedence::AUG_ASSIGN);
+                    self.p(" ");
+                    self.p(match op {
+                        CrementKind::Increment => "+= 1",
+                        CrementKind::Decrement => "-= 1",
+                    });
+                });
+            }
             Stmt::AnnAssign(ast::StmtAnnAssign {
                 target,
                 annotation,

crates/ruff_python_formatter/src/lib.rs

@@ -135,7 +135,7 @@ pub fn format_module_source(
     let source_type = options.source_type();
     let (tokens, comment_ranges) =
         tokens_and_ranges(source, source_type).map_err(|err| ParseError {
-            location: err.location(),
+            offset: err.location(),
             error: ParseErrorType::Lexical(err.into_error()),
         })?;
     let module = parse_tokens(tokens, source, source_type.as_mode())?;

crates/ruff_python_formatter/src/range.rs

@@ -73,7 +73,7 @@ pub fn format_range(
 
     let (tokens, comment_ranges) =
         tokens_and_ranges(source, options.source_type()).map_err(|err| ParseError {
-            location: err.location(),
+            offset: err.location(),
             error: ParseErrorType::Lexical(err.into_error()),
         })?;
 
@@ -706,7 +706,8 @@ impl Format<PyFormatContext<'_>> for FormatEnclosingNode<'_> {
             | AnyNodeRef::TypeParamTypeVar(_)
             | AnyNodeRef::TypeParamTypeVarTuple(_)
             | AnyNodeRef::TypeParamParamSpec(_)
-            | AnyNodeRef::BytesLiteral(_) => {
+            | AnyNodeRef::BytesLiteral(_)
+            | AnyNodeRef::StmtCrement(_) => {
                 panic!("Range formatting only supports formatting logical lines")
             }
         }

crates/ruff_python_formatter/src/statement/mod.rs

@@ -45,6 +45,19 @@ impl FormatRule<Stmt, PyFormatContext<'_>> for FormatStmt {
             Stmt::Delete(x) => x.format().fmt(f),
             Stmt::Assign(x) => x.format().fmt(f),
             Stmt::AugAssign(x) => x.format().fmt(f),
+            Stmt::Crement(x) => {
+                x.target.format().fmt(f)?;
+                use ruff_formatter::prelude::*;
+                match x.op {
+                    ruff_python_ast::CrementKind::Increment => {
+                        ruff_formatter::write!(f, [space(), text("+="), space(), text("1")])?;
+                    }
+                    ruff_python_ast::CrementKind::Decrement => {
+                        ruff_formatter::write!(f, [space(), text("-="), space(), text("1")])?;
+                    }
+                }
+                Ok(())
+            }
             Stmt::AnnAssign(x) => x.format().fmt(f),
             Stmt::For(x) => x.format().fmt(f),
             Stmt::While(x) => x.format().fmt(f),

crates/ruff_python_formatter/tests/snapshots/format@empty_multiple_trailing_newlines.py.snap

@@ -13,3 +13,6 @@ input_file: crates/ruff_python_formatter/resources/test/fixtures/ruff/empty_mult
 ```python
 
 ```
+
+
+

crates/ruff_python_formatter/tests/snapshots/format@empty_trailing_newline.py.snap

@@ -11,3 +11,6 @@ input_file: crates/ruff_python_formatter/resources/test/fixtures/ruff/empty_trai
 ```python
 
 ```
+
+
+

crates/ruff_python_parser/Cargo.toml

@@ -19,7 +19,6 @@ ruff_text_size = { path = "../ruff_text_size" }
 
 anyhow = { workspace = true }
 bitflags = { workspace = true }
-drop_bomb = { workspace = true }
 bstr = { workspace = true }
 is-macro = { workspace = true }
 itertools = { workspace = true }
@@ -31,10 +30,7 @@ unicode-ident = { workspace = true }
 unicode_names2 = { workspace = true }
 
 [dev-dependencies]
-ruff_source_file = { path = "../ruff_source_file" }
-
-annotate-snippets = { workspace = true }
-insta = { workspace = true, features = ["glob"] }
+insta = { workspace = true }
 
 [build-dependencies]
 anyhow = { workspace = true }

crates/ruff_python_parser/build.rs

@@ -5,7 +5,7 @@ use std::path::{Path, PathBuf};
 use tiny_keccak::{Hasher, Sha3};
 
 fn main() {
-    const SOURCE: &str = "src/lalrpop/python.lalrpop";
+    const SOURCE: &str = "src/python.lalrpop";
     println!("cargo:rerun-if-changed={SOURCE}");
 
     let target;
@@ -14,12 +14,12 @@ fn main() {
     #[cfg(feature = "lalrpop")]
     {
         let out_dir = PathBuf::from(std::env::var_os("OUT_DIR").unwrap());
-        target = out_dir.join("src/lalrpop/python.rs");
+        target = out_dir.join("src/python.rs");
     }
     #[cfg(not(feature = "lalrpop"))]
     {
-        target = PathBuf::from("src/lalrpop/python.rs");
-        error = "python.lalrpop and src/lalrpop/python.rs doesn't match. This is a ruff_python_parser bug. Please report it unless you are editing ruff_python_parser. Run `lalrpop src/lalrpop/python.lalrpop` to build ruff_python_parser again.";
+        target = PathBuf::from("src/python.rs");
+        error = "python.lalrpop and src/python.rs doesn't match. This is a ruff_python_parser bug. Please report it unless you are editing ruff_python_parser. Run `lalrpop src/python.lalrpop` to build ruff_python_parser again.";
     }
 
     let Some(message) = requires_lalrpop(SOURCE, &target) else {

crates/ruff_python_parser/resources/.editorconfig

@@ -1,6 +0,0 @@
-# Check http://editorconfig.org for more information
-# This is the main config file for this project:
-root = true
-
-[*.py]
-insert_final_newline = false

crates/ruff_python_parser/resources/invalid/expressions/call_invalid_argument_ordering.py

@@ -1,6 +0,0 @@
-f(b=20, c)
-
-f(**b, *c)
-
-# Duplicate keyword argument
-f(a=20, a=30)

crates/ruff_python_parser/resources/invalid/expressions/emoji_identifiers.py

@@ -1,7 +0,0 @@
-a = (🐶
-    # comment 🐶
-)
-
-a = (🐶 +
-    # comment
-🐶)

crates/ruff_python_parser/resources/invalid/expressions/emoji_statement.py

@@ -1 +0,0 @@
-👍

crates/ruff_python_parser/resources/invalid/expressions/lambda_default_parameters.py

@@ -1,2 +0,0 @@
-# TODO(micha): The offset of the generated error message is off by one.
-lambda a, b=20, c: 1

crates/ruff_python_parser/resources/invalid/expressions/lambda_duplicate_parameters.py

@@ -1,9 +0,0 @@
-lambda a, a: 1
-
-lambda a, *, a: 1
-
-lambda a, a=20: 1
-
-lambda a, *a: 1
-
-lambda a, *, **a: 1

crates/ruff_python_parser/resources/invalid/expressions/unclosed_fstring_lbrace.py

@@ -1,8 +0,0 @@
-# TODO(dhruvmanila): Remove the dummy test case and uncomment the others when this is fixed. See PR #10372
-x +
-
-# f'{'
-# f'{foo!r'
-# f'{foo='
-# f"{"
-# f"""{"""

crates/ruff_python_parser/resources/invalid/statements/assignment_keyword_target.py

@@ -1,7 +0,0 @@
-a = pass = c
-
-a + b
-
-a = b = pass = c
-
-a + b

crates/ruff_python_parser/resources/invalid/statements/assignment_missing_target.py

@@ -1,7 +0,0 @@
-a = = c
-
-a + b
-
-
-
-

crates/ruff_python_parser/resources/invalid/statements/assignment_missing_value.py

@@ -1,3 +0,0 @@
-a =
-
-a + b

crates/ruff_python_parser/resources/invalid/statements/function_default_parameters.py

@@ -1,2 +0,0 @@
-# TODO(micha): The range of the generated error message is off by one.
-def f(a, b=20, c): pass

crates/ruff_python_parser/resources/invalid/statements/function_duplicate_parameters.py

@@ -1,12 +0,0 @@
-def f(a, a): pass
-
-def f2(a, *, a): pass
-
-def f3(a, a=20): pass
-
-def f4(a, *a): pass
-
-def f5(a, *, **a): pass
-
-# TODO(micha): This is inconsistent. All other examples only highlight the argument name.
-def f6(a, a: str): pass

crates/ruff_python_parser/resources/invalid/statements/function_type_parameters.py

@@ -1,24 +0,0 @@
-# FIXME: The type param related error message and the parser recovery are looking pretty good **except**
-# that the lexer never recovers from the unclosed `[`, resulting in it lexing `NonLogicalNewline` tokens instead of `Newline` tokens.
-# That's because the parser has no way of feeding the error recovery back to the lexer,
-# so they don't agree on the state of the world which can lead to all kind of errors further down in the file.
-# This is not just a problem with parentheses but also with the transformation made by the
-# `SoftKeywordTransformer` because the `Parser` and `Transfomer` may not agree if they're
-# currently in a position where the `type` keyword is allowed or not.
-# That roughly means that any kind of recovery can lead to unrelated syntax errors
-# on following lines.
-
-def unclosed[A, *B(test: name):
-    pass
-
-    a + b
-
-def keyword[A, await](): ...
-
-def not_a_type_param[A, |, B](): ...
-
-def multiple_commas[A,,B](): ...
-
-def multiple_trailing_commas[A,,](): ...
-
-def multiple_commas_and_recovery[A,,100](): ...

crates/ruff_python_parser/resources/invalid/statements/if_elif.py

@@ -1,7 +0,0 @@
-if True:
-    pass
-elif False:
-    pass
-elf:
-    pass
-

crates/ruff_python_parser/resources/invalid/statements/if_extra_closing_parentheses.py

@@ -1,3 +0,0 @@
-# FIXME(micha): This creates two syntax errors instead of just one (and overlapping ones)
-if True)):
-    pass

crates/ruff_python_parser/resources/invalid/statements/if_extra_indent.py

@@ -1,8 +0,0 @@
-# Improving the recovery would require changing the lexer to emit an extra dedent token after `a + b`.
-if True:
-    pass
-        a + b
-
-    pass
-
-a = 10

crates/ruff_python_parser/resources/invalid/statements/if_missing_body.py

@@ -1,6 +0,0 @@
-if True:
-
-
-a + b
-
-if False: # This if statement has neither an indent nor a newline.

crates/ruff_python_parser/resources/invalid/statements/if_missing_colon.py

@@ -1,4 +0,0 @@
-if True
-    pass
-
-a = 10

crates/ruff_python_parser/resources/invalid/statements/import_from.py

@@ -1,11 +0,0 @@
-from abc import a, b,
-
-a + b
-
-from abc import ,,
-
-from abc import
-
-from abc import (a, b, c
-
-a + b

crates/ruff_python_parser/resources/invalid/statements/invalid_assignment_targets.py

@@ -1,42 +0,0 @@
-# Regression test: https://github.com/astral-sh/ruff/issues/6895
-# First we test, broadly, that various kinds of assignments are now
-# rejected by the parser. e.g., `5 = 3`, `5 += 3`, `(5): int = 3`.
-
-5 = 3
-
-5 += 3
-
-(5): int = 3
-
-# Now we exhaustively test all possible cases where assignment can fail.
-x or y = 42
-(x := 5) = 42
-x + y = 42
--x = 42
-(lambda _: 1) = 42
-a if b else c = 42
-{"a": 5} = 42
-{a} = 42
-[x for x in xs] = 42
-{x for x in xs} = 42
-{x: x * 2 for x in xs} = 42
-(x for x in xs) = 42
-await x = 42
-(yield x) = 42
-(yield from xs) = 42
-a < b < c = 42
-foo() = 42
-
-f"{quux}" = 42
-f"{foo} and {bar}" = 42
-
-"foo" = 42
-b"foo" = 42
-123 = 42
-True = 42
-None = 42
-... = 42
-*foo() = 42
-[x, foo(), y] = [42, 42, 42]
-[[a, b], [[42]], d] = [[1, 2], [[3]], 4]
-(x, foo(), y) = (42, 42, 42)

crates/ruff_python_parser/resources/invalid/statements/invalid_augmented_assignment_target.py

@@ -1,34 +0,0 @@
-# This is similar to `./invalid_assignment_targets.py`, but for augmented
-# assignment targets.
-
-x or y += 42
-(x := 5) += 42
-x + y += 42
--x += 42
-(lambda _: 1) += 42
-a if b else c += 42
-{"a": 5} += 42
-{a} += 42
-[x for x in xs] += 42
-{x for x in xs} += 42
-{x: x * 2 for x in xs} += 42
-(x for x in xs) += 42
-await x += 42
-(yield x) += 42
-(yield from xs) += 42
-a < b < c += 42
-foo() += 42
-
-f"{quux}" += 42
-f"{foo} and {bar}" += 42
-
-"foo" += 42
-b"foo" += 42
-123 += 42
-True += 42
-None += 42
-... += 42
-*foo() += 42
-[x, foo(), y] += [42, 42, 42]
-[[a, b], [[42]], d] += [[1, 2], [[3]], 4]
-(x, foo(), y) += (42, 42, 42)

crates/ruff_python_parser/resources/invalid/statements/named_expression_statement.py

@@ -1,3 +0,0 @@
-# This test previously passed before the assignment operator checking
-# above, but we include it here for good measure.
-(5 := 3)

crates/ruff_python_parser/resources/valid/expressions/dictionary_comprehension.py

@@ -1 +0,0 @@
-x = {y for y in (1, 2, 3)}

crates/ruff_python_parser/resources/valid/expressions/lambda.py

@@ -1,11 +0,0 @@
-lambda: 1
-
-lambda a, b, c: 1
-
-lambda a, b=20, c=30: 1
-
-lambda *, a, b, c: 1
-
-lambda *, a, b=20, c=30: 1
-
-lambda a, b, c, *, d, e: 0

crates/ruff_python_parser/resources/valid/expressions/list_comprehension.py

@@ -1 +0,0 @@
-x = [y for y in (1, 2, 3)]

crates/ruff_python_parser/resources/valid/expressions/named_expression.py

@@ -1,4 +0,0 @@
-if x := 1:
-    pass
-
-(x := 5)

crates/ruff_python_parser/resources/valid/statement/annotated_assignment.py

@@ -1 +0,0 @@
-x: int = 1

crates/ruff_python_parser/resources/valid/statement/assignment.py

@@ -1,40 +0,0 @@
-x = (1, 2, 3)
-
-(x, y) = (1, 2, 3)
-
-[x, y] = (1, 2, 3)
-
-x.y = (1, 2, 3)
-
-x[y] = (1, 2, 3)
-
-(x, *y) = (1, 2, 3)
-
-
-# This last group of tests checks that assignments we expect to be parsed
-# (including some interesting ones) continue to be parsed successfully.
-
-*foo = 42
-
-[x, y, z] = [1, 2, 3]
-
-(x, y, z) = (1, 2, 3)
-x[0] = 42
-
-# This is actually a type error, not a syntax error. So check that it
-# doesn't fail parsing.
-
-5[0] = 42
-x[1:2] = [42]
-
-# This is actually a type error, not a syntax error. So check that it
-# doesn't fail parsing.
-5[1:2] = [42]
-
-foo.bar = 42
-
-# This is actually an attribute error, not a syntax error. So check that
-# it doesn't fail parsing.
-"foo".y = 42
-
-foo = 42

crates/ruff_python_parser/resources/valid/statement/augmented_assignment.py

@@ -1,18 +0,0 @@
-x += 1
-x.y += (1, 2, 3)
-x[y] += (1, 2, 3)
-
-# All possible augmented assignment tokens
-x += 1
-x -= 1
-x *= 1
-x /= 1
-x //= 1
-x %= 1
-x **= 1
-x &= 1
-x |= 1
-x ^= 1
-x <<= 1
-x >>= 1
-x @= 1

crates/ruff_python_parser/resources/valid/statement/delete.py

@@ -1,3 +0,0 @@
-del x
-del x.y
-del x[y]

crates/ruff_python_parser/resources/valid/statement/for.py

@@ -1,2 +0,0 @@
-for x in (1, 2, 3):
-    pass

crates/ruff_python_parser/resources/valid/statement/function.py

@@ -1,38 +0,0 @@
-def no_parameters():
-    pass
-
-
-def positional_parameters(a, b, c):
-    pass
-
-
-def positional_parameters_with_default_values(a, b=20, c=30):
-    pass
-
-
-def keyword_only_parameters(*, a, b, c):
-    pass
-
-
-def keyword_only_parameters_with_defaults(*, a, b=20, c=30):
-    pass
-
-
-def positional_and_keyword_parameters(a, b, c, *, d, e, f):
-    pass
-
-
-def positional_and_keyword_parameters_with_defaults(a, b, c, *, d, e=20, f=30):
-    pass
-
-
-def positional_and_keyword_parameters_with_defaults_and_varargs(
-    a, b, c, *args, d, e=20, f=30
-):
-    pass
-
-
-def positional_and_keyword_parameters_with_defaults_and_varargs_and_kwargs(
-    a, b, c, *args, d, e=20, f=30, **kwargs
-):
-    pass

crates/ruff_python_parser/resources/valid/statement/with.py

@@ -1,2 +0,0 @@
-with 1 as x:
-    pass

crates/ruff_python_parser/src/context.rs

@@ -1,18 +1,18 @@
 use ruff_python_ast::{self as ast, Expr, ExprContext};
 
-pub(super) fn set_context(expr: Expr, ctx: ExprContext) -> Expr {
+pub(crate) fn set_context(expr: Expr, ctx: ExprContext) -> Expr {
     match expr {
         Expr::Name(ast::ExprName { id, range, .. }) => ast::ExprName { range, id, ctx }.into(),
         Expr::Tuple(ast::ExprTuple {
             elts,
             range,
-            parenthesized,
+            parenthesized: is_parenthesized,
             ctx: _,
         }) => ast::ExprTuple {
             elts: elts.into_iter().map(|elt| set_context(elt, ctx)).collect(),
             range,
             ctx,
-            parenthesized,
+            parenthesized: is_parenthesized,
         }
         .into(),
 
@@ -55,7 +55,7 @@ pub(super) fn set_context(expr: Expr, ctx: ExprContext) -> Expr {
 
 #[cfg(test)]
 mod tests {
-    use crate::parse_suite;
+    use crate::parser::parse_suite;
 
     #[test]
     fn test_assign_name() {

crates/ruff_python_parser/src/error.rs

@@ -1,221 +0,0 @@
-use std::fmt;
-
-use ruff_text_size::TextRange;
-
-use crate::{
-    lexer::{LexicalError, LexicalErrorType},
-    Tok, TokenKind,
-};
-
-/// Represents represent errors that occur during parsing and are
-/// returned by the `parse_*` functions.
-#[derive(Debug, PartialEq)]
-pub struct ParseError {
-    pub error: ParseErrorType,
-    pub location: TextRange,
-}
-
-impl std::ops::Deref for ParseError {
-    type Target = ParseErrorType;
-
-    fn deref(&self) -> &Self::Target {
-        &self.error
-    }
-}
-
-impl std::error::Error for ParseError {
-    fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
-        Some(&self.error)
-    }
-}
-
-impl fmt::Display for ParseError {
-    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
-        write!(f, "{} at byte range {:?}", &self.error, self.location)
-    }
-}
-
-impl From<LexicalError> for ParseError {
-    fn from(error: LexicalError) -> Self {
-        ParseError {
-            location: error.location(),
-            error: ParseErrorType::Lexical(error.into_error()),
-        }
-    }
-}
-
-impl ParseError {
-    pub fn error(self) -> ParseErrorType {
-        self.error
-    }
-}
-
-/// Represents the different types of errors that can occur during parsing of an f-string.
-#[derive(Debug, Clone, PartialEq)]
-pub enum FStringErrorType {
-    /// Expected a right brace after an opened left brace.
-    UnclosedLbrace,
-    /// An invalid conversion flag was encountered.
-    InvalidConversionFlag,
-    /// A single right brace was encountered.
-    SingleRbrace,
-    /// Unterminated string.
-    UnterminatedString,
-    /// Unterminated triple-quoted string.
-    UnterminatedTripleQuotedString,
-    /// A lambda expression without parentheses was encountered.
-    LambdaWithoutParentheses,
-}
-
-impl std::fmt::Display for FStringErrorType {
-    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
-        use FStringErrorType::{
-            InvalidConversionFlag, LambdaWithoutParentheses, SingleRbrace, UnclosedLbrace,
-            UnterminatedString, UnterminatedTripleQuotedString,
-        };
-        match self {
-            UnclosedLbrace => write!(f, "expecting '}}'"),
-            InvalidConversionFlag => write!(f, "invalid conversion character"),
-            SingleRbrace => write!(f, "single '}}' is not allowed"),
-            UnterminatedString => write!(f, "unterminated string"),
-            UnterminatedTripleQuotedString => write!(f, "unterminated triple-quoted string"),
-            LambdaWithoutParentheses => {
-                write!(f, "lambda expressions are not allowed without parentheses")
-            }
-        }
-    }
-}
-
-/// Represents the different types of errors that can occur during parsing.
-#[derive(Debug, PartialEq)]
-pub enum ParseErrorType {
-    /// An unexpected error occurred.
-    OtherError(String),
-    /// An empty slice was found during parsing, e.g `l[]`.
-    EmptySlice,
-    /// An invalid expression was found in the assignment `target`.
-    InvalidAssignmentTarget,
-    /// An invalid expression was found in the named assignment `target`.
-    InvalidNamedAssignmentTarget,
-    /// An invalid expression was found in the augmented assignment `target`.
-    InvalidAugmentedAssignmentTarget,
-    /// An invalid expression was found in the delete `target`.
-    InvalidDeleteTarget,
-    /// Multiple simple statements were found in the same line without a `;` separating them.
-    SimpleStmtsInSameLine,
-    /// An unexpected indentation was found during parsing.
-    UnexpectedIndentation,
-    /// The statement being parsed cannot be `async`.
-    StmtIsNotAsync(TokenKind),
-    /// A parameter was found after a vararg
-    ParamFollowsVarKeywordParam,
-    /// A positional argument follows a keyword argument.
-    PositionalArgumentError,
-    /// An iterable argument unpacking `*args` follows keyword argument unpacking `**kwargs`.
-    UnpackedArgumentError,
-    /// A non-default argument follows a default argument.
-    DefaultArgumentError,
-    /// A simple statement and a compound statement was found in the same line.
-    SimpleStmtAndCompoundStmtInSameLine,
-    /// An invalid `match` case pattern was found.
-    InvalidMatchPatternLiteral { pattern: TokenKind },
-    /// The parser expected a specific token that was not found.
-    ExpectedToken {
-        expected: TokenKind,
-        found: TokenKind,
-    },
-    /// A duplicate argument was found in a function definition.
-    DuplicateArgumentError(String),
-    /// A keyword argument was repeated.
-    DuplicateKeywordArgumentError(String),
-    /// An f-string error containing the [`FStringErrorType`].
-    FStringError(FStringErrorType),
-    /// Parser encountered an error during lexing.
-    Lexical(LexicalErrorType),
-
-    // RustPython specific.
-    /// Parser encountered an extra token
-    ExtraToken(Tok),
-    /// Parser encountered an invalid token
-    InvalidToken,
-    /// Parser encountered an unexpected token
-    UnrecognizedToken(Tok, Option<String>),
-    /// Parser encountered an unexpected end of input
-    Eof,
-}
-
-impl std::error::Error for ParseErrorType {}
-
-impl std::fmt::Display for ParseErrorType {
-    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
-        match self {
-            ParseErrorType::OtherError(msg) => write!(f, "{msg}"),
-            ParseErrorType::ExpectedToken { found, expected } => {
-                write!(f, "expected {expected:?}, found {found:?}")
-            }
-            ParseErrorType::Lexical(ref lex_error) => write!(f, "{lex_error}"),
-            ParseErrorType::SimpleStmtsInSameLine => {
-                write!(f, "use `;` to separate simple statements")
-            }
-            ParseErrorType::SimpleStmtAndCompoundStmtInSameLine => write!(
-                f,
-                "compound statements not allowed in the same line as simple statements"
-            ),
-            ParseErrorType::StmtIsNotAsync(kind) => {
-                write!(f, "`{kind:?}` statement cannot be async")
-            }
-            ParseErrorType::UnpackedArgumentError => {
-                write!(
-                    f,
-                    "iterable argument unpacking follows keyword argument unpacking"
-                )
-            }
-            ParseErrorType::PositionalArgumentError => {
-                write!(f, "positional argument follows keyword argument unpacking")
-            }
-            ParseErrorType::EmptySlice => write!(f, "slice cannot be empty"),
-            ParseErrorType::ParamFollowsVarKeywordParam => {
-                write!(f, "parameters cannot follow var-keyword parameter")
-            }
-            ParseErrorType::DefaultArgumentError => {
-                write!(f, "non-default argument follows default argument")
-            }
-            ParseErrorType::InvalidMatchPatternLiteral { pattern } => {
-                write!(f, "invalid pattern `{pattern:?}`")
-            }
-            ParseErrorType::UnexpectedIndentation => write!(f, "unexpected indentation"),
-            ParseErrorType::InvalidAssignmentTarget => write!(f, "invalid assignment target"),
-            ParseErrorType::InvalidNamedAssignmentTarget => {
-                write!(f, "invalid named assignment target")
-            }
-            ParseErrorType::InvalidAugmentedAssignmentTarget => {
-                write!(f, "invalid augmented assignment target")
-            }
-            ParseErrorType::InvalidDeleteTarget => {
-                write!(f, "invalid delete target")
-            }
-            ParseErrorType::DuplicateArgumentError(arg_name) => {
-                write!(f, "duplicate argument '{arg_name}' in function definition")
-            }
-            ParseErrorType::DuplicateKeywordArgumentError(arg_name) => {
-                write!(f, "keyword argument repeated: {arg_name}")
-            }
-            ParseErrorType::FStringError(ref fstring_error) => {
-                write!(f, "f-string: {fstring_error}")
-            }
-            // RustPython specific.
-            ParseErrorType::Eof => write!(f, "Got unexpected EOF"),
-            ParseErrorType::ExtraToken(ref tok) => write!(f, "Got extraneous token: {tok:?}"),
-            ParseErrorType::InvalidToken => write!(f, "Got invalid token"),
-            ParseErrorType::UnrecognizedToken(ref tok, ref expected) => {
-                if *tok == Tok::Indent {
-                    write!(f, "Unexpected indent")
-                } else if expected.as_deref() == Some("Indent") {
-                    write!(f, "Expected an indented block")
-                } else {
-                    write!(f, "Unexpected token {tok}")
-                }
-            }
-        }
-    }
-}

crates/ruff_python_parser/src/invalid.rs

@@ -0,0 +1,699 @@
+/*!
+Defines some helper routines for rejecting invalid Python programs.
+
+These routines are named in a way that supports qualified use. For example,
+`invalid::assignment_targets`.
+*/
+
+use {ruff_python_ast::Expr, ruff_text_size::TextSize};
+
+use crate::lexer::{LexicalError, LexicalErrorType};
+
+/// Returns an error for invalid assignment targets.
+///
+/// # Errors
+///
+/// This returns an error when any of the given expressions are themselves
+/// or contain an expression that is invalid on the left hand side of an
+/// assignment. For example, all literal expressions are invalid assignment
+/// targets.
+pub(crate) fn assignment_targets(targets: &[Expr]) -> Result<(), LexicalError> {
+    for t in targets {
+        assignment_target(t)?;
+    }
+    Ok(())
+}
+
+/// Returns an error if the given target is invalid for the left hand side of
+/// an assignment.
+///
+/// # Errors
+///
+/// This returns an error when the given expression is itself or contains an
+/// expression that is invalid on the left hand side of an assignment. For
+/// example, all literal expressions are invalid assignment targets.
+pub(crate) fn assignment_target(target: &Expr) -> Result<(), LexicalError> {
+    // Allowing a glob import here because of its limited scope.
+    #[allow(clippy::enum_glob_use)]
+    use self::Expr::*;
+
+    let err = |location: TextSize| -> LexicalError {
+        let error = LexicalErrorType::AssignmentError;
+        LexicalError::new(error, location)
+    };
+    match *target {
+        BoolOp(ref e) => Err(err(e.range.start())),
+        Named(ref e) => Err(err(e.range.start())),
+        BinOp(ref e) => Err(err(e.range.start())),
+        UnaryOp(ref e) => Err(err(e.range.start())),
+        Lambda(ref e) => Err(err(e.range.start())),
+        If(ref e) => Err(err(e.range.start())),
+        Dict(ref e) => Err(err(e.range.start())),
+        Set(ref e) => Err(err(e.range.start())),
+        ListComp(ref e) => Err(err(e.range.start())),
+        SetComp(ref e) => Err(err(e.range.start())),
+        DictComp(ref e) => Err(err(e.range.start())),
+        Generator(ref e) => Err(err(e.range.start())),
+        Await(ref e) => Err(err(e.range.start())),
+        Yield(ref e) => Err(err(e.range.start())),
+        YieldFrom(ref e) => Err(err(e.range.start())),
+        Compare(ref e) => Err(err(e.range.start())),
+        Call(ref e) => Err(err(e.range.start())),
+        // FString is recursive, but all its forms are invalid as an
+        // assignment target, so we can reject it without exploring it.
+        FString(ref e) => Err(err(e.range.start())),
+        StringLiteral(ref e) => Err(err(e.range.start())),
+        BytesLiteral(ref e) => Err(err(e.range.start())),
+        NumberLiteral(ref e) => Err(err(e.range.start())),
+        BooleanLiteral(ref e) => Err(err(e.range.start())),
+        NoneLiteral(ref e) => Err(err(e.range.start())),
+        EllipsisLiteral(ref e) => Err(err(e.range.start())),
+        // This isn't in the Python grammar but is Jupyter notebook specific.
+        // It seems like this should be an error. It does also seem like the
+        // parser prevents this from ever appearing as an assignment target
+        // anyway. ---AG
+        IpyEscapeCommand(ref e) => Err(err(e.range.start())),
+        // The only nested expressions allowed as an assignment target
+        // are star exprs, lists and tuples.
+        Starred(ref e) => assignment_target(&e.value),
+        List(ref e) => assignment_targets(&e.elts),
+        Tuple(ref e) => assignment_targets(&e.elts),
+        // Subscript is recursive and can be invalid, but aren't syntax errors.
+        // For example, `5[1] = 42` is a type error.
+        Subscript(_) => Ok(()),
+        // Similar to Subscript, e.g., `5[1:2] = [42]` is a type error.
+        Slice(_) => Ok(()),
+        // Similar to Subscript, e.g., `"foo".y = 42` is an attribute error.
+        Attribute(_) => Ok(()),
+        // These are always valid as assignment targets.
+        Name(_) => Ok(()),
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use crate::parse_suite;
+
+    // First we test, broadly, that various kinds of assignments are now
+    // rejected by the parser. e.g., `5 = 3`, `5 += 3`, `(5): int = 3`.
+
+    // Regression test: https://github.com/astral-sh/ruff/issues/6895
+    #[test]
+    fn err_literal_assignment() {
+        let ast = parse_suite(r"5 = 3");
+        insta::assert_debug_snapshot!(ast, @r###"
+        Err(
+            ParseError {
+                error: Lexical(
+                    AssignmentError,
+                ),
+                offset: 0,
+            },
+        )
+        "###);
+    }
+
+    // This test previously passed before the assignment operator checking
+    // above, but we include it here for good measure.
+    #[test]
+    fn err_assignment_expr() {
+        let ast = parse_suite(r"(5 := 3)");
+        insta::assert_debug_snapshot!(ast, @r###"
+        Err(
+            ParseError {
+                error: UnrecognizedToken(
+                    ColonEqual,
+                    None,
+                ),
+                offset: 3,
+            },
+        )
+        "###);
+    }
+
+    #[test]
+    fn err_literal_augment_assignment() {
+        let ast = parse_suite(r"5 += 3");
+        insta::assert_debug_snapshot!(ast, @r###"
+        Err(
+            ParseError {
+                error: Lexical(
+                    AssignmentError,
+                ),
+                offset: 0,
+            },
+        )
+        "###);
+    }
+
+    #[test]
+    fn err_literal_annotation_assignment() {
+        let ast = parse_suite(r"(5): int = 3");
+        insta::assert_debug_snapshot!(ast, @r###"
+        Err(
+            ParseError {
+                error: Lexical(
+                    AssignmentError,
+                ),
+                offset: 1,
+            },
+        )
+        "###);
+    }
+
+    // Now we exhaustively test all possible cases where assignment can fail.
+
+    #[test]
+    fn err_bool_op() {
+        let ast = parse_suite(r"x or y = 42");
+        insta::assert_debug_snapshot!(ast, @r###"
+        Err(
+            ParseError {
+                error: Lexical(
+                    AssignmentError,
+                ),
+                offset: 0,
+            },
+        )
+        "###);
+    }
+
+    #[test]
+    fn err_named_expr() {
+        let ast = parse_suite(r"(x := 5) = 42");
+        insta::assert_debug_snapshot!(ast, @r###"
+        Err(
+            ParseError {
+                error: Lexical(
+                    AssignmentError,
+                ),
+                offset: 1,
+            },
+        )
+        "###);
+    }
+
+    #[test]
+    fn err_bin_op() {
+        let ast = parse_suite(r"x + y = 42");
+        insta::assert_debug_snapshot!(ast, @r###"
+        Err(
+            ParseError {
+                error: Lexical(
+                    AssignmentError,
+                ),
+                offset: 0,
+            },
+        )
+        "###);
+    }
+
+    #[test]
+    fn err_unary_op() {
+        let ast = parse_suite(r"-x = 42");
+        insta::assert_debug_snapshot!(ast, @r###"
+        Err(
+            ParseError {
+                error: Lexical(
+                    AssignmentError,
+                ),
+                offset: 0,
+            },
+        )
+        "###);
+    }
+
+    #[test]
+    fn err_lambda() {
+        let ast = parse_suite(r"(lambda _: 1) = 42");
+        insta::assert_debug_snapshot!(ast, @r###"
+        Err(
+            ParseError {
+                error: Lexical(
+                    AssignmentError,
+                ),
+                offset: 1,
+            },
+        )
+        "###);
+    }
+
+    #[test]
+    fn err_if_exp() {
+        let ast = parse_suite(r"a if b else c = 42");
+        insta::assert_debug_snapshot!(ast, @r###"
+        Err(
+            ParseError {
+                error: Lexical(
+                    AssignmentError,
+                ),
+                offset: 0,
+            },
+        )
+        "###);
+    }
+
+    #[test]
+    fn err_dict() {
+        let ast = parse_suite(r"{'a':5} = 42");
+        insta::assert_debug_snapshot!(ast, @r###"
+        Err(
+            ParseError {
+                error: Lexical(
+                    AssignmentError,
+                ),
+                offset: 0,
+            },
+        )
+        "###);
+    }
+
+    #[test]
+    fn err_set() {
+        let ast = parse_suite(r"{a} = 42");
+        insta::assert_debug_snapshot!(ast, @r###"
+        Err(
+            ParseError {
+                error: Lexical(
+                    AssignmentError,
+                ),
+                offset: 0,
+            },
+        )
+        "###);
+    }
+
+    #[test]
+    fn err_list_comp() {
+        let ast = parse_suite(r"[x for x in xs] = 42");
+        insta::assert_debug_snapshot!(ast, @r###"
+        Err(
+            ParseError {
+                error: Lexical(
+                    AssignmentError,
+                ),
+                offset: 0,
+            },
+        )
+        "###);
+    }
+
+    #[test]
+    fn err_set_comp() {
+        let ast = parse_suite(r"{x for x in xs} = 42");
+        insta::assert_debug_snapshot!(ast, @r###"
+        Err(
+            ParseError {
+                error: Lexical(
+                    AssignmentError,
+                ),
+                offset: 0,
+            },
+        )
+        "###);
+    }
+
+    #[test]
+    fn err_dict_comp() {
+        let ast = parse_suite(r"{x: x*2 for x in xs} = 42");
+        insta::assert_debug_snapshot!(ast, @r###"
+        Err(
+            ParseError {
+                error: Lexical(
+                    AssignmentError,
+                ),
+                offset: 0,
+            },
+        )
+        "###);
+    }
+
+    #[test]
+    fn err_generator_exp() {
+        let ast = parse_suite(r"(x for x in xs) = 42");
+        insta::assert_debug_snapshot!(ast, @r###"
+        Err(
+            ParseError {
+                error: Lexical(
+                    AssignmentError,
+                ),
+                offset: 0,
+            },
+        )
+        "###);
+    }
+
+    #[test]
+    fn err_await() {
+        let ast = parse_suite(r"await x = 42");
+        insta::assert_debug_snapshot!(ast, @r###"
+        Err(
+            ParseError {
+                error: Lexical(
+                    AssignmentError,
+                ),
+                offset: 0,
+            },
+        )
+        "###);
+    }
+
+    #[test]
+    fn err_yield() {
+        let ast = parse_suite(r"(yield x) = 42");
+        insta::assert_debug_snapshot!(ast, @r###"
+        Err(
+            ParseError {
+                error: Lexical(
+                    AssignmentError,
+                ),
+                offset: 1,
+            },
+        )
+        "###);
+    }
+
+    #[test]
+    fn err_yield_from() {
+        let ast = parse_suite(r"(yield from xs) = 42");
+        insta::assert_debug_snapshot!(ast, @r###"
+        Err(
+            ParseError {
+                error: Lexical(
+                    AssignmentError,
+                ),
+                offset: 1,
+            },
+        )
+        "###);
+    }
+
+    #[test]
+    fn err_compare() {
+        let ast = parse_suite(r"a < b < c = 42");
+        insta::assert_debug_snapshot!(ast, @r###"
+        Err(
+            ParseError {
+                error: Lexical(
+                    AssignmentError,
+                ),
+                offset: 0,
+            },
+        )
+        "###);
+    }
+
+    #[test]
+    fn err_call() {
+        let ast = parse_suite(r"foo() = 42");
+        insta::assert_debug_snapshot!(ast, @r###"
+        Err(
+            ParseError {
+                error: Lexical(
+                    AssignmentError,
+                ),
+                offset: 0,
+            },
+        )
+        "###);
+    }
+
+    #[test]
+    fn err_formatted_value() {
+        // N.B. It looks like the parser can't generate a top-level
+        // FormattedValue, where as the official Python AST permits
+        // representing a single f-string containing just a variable as a
+        // FormattedValue directly.
+        //
+        // Bottom line is that because of this, this test is (at present)
+        // duplicative with the `fstring` test. That is, in theory these tests
+        // could fail independently, but in practice their failure or success
+        // is coupled.
+        //
+        // See: https://docs.python.org/3/library/ast.html#ast.FormattedValue
+        let ast = parse_suite(r#"f"{quux}" = 42"#);
+        insta::assert_debug_snapshot!(ast, @r###"
+        Err(
+            ParseError {
+                error: Lexical(
+                    AssignmentError,
+                ),
+                offset: 0,
+            },
+        )
+        "###);
+    }
+
+    #[test]
+    fn err_fstring() {
+        let ast = parse_suite(r#"f"{foo} and {bar}" = 42"#);
+        insta::assert_debug_snapshot!(ast, @r###"
+        Err(
+            ParseError {
+                error: Lexical(
+                    AssignmentError,
+                ),
+                offset: 0,
+            },
+        )
+        "###);
+    }
+
+    #[test]
+    fn err_string_literal() {
+        let ast = parse_suite(r#""foo" = 42"#);
+        insta::assert_debug_snapshot!(ast, @r###"
+        Err(
+            ParseError {
+                error: Lexical(
+                    AssignmentError,
+                ),
+                offset: 0,
+            },
+        )
+        "###);
+    }
+
+    #[test]
+    fn err_bytes_literal() {
+        let ast = parse_suite(r#"b"foo" = 42"#);
+        insta::assert_debug_snapshot!(ast, @r###"
+        Err(
+            ParseError {
+                error: Lexical(
+                    AssignmentError,
+                ),
+                offset: 0,
+            },
+        )
+        "###);
+    }
+
+    #[test]
+    fn err_number_literal() {
+        let ast = parse_suite(r"123 = 42");
+        insta::assert_debug_snapshot!(ast, @r###"
+        Err(
+            ParseError {
+                error: Lexical(
+                    AssignmentError,
+                ),
+                offset: 0,
+            },
+        )
+        "###);
+    }
+
+    #[test]
+    fn err_boolean_literal() {
+        let ast = parse_suite(r"True = 42");
+        insta::assert_debug_snapshot!(ast, @r###"
+        Err(
+            ParseError {
+                error: Lexical(
+                    AssignmentError,
+                ),
+                offset: 0,
+            },
+        )
+        "###);
+    }
+
+    #[test]
+    fn err_none_literal() {
+        let ast = parse_suite(r"None = 42");
+        insta::assert_debug_snapshot!(ast, @r###"
+        Err(
+            ParseError {
+                error: Lexical(
+                    AssignmentError,
+                ),
+                offset: 0,
+            },
+        )
+        "###);
+    }
+
+    #[test]
+    fn err_ellipsis_literal() {
+        let ast = parse_suite(r"... = 42");
+        insta::assert_debug_snapshot!(ast, @r###"
+        Err(
+            ParseError {
+                error: Lexical(
+                    AssignmentError,
+                ),
+                offset: 0,
+            },
+        )
+        "###);
+    }
+
+    #[test]
+    fn err_starred() {
+        let ast = parse_suite(r"*foo() = 42");
+        insta::assert_debug_snapshot!(ast, @r###"
+        Err(
+            ParseError {
+                error: Lexical(
+                    AssignmentError,
+                ),
+                offset: 1,
+            },
+        )
+        "###);
+    }
+
+    #[test]
+    fn err_list() {
+        let ast = parse_suite(r"[x, foo(), y] = [42, 42, 42]");
+        insta::assert_debug_snapshot!(ast, @r###"
+        Err(
+            ParseError {
+                error: Lexical(
+                    AssignmentError,
+                ),
+                offset: 4,
+            },
+        )
+        "###);
+    }
+
+    #[test]
+    fn err_list_nested() {
+        let ast = parse_suite(r"[[a, b], [[42]], d] = [[1, 2], [[3]], 4]");
+        insta::assert_debug_snapshot!(ast, @r###"
+        Err(
+            ParseError {
+                error: Lexical(
+                    AssignmentError,
+                ),
+                offset: 11,
+            },
+        )
+        "###);
+    }
+
+    #[test]
+    fn err_tuple() {
+        let ast = parse_suite(r"(x, foo(), y) = (42, 42, 42)");
+        insta::assert_debug_snapshot!(ast, @r###"
+        Err(
+            ParseError {
+                error: Lexical(
+                    AssignmentError,
+                ),
+                offset: 4,
+            },
+        )
+        "###);
+    }
+
+    // This last group of tests checks that assignments we expect to be parsed
+    // (including some interesting ones) continue to be parsed successfully.
+
+    #[test]
+    fn ok_starred() {
+        let ast = parse_suite(r"*foo = 42");
+        insta::assert_debug_snapshot!(ast);
+    }
+
+    #[test]
+    fn ok_list() {
+        let ast = parse_suite(r"[x, y, z] = [1, 2, 3]");
+        insta::assert_debug_snapshot!(ast);
+    }
+
+    #[test]
+    fn ok_tuple() {
+        let ast = parse_suite(r"(x, y, z) = (1, 2, 3)");
+        insta::assert_debug_snapshot!(ast);
+    }
+
+    #[test]
+    fn ok_subscript_normal() {
+        let ast = parse_suite(r"x[0] = 42");
+        insta::assert_debug_snapshot!(ast);
+    }
+
+    // This is actually a type error, not a syntax error. So check that it
+    // doesn't fail parsing.
+    #[test]
+    fn ok_subscript_weird() {
+        let ast = parse_suite(r"5[0] = 42");
+        insta::assert_debug_snapshot!(ast);
+    }
+
+    #[test]
+    fn ok_slice_normal() {
+        let ast = parse_suite(r"x[1:2] = [42]");
+        insta::assert_debug_snapshot!(ast);
+    }
+
+    // This is actually a type error, not a syntax error. So check that it
+    // doesn't fail parsing.
+    #[test]
+    fn ok_slice_weird() {
+        let ast = parse_suite(r"5[1:2] = [42]");
+        insta::assert_debug_snapshot!(ast);
+    }
+
+    #[test]
+    fn ok_attribute_normal() {
+        let ast = parse_suite(r"foo.bar = 42");
+        insta::assert_debug_snapshot!(ast);
+    }
+
+    // This is actually an attribute error, not a syntax error. So check that
+    // it doesn't fail parsing.
+    #[test]
+    fn ok_attribute_weird() {
+        let ast = parse_suite(r#""foo".y = 42"#);
+        insta::assert_debug_snapshot!(ast);
+    }
+
+    #[test]
+    fn ok_name() {
+        let ast = parse_suite(r"foo = 42");
+        insta::assert_debug_snapshot!(ast);
+    }
+
+    // This is a sanity test for what looks like an ipython directive being
+    // assigned to. Although this doesn't actually parse as an assignment
+    // statement, but rather, a directive whose value is `foo = 42`.
+    #[test]
+    fn ok_ipy_escape_command() {
+        use crate::Mode;
+
+        let src = r"!foo = 42";
+        let tokens = crate::lexer::lex(src, Mode::Ipython);
+        let ast = crate::parse_tokens(tokens.collect(), src, Mode::Ipython);
+        insta::assert_debug_snapshot!(ast);
+    }
+
+    #[test]
+    fn ok_assignment_expr() {
+        let ast = parse_suite(r"(x := 5)");
+        insta::assert_debug_snapshot!(ast);
+    }
+}

crates/ruff_python_parser/src/lalrpop/function.rs

@@ -1,138 +0,0 @@
-use std::hash::BuildHasherDefault;
-// Contains functions that perform validation and parsing of arguments and parameters.
-// Checks apply both to functions and to lambdas.
-use crate::lexer::{LexicalError, LexicalErrorType};
-use ruff_python_ast::{self as ast};
-use ruff_text_size::{Ranged, TextRange, TextSize};
-use rustc_hash::FxHashSet;
-
-pub(crate) struct ArgumentList {
-    pub(crate) args: Vec<ast::Expr>,
-    pub(crate) keywords: Vec<ast::Keyword>,
-}
-
-// Perform validation of function/lambda arguments in a function definition.
-pub(super) fn validate_arguments(arguments: &ast::Parameters) -> Result<(), LexicalError> {
-    let mut all_arg_names = FxHashSet::with_capacity_and_hasher(
-        arguments.posonlyargs.len()
-            + arguments.args.len()
-            + usize::from(arguments.vararg.is_some())
-            + arguments.kwonlyargs.len()
-            + usize::from(arguments.kwarg.is_some()),
-        BuildHasherDefault::default(),
-    );
-
-    let posonlyargs = arguments.posonlyargs.iter();
-    let args = arguments.args.iter();
-    let kwonlyargs = arguments.kwonlyargs.iter();
-
-    let vararg: Option<&ast::Parameter> = arguments.vararg.as_deref();
-    let kwarg: Option<&ast::Parameter> = arguments.kwarg.as_deref();
-
-    for arg in posonlyargs
-        .chain(args)
-        .chain(kwonlyargs)
-        .map(|arg| &arg.parameter)
-        .chain(vararg)
-        .chain(kwarg)
-    {
-        let range = arg.range;
-        let arg_name = arg.name.as_str();
-        if !all_arg_names.insert(arg_name) {
-            return Err(LexicalError::new(
-                LexicalErrorType::DuplicateArgumentError(arg_name.to_string().into_boxed_str()),
-                range,
-            ));
-        }
-    }
-
-    Ok(())
-}
-
-pub(super) fn validate_pos_params(
-    args: &(
-        Vec<ast::ParameterWithDefault>,
-        Vec<ast::ParameterWithDefault>,
-    ),
-) -> Result<(), LexicalError> {
-    let (posonlyargs, args) = args;
-    #[allow(clippy::skip_while_next)]
-    let first_invalid = posonlyargs
-        .iter()
-        .chain(args.iter()) // for all args
-        .skip_while(|arg| arg.default.is_none()) // starting with args without default
-        .skip_while(|arg| arg.default.is_some()) // and then args with default
-        .next(); // there must not be any more args without default
-    if let Some(invalid) = first_invalid {
-        return Err(LexicalError::new(
-            LexicalErrorType::DefaultArgumentError,
-            invalid.parameter.range(),
-        ));
-    }
-    Ok(())
-}
-
-type FunctionArgument = (
-    Option<(TextSize, TextSize, Option<ast::Identifier>)>,
-    ast::Expr,
-);
-
-// Parse arguments as supplied during a function/lambda *call*.
-pub(super) fn parse_arguments(
-    function_arguments: Vec<FunctionArgument>,
-) -> Result<ArgumentList, LexicalError> {
-    let mut args = vec![];
-    let mut keywords = vec![];
-
-    let mut keyword_names = FxHashSet::with_capacity_and_hasher(
-        function_arguments.len(),
-        BuildHasherDefault::default(),
-    );
-    let mut double_starred = false;
-    for (name, value) in function_arguments {
-        if let Some((start, end, name)) = name {
-            // Check for duplicate keyword arguments in the call.
-            if let Some(keyword_name) = &name {
-                if !keyword_names.insert(keyword_name.to_string()) {
-                    return Err(LexicalError::new(
-                        LexicalErrorType::DuplicateKeywordArgumentError(
-                            keyword_name.to_string().into_boxed_str(),
-                        ),
-                        TextRange::new(start, end),
-                    ));
-                }
-            } else {
-                double_starred = true;
-            }
-
-            keywords.push(ast::Keyword {
-                arg: name,
-                value,
-                range: TextRange::new(start, end),
-            });
-        } else {
-            // Positional arguments mustn't follow keyword arguments.
-            if !keywords.is_empty() && !is_starred(&value) {
-                return Err(LexicalError::new(
-                    LexicalErrorType::PositionalArgumentError,
-                    value.range(),
-                ));
-                // Allow starred arguments after keyword arguments but
-                // not after double-starred arguments.
-            } else if double_starred {
-                return Err(LexicalError::new(
-                    LexicalErrorType::UnpackedArgumentError,
-                    value.range(),
-                ));
-            }
-
-            args.push(value);
-        }
-    }
-    Ok(ArgumentList { args, keywords })
-}
-
-// Check if an expression is a starred expression.
-const fn is_starred(exp: &ast::Expr) -> bool {
-    exp.is_starred_expr()
-}

crates/ruff_python_parser/src/lalrpop/invalid.rs

@@ -1,93 +0,0 @@
-/*!
-Defines some helper routines for rejecting invalid Python programs.
-
-These routines are named in a way that supports qualified use. For example,
-`invalid::assignment_targets`.
-*/
-
-use ruff_python_ast::Expr;
-
-use ruff_text_size::TextRange;
-
-use crate::lexer::{LexicalError, LexicalErrorType};
-
-/// Returns an error for invalid assignment targets.
-///
-/// # Errors
-///
-/// This returns an error when any of the given expressions are themselves
-/// or contain an expression that is invalid on the left hand side of an
-/// assignment. For example, all literal expressions are invalid assignment
-/// targets.
-pub(crate) fn assignment_targets(targets: &[Expr]) -> Result<(), LexicalError> {
-    for t in targets {
-        assignment_target(t)?;
-    }
-    Ok(())
-}
-
-/// Returns an error if the given target is invalid for the left hand side of
-/// an assignment.
-///
-/// # Errors
-///
-/// This returns an error when the given expression is itself or contains an
-/// expression that is invalid on the left hand side of an assignment. For
-/// example, all literal expressions are invalid assignment targets.
-pub(crate) fn assignment_target(target: &Expr) -> Result<(), LexicalError> {
-    // Allowing a glob import here because of its limited scope.
-    #[allow(clippy::enum_glob_use)]
-    use self::Expr::*;
-
-    let err = |location: TextRange| -> LexicalError {
-        let error = LexicalErrorType::AssignmentError;
-        LexicalError::new(error, location)
-    };
-    match *target {
-        BoolOp(ref e) => Err(err(e.range)),
-        Named(ref e) => Err(err(e.range)),
-        BinOp(ref e) => Err(err(e.range)),
-        UnaryOp(ref e) => Err(err(e.range)),
-        Lambda(ref e) => Err(err(e.range)),
-        If(ref e) => Err(err(e.range)),
-        Dict(ref e) => Err(err(e.range)),
-        Set(ref e) => Err(err(e.range)),
-        ListComp(ref e) => Err(err(e.range)),
-        SetComp(ref e) => Err(err(e.range)),
-        DictComp(ref e) => Err(err(e.range)),
-        Generator(ref e) => Err(err(e.range)),
-        Await(ref e) => Err(err(e.range)),
-        Yield(ref e) => Err(err(e.range)),
-        YieldFrom(ref e) => Err(err(e.range)),
-        Compare(ref e) => Err(err(e.range)),
-        Call(ref e) => Err(err(e.range)),
-        // FString is recursive, but all its forms are invalid as an
-        // assignment target, so we can reject it without exploring it.
-        FString(ref e) => Err(err(e.range)),
-        StringLiteral(ref e) => Err(err(e.range)),
-        BytesLiteral(ref e) => Err(err(e.range)),
-        NumberLiteral(ref e) => Err(err(e.range)),
-        BooleanLiteral(ref e) => Err(err(e.range)),
-        NoneLiteral(ref e) => Err(err(e.range)),
-        EllipsisLiteral(ref e) => Err(err(e.range)),
-        // This isn't in the Python grammar but is Jupyter notebook specific.
-        // It seems like this should be an error. It does also seem like the
-        // parser prevents this from ever appearing as an assignment target
-        // anyway. ---AG
-        IpyEscapeCommand(ref e) => Err(err(e.range)),
-        // The only nested expressions allowed as an assignment target
-        // are star exprs, lists and tuples.
-        Starred(ref e) => assignment_target(&e.value),
-        List(ref e) => assignment_targets(&e.elts),
-        Tuple(ref e) => assignment_targets(&e.elts),
-        // Subscript is recursive and can be invalid, but aren't syntax errors.
-        // For example, `5[1] = 42` is a type error.
-        Subscript(_) => Ok(()),
-        // Similar to Subscript, e.g., `5[1:2] = [42]` is a type error.
-        Slice(_) => Ok(()),
-        // Similar to Subscript, e.g., `"foo".y = 42` is an attribute error.
-        Attribute(_) => Ok(()),
-        // These are always valid as assignment targets.
-        Name(_) => Ok(()),
-    }
-}

crates/ruff_python_parser/src/lalrpop/mod.rs

@@ -1,311 +0,0 @@
-//! The LALRPOP based parser implementation.
-
-use itertools::Itertools;
-use lalrpop_util::ParseError as LalrpopError;
-
-use ruff_python_ast::{
-    Expr, ExprAttribute, ExprAwait, ExprBinOp, ExprBoolOp, ExprBooleanLiteral, ExprBytesLiteral,
-    ExprCall, ExprCompare, ExprDict, ExprDictComp, ExprEllipsisLiteral, ExprFString, ExprGenerator,
-    ExprIf, ExprIpyEscapeCommand, ExprLambda, ExprList, ExprListComp, ExprName, ExprNamed,
-    ExprNoneLiteral, ExprNumberLiteral, ExprSet, ExprSetComp, ExprSlice, ExprStarred,
-    ExprStringLiteral, ExprSubscript, ExprTuple, ExprUnaryOp, ExprYield, ExprYieldFrom, Mod,
-};
-use ruff_text_size::{Ranged, TextRange, TextSize};
-
-use crate::lexer::{LexResult, LexicalError, LexicalErrorType};
-use crate::{Mode, ParseError, ParseErrorType, Tok};
-
-mod context;
-mod function;
-mod invalid;
-
-#[rustfmt::skip]
-#[allow(unreachable_pub)]
-#[allow(clippy::type_complexity)]
-#[allow(clippy::extra_unused_lifetimes)]
-#[allow(clippy::needless_lifetimes)]
-#[allow(clippy::unused_self)]
-#[allow(clippy::cast_sign_loss)]
-#[allow(clippy::default_trait_access)]
-#[allow(clippy::let_unit_value)]
-#[allow(clippy::just_underscores_and_digits)]
-#[allow(clippy::no_effect_underscore_binding)]
-#[allow(clippy::trivially_copy_pass_by_ref)]
-#[allow(clippy::option_option)]
-#[allow(clippy::unnecessary_wraps)]
-#[allow(clippy::uninlined_format_args)]
-#[allow(clippy::cloned_instead_of_copied)]
-mod python {
-
-    #[cfg(feature = "lalrpop")]
-    include!(concat!(env!("OUT_DIR"), "/src/lalrpop/python.rs"));
-
-    #[cfg(not(feature = "lalrpop"))]
-    include!("python.rs");
-}
-
-pub(crate) fn parse_tokens(
-    tokens: Vec<LexResult>,
-    source: &str,
-    mode: Mode,
-) -> Result<Mod, ParseError> {
-    let marker_token = (Tok::start_marker(mode), TextRange::default());
-    let lexer = std::iter::once(Ok(marker_token)).chain(
-        tokens
-            .into_iter()
-            .filter_ok(|token| !matches!(token, (Tok::Comment(..) | Tok::NonLogicalNewline, _))),
-    );
-    python::TopParser::new()
-        .parse(
-            source,
-            mode,
-            lexer.map_ok(|(t, range)| (range.start(), t, range.end())),
-        )
-        .map_err(parse_error_from_lalrpop)
-}
-
-fn parse_error_from_lalrpop(err: LalrpopError<TextSize, Tok, LexicalError>) -> ParseError {
-    match err {
-        // TODO: Are there cases where this isn't an EOF?
-        LalrpopError::InvalidToken { location } => ParseError {
-            error: ParseErrorType::Eof,
-            location: TextRange::empty(location),
-        },
-        LalrpopError::ExtraToken { token } => ParseError {
-            error: ParseErrorType::ExtraToken(token.1),
-            location: TextRange::new(token.0, token.2),
-        },
-        LalrpopError::User { error } => ParseError {
-            location: error.location(),
-            error: ParseErrorType::Lexical(error.into_error()),
-        },
-        LalrpopError::UnrecognizedToken { token, expected } => {
-            // Hacky, but it's how CPython does it. See PyParser_AddToken,
-            // in particular "Only one possible expected token" comment.
-            let expected = (expected.len() == 1).then(|| expected[0].clone());
-            ParseError {
-                error: ParseErrorType::UnrecognizedToken(token.1, expected),
-                location: TextRange::new(token.0, token.2),
-            }
-        }
-        LalrpopError::UnrecognizedEof { location, expected } => {
-            // This could be an initial indentation error that we should ignore
-            let indent_error = expected == ["Indent"];
-            if indent_error {
-                ParseError {
-                    error: ParseErrorType::Lexical(LexicalErrorType::IndentationError),
-                    location: TextRange::empty(location),
-                }
-            } else {
-                ParseError {
-                    error: ParseErrorType::Eof,
-                    location: TextRange::empty(location),
-                }
-            }
-        }
-    }
-}
-
-/// An expression that may be parenthesized.
-#[derive(Clone, Debug)]
-struct ParenthesizedExpr {
-    /// The range of the expression, including any parentheses.
-    range: TextRange,
-    /// The underlying expression.
-    expr: Expr,
-}
-
-impl ParenthesizedExpr {
-    /// Returns `true` if the expression is parenthesized.
-    fn is_parenthesized(&self) -> bool {
-        self.range.start() != self.expr.range().start()
-    }
-}
-
-impl Ranged for ParenthesizedExpr {
-    fn range(&self) -> TextRange {
-        self.range
-    }
-}
-impl From<Expr> for ParenthesizedExpr {
-    fn from(expr: Expr) -> Self {
-        ParenthesizedExpr {
-            range: expr.range(),
-            expr,
-        }
-    }
-}
-impl From<ParenthesizedExpr> for Expr {
-    fn from(parenthesized_expr: ParenthesizedExpr) -> Self {
-        parenthesized_expr.expr
-    }
-}
-impl From<ExprIpyEscapeCommand> for ParenthesizedExpr {
-    fn from(payload: ExprIpyEscapeCommand) -> Self {
-        Expr::IpyEscapeCommand(payload).into()
-    }
-}
-impl From<ExprBoolOp> for ParenthesizedExpr {
-    fn from(payload: ExprBoolOp) -> Self {
-        Expr::BoolOp(payload).into()
-    }
-}
-impl From<ExprNamed> for ParenthesizedExpr {
-    fn from(payload: ExprNamed) -> Self {
-        Expr::Named(payload).into()
-    }
-}
-impl From<ExprBinOp> for ParenthesizedExpr {
-    fn from(payload: ExprBinOp) -> Self {
-        Expr::BinOp(payload).into()
-    }
-}
-impl From<ExprUnaryOp> for ParenthesizedExpr {
-    fn from(payload: ExprUnaryOp) -> Self {
-        Expr::UnaryOp(payload).into()
-    }
-}
-impl From<ExprLambda> for ParenthesizedExpr {
-    fn from(payload: ExprLambda) -> Self {
-        Expr::Lambda(payload).into()
-    }
-}
-impl From<ExprIf> for ParenthesizedExpr {
-    fn from(payload: ExprIf) -> Self {
-        Expr::If(payload).into()
-    }
-}
-impl From<ExprDict> for ParenthesizedExpr {
-    fn from(payload: ExprDict) -> Self {
-        Expr::Dict(payload).into()
-    }
-}
-impl From<ExprSet> for ParenthesizedExpr {
-    fn from(payload: ExprSet) -> Self {
-        Expr::Set(payload).into()
-    }
-}
-impl From<ExprListComp> for ParenthesizedExpr {
-    fn from(payload: ExprListComp) -> Self {
-        Expr::ListComp(payload).into()
-    }
-}
-impl From<ExprSetComp> for ParenthesizedExpr {
-    fn from(payload: ExprSetComp) -> Self {
-        Expr::SetComp(payload).into()
-    }
-}
-impl From<ExprDictComp> for ParenthesizedExpr {
-    fn from(payload: ExprDictComp) -> Self {
-        Expr::DictComp(payload).into()
-    }
-}
-impl From<ExprGenerator> for ParenthesizedExpr {
-    fn from(payload: ExprGenerator) -> Self {
-        Expr::Generator(payload).into()
-    }
-}
-impl From<ExprAwait> for ParenthesizedExpr {
-    fn from(payload: ExprAwait) -> Self {
-        Expr::Await(payload).into()
-    }
-}
-impl From<ExprYield> for ParenthesizedExpr {
-    fn from(payload: ExprYield) -> Self {
-        Expr::Yield(payload).into()
-    }
-}
-impl From<ExprYieldFrom> for ParenthesizedExpr {
-    fn from(payload: ExprYieldFrom) -> Self {
-        Expr::YieldFrom(payload).into()
-    }
-}
-impl From<ExprCompare> for ParenthesizedExpr {
-    fn from(payload: ExprCompare) -> Self {
-        Expr::Compare(payload).into()
-    }
-}
-impl From<ExprCall> for ParenthesizedExpr {
-    fn from(payload: ExprCall) -> Self {
-        Expr::Call(payload).into()
-    }
-}
-impl From<ExprFString> for ParenthesizedExpr {
-    fn from(payload: ExprFString) -> Self {
-        Expr::FString(payload).into()
-    }
-}
-impl From<ExprStringLiteral> for ParenthesizedExpr {
-    fn from(payload: ExprStringLiteral) -> Self {
-        Expr::StringLiteral(payload).into()
-    }
-}
-impl From<ExprBytesLiteral> for ParenthesizedExpr {
-    fn from(payload: ExprBytesLiteral) -> Self {
-        Expr::BytesLiteral(payload).into()
-    }
-}
-impl From<ExprNumberLiteral> for ParenthesizedExpr {
-    fn from(payload: ExprNumberLiteral) -> Self {
-        Expr::NumberLiteral(payload).into()
-    }
-}
-impl From<ExprBooleanLiteral> for ParenthesizedExpr {
-    fn from(payload: ExprBooleanLiteral) -> Self {
-        Expr::BooleanLiteral(payload).into()
-    }
-}
-impl From<ExprNoneLiteral> for ParenthesizedExpr {
-    fn from(payload: ExprNoneLiteral) -> Self {
-        Expr::NoneLiteral(payload).into()
-    }
-}
-impl From<ExprEllipsisLiteral> for ParenthesizedExpr {
-    fn from(payload: ExprEllipsisLiteral) -> Self {
-        Expr::EllipsisLiteral(payload).into()
-    }
-}
-impl From<ExprAttribute> for ParenthesizedExpr {
-    fn from(payload: ExprAttribute) -> Self {
-        Expr::Attribute(payload).into()
-    }
-}
-impl From<ExprSubscript> for ParenthesizedExpr {
-    fn from(payload: ExprSubscript) -> Self {
-        Expr::Subscript(payload).into()
-    }
-}
-impl From<ExprStarred> for ParenthesizedExpr {
-    fn from(payload: ExprStarred) -> Self {
-        Expr::Starred(payload).into()
-    }
-}
-impl From<ExprName> for ParenthesizedExpr {
-    fn from(payload: ExprName) -> Self {
-        Expr::Name(payload).into()
-    }
-}
-impl From<ExprList> for ParenthesizedExpr {
-    fn from(payload: ExprList) -> Self {
-        Expr::List(payload).into()
-    }
-}
-impl From<ExprTuple> for ParenthesizedExpr {
-    fn from(payload: ExprTuple) -> Self {
-        Expr::Tuple(payload).into()
-    }
-}
-impl From<ExprSlice> for ParenthesizedExpr {
-    fn from(payload: ExprSlice) -> Self {
-        Expr::Slice(payload).into()
-    }
-}
-
-#[cfg(target_pointer_width = "64")]
-mod size_assertions {
-    use static_assertions::assert_eq_size;
-
-    use super::ParenthesizedExpr;
-
-    assert_eq_size!(ParenthesizedExpr, [u8; 72]);
-}

crates/ruff_python_parser/src/lexer.rs

@@ -36,12 +36,12 @@ use unicode_ident::{is_xid_continue, is_xid_start};
 use ruff_python_ast::{Int, IpyEscapeKind};
 use ruff_text_size::{TextLen, TextRange, TextSize};
 
-use crate::error::FStringErrorType;
 use crate::lexer::cursor::{Cursor, EOF_CHAR};
 use crate::lexer::fstring::{FStringContext, FStrings};
 use crate::lexer::indentation::{Indentation, Indentations};
 use crate::{
     soft_keywords::SoftKeywordTransformer,
+    string::FStringErrorType,
     string_token_flags::{StringKind, StringPrefix},
     token::Tok,
     Mode,
@@ -287,7 +287,7 @@ impl<'source> Lexer<'source> {
             Err(err) => {
                 return Err(LexicalError::new(
                     LexicalErrorType::OtherError(format!("{err:?}").into_boxed_str()),
-                    self.token_range(),
+                    self.token_range().start(),
                 ));
             }
         };
@@ -312,7 +312,7 @@ impl<'source> Lexer<'source> {
             if self.cursor.eat_char('_') {
                 return Err(LexicalError::new(
                     LexicalErrorType::OtherError("Invalid Syntax".to_string().into_boxed_str()),
-                    TextRange::new(self.offset() - TextSize::new(1), self.offset()),
+                    self.offset() - TextSize::new(1),
                 ));
             }
 
@@ -346,7 +346,7 @@ impl<'source> Lexer<'source> {
                     LexicalErrorType::OtherError(
                         "Invalid decimal literal".to_string().into_boxed_str(),
                     ),
-                    self.token_range(),
+                    self.token_start(),
                 )
             })?;
 
@@ -371,11 +371,9 @@ impl<'source> Lexer<'source> {
                             // Leading zeros in decimal integer literals are not permitted.
                             return Err(LexicalError::new(
                                 LexicalErrorType::OtherError(
-                                    "Invalid decimal integer literal"
-                                        .to_string()
-                                        .into_boxed_str(),
+                                    "Invalid Token".to_string().into_boxed_str(),
                                 ),
-                                self.token_range(),
+                                self.token_range().start(),
                             ));
                         }
                         value
@@ -383,7 +381,7 @@ impl<'source> Lexer<'source> {
                     Err(err) => {
                         return Err(LexicalError::new(
                             LexicalErrorType::OtherError(format!("{err:?}").into_boxed_str()),
-                            self.token_range(),
+                            self.token_range().start(),
                         ))
                     }
                 };
@@ -600,7 +598,7 @@ impl<'source> Lexer<'source> {
                     };
                     return Err(LexicalError::new(
                         LexicalErrorType::FStringError(error),
-                        self.token_range(),
+                        self.offset(),
                     ));
                 }
                 '\n' | '\r' if !fstring.is_triple_quoted() => {
@@ -613,7 +611,7 @@ impl<'source> Lexer<'source> {
                     }
                     return Err(LexicalError::new(
                         LexicalErrorType::FStringError(FStringErrorType::UnterminatedString),
-                        self.token_range(),
+                        self.offset(),
                     ));
                 }
                 '\\' => {
@@ -723,9 +721,22 @@ impl<'source> Lexer<'source> {
                 let Some(index) = memchr::memchr(quote_byte, self.cursor.rest().as_bytes()) else {
                     self.cursor.skip_to_end();
 
+                    if let Some(fstring) = self.fstrings.current() {
+                        // When we are in an f-string, check whether the initial quote
+                        // matches with f-strings quotes and if it is, then this must be a
+                        // missing '}' token so raise the proper error.
+                        if fstring.quote_char() == quote
+                            && fstring.is_triple_quoted() == kind.is_triple_quoted()
+                        {
+                            return Err(LexicalError::new(
+                                LexicalErrorType::FStringError(FStringErrorType::UnclosedLbrace),
+                                self.cursor.text_len(),
+                            ));
+                        }
+                    }
                     return Err(LexicalError::new(
-                        LexicalErrorType::UnclosedStringError,
-                        self.token_range(),
+                        LexicalErrorType::Eof,
+                        self.cursor.text_len(),
                     ));
                 };
 
@@ -759,9 +770,22 @@ impl<'source> Lexer<'source> {
                 else {
                     self.cursor.skip_to_end();
 
+                    if let Some(fstring) = self.fstrings.current() {
+                        // When we are in an f-string, check whether the initial quote
+                        // matches with f-strings quotes and if it is, then this must be a
+                        // missing '}' token so raise the proper error.
+                        if fstring.quote_char() == quote
+                            && fstring.is_triple_quoted() == kind.is_triple_quoted()
+                        {
+                            return Err(LexicalError::new(
+                                LexicalErrorType::FStringError(FStringErrorType::UnclosedLbrace),
+                                self.offset(),
+                            ));
+                        }
+                    }
                     return Err(LexicalError::new(
                         LexicalErrorType::StringError,
-                        self.token_range(),
+                        self.offset(),
                     ));
                 };
 
@@ -787,9 +811,26 @@ impl<'source> Lexer<'source> {
 
                 match ch {
                     Some('\r' | '\n') => {
+                        if let Some(fstring) = self.fstrings.current() {
+                            // When we are in an f-string, check whether the initial quote
+                            // matches with f-strings quotes and if it is, then this must be a
+                            // missing '}' token so raise the proper error.
+                            if fstring.quote_char() == quote && !fstring.is_triple_quoted() {
+                                return Err(LexicalError::new(
+                                    LexicalErrorType::FStringError(
+                                        FStringErrorType::UnclosedLbrace,
+                                    ),
+                                    self.offset() - TextSize::new(1),
+                                ));
+                            }
+                        }
                         return Err(LexicalError::new(
-                            LexicalErrorType::UnclosedStringError,
-                            self.token_range(),
+                            LexicalErrorType::OtherError(
+                                "EOL while scanning string literal"
+                                    .to_string()
+                                    .into_boxed_str(),
+                            ),
+                            self.offset() - TextSize::new(1),
                         ));
                     }
                     Some(ch) if ch == quote => {
@@ -838,7 +879,7 @@ impl<'source> Lexer<'source> {
                     self.pending_indentation = Some(indentation);
                     let offset = self.offset();
                     self.indentations.dedent_one(indentation).map_err(|_| {
-                        LexicalError::new(LexicalErrorType::IndentationError, self.token_range())
+                        LexicalError::new(LexicalErrorType::IndentationError, offset)
                     })?;
                     return Ok((Tok::Dedent, TextRange::empty(offset)));
                 }
@@ -846,7 +887,7 @@ impl<'source> Lexer<'source> {
                 Err(_) => {
                     return Err(LexicalError::new(
                         LexicalErrorType::IndentationError,
-                        self.token_range(),
+                        self.offset(),
                     ));
                 }
             }
@@ -872,7 +913,7 @@ impl<'source> Lexer<'source> {
             } else {
                 Err(LexicalError::new(
                     LexicalErrorType::UnrecognizedToken { tok: c },
-                    self.token_range(),
+                    self.token_start(),
                 ))
             }
         } else {
@@ -896,11 +937,11 @@ impl<'source> Lexer<'source> {
                     if self.cursor.eat_char('\r') {
                         self.cursor.eat_char('\n');
                     } else if self.cursor.is_eof() {
-                        return Err(LexicalError::new(LexicalErrorType::Eof, self.token_range()));
+                        return Err(LexicalError::new(LexicalErrorType::Eof, self.token_start()));
                     } else if !self.cursor.eat_char('\n') {
                         return Err(LexicalError::new(
                             LexicalErrorType::LineContinuationError,
-                            self.token_range(),
+                            self.token_start(),
                         ));
                     }
                 }
@@ -934,11 +975,11 @@ impl<'source> Lexer<'source> {
                     if self.cursor.eat_char('\r') {
                         self.cursor.eat_char('\n');
                     } else if self.cursor.is_eof() {
-                        return Err(LexicalError::new(LexicalErrorType::Eof, self.token_range()));
+                        return Err(LexicalError::new(LexicalErrorType::Eof, self.token_start()));
                     } else if !self.cursor.eat_char('\n') {
                         return Err(LexicalError::new(
                             LexicalErrorType::LineContinuationError,
-                            self.token_range(),
+                            self.token_start(),
                         ));
                     }
                     indentation = Indentation::root();
@@ -976,7 +1017,7 @@ impl<'source> Lexer<'source> {
                 self.pending_indentation = Some(indentation);
 
                 self.indentations.dedent_one(indentation).map_err(|_| {
-                    LexicalError::new(LexicalErrorType::IndentationError, self.token_range())
+                    LexicalError::new(LexicalErrorType::IndentationError, self.offset())
                 })?;
 
                 Some((Tok::Dedent, TextRange::empty(self.offset())))
@@ -992,7 +1033,7 @@ impl<'source> Lexer<'source> {
             Err(_) => {
                 return Err(LexicalError::new(
                     LexicalErrorType::IndentationError,
-                    self.token_range(),
+                    self.offset(),
                 ));
             }
         };
@@ -1006,7 +1047,7 @@ impl<'source> Lexer<'source> {
         if self.nesting > 0 {
             // Reset the nesting to avoid going into infinite loop.
             self.nesting = 0;
-            return Err(LexicalError::new(LexicalErrorType::Eof, self.token_range()));
+            return Err(LexicalError::new(LexicalErrorType::Eof, self.offset()));
         }
 
         // Next, insert a trailing newline, if required.
@@ -1038,7 +1079,9 @@ impl<'source> Lexer<'source> {
                 }
             }
             '+' => {
-                if self.cursor.eat_char('=') {
+                if self.cursor.eat_char('+') {
+                    Tok::Increment
+                } else if self.cursor.eat_char('=') {
                     Tok::PlusEqual
                 } else {
                     Tok::Plus
@@ -1125,7 +1168,9 @@ impl<'source> Lexer<'source> {
                 }
             }
             '-' => {
-                if self.cursor.eat_char('=') {
+                if self.cursor.eat_char('-') {
+                    Tok::Increment
+                } else if self.cursor.eat_char('=') {
                     Tok::MinusEqual
                 } else if self.cursor.eat_char('>') {
                     Tok::Rarrow
@@ -1173,7 +1218,7 @@ impl<'source> Lexer<'source> {
                     if fstring.nesting() == self.nesting {
                         return Err(LexicalError::new(
                             LexicalErrorType::FStringError(FStringErrorType::SingleRbrace),
-                            self.token_range(),
+                            self.token_start(),
                         ));
                     }
                     fstring.try_end_format_spec(self.nesting);
@@ -1267,7 +1312,7 @@ impl<'source> Lexer<'source> {
 
                 return Err(LexicalError::new(
                     LexicalErrorType::UnrecognizedToken { tok: c },
-                    self.token_range(),
+                    self.token_start(),
                 ));
             }
         };
@@ -1331,12 +1376,12 @@ pub struct LexicalError {
     /// The type of error that occurred.
     error: LexicalErrorType,
     /// The location of the error.
-    location: TextRange,
+    location: TextSize,
 }
 
 impl LexicalError {
     /// Creates a new `LexicalError` with the given error type and location.
-    pub fn new(error: LexicalErrorType, location: TextRange) -> Self {
+    pub fn new(error: LexicalErrorType, location: TextSize) -> Self {
         Self { error, location }
     }
 
@@ -1348,7 +1393,7 @@ impl LexicalError {
         self.error
     }
 
-    pub fn location(&self) -> TextRange {
+    pub fn location(&self) -> TextSize {
         self.location
     }
 }
@@ -1373,7 +1418,7 @@ impl std::fmt::Display for LexicalError {
             f,
             "{} at byte offset {}",
             self.error(),
-            u32::from(self.location().start())
+            u32::from(self.location())
         )
     }
 }
@@ -1388,14 +1433,9 @@ pub enum LexicalErrorType {
     // to use the `UnicodeError` variant instead.
     #[doc(hidden)]
     StringError,
-    /// A string literal without the closing quote.
-    UnclosedStringError,
+    // TODO: Should take a start/end position to report.
     /// Decoding of a unicode escape sequence in a string literal failed.
     UnicodeError,
-    /// Missing the `{` for unicode escape sequence.
-    MissingUnicodeLbrace,
-    /// Missing the `}` for unicode escape sequence.
-    MissingUnicodeRbrace,
     /// The nesting of brackets/braces/parentheses is not balanced.
     NestingError,
     /// The indentation is not consistent.
@@ -1417,8 +1457,6 @@ pub enum LexicalErrorType {
     UnrecognizedToken { tok: char },
     /// An f-string error containing the [`FStringErrorType`].
     FStringError(FStringErrorType),
-    /// Invalid character encountered in a byte literal.
-    InvalidByteLiteral,
     /// An unexpected character was encountered after a line continuation.
     LineContinuationError,
     /// An unexpected end of file was encountered.
@@ -1436,9 +1474,6 @@ impl std::fmt::Display for LexicalErrorType {
         match self {
             LexicalErrorType::StringError => write!(f, "Got unexpected string"),
             LexicalErrorType::FStringError(error) => write!(f, "f-string: {error}"),
-            LexicalErrorType::InvalidByteLiteral => {
-                write!(f, "bytes can only contain ASCII literal characters")
-            }
             LexicalErrorType::UnicodeError => write!(f, "Got unexpected unicode"),
             LexicalErrorType::NestingError => write!(f, "Got unexpected nesting"),
             LexicalErrorType::IndentationError => {
@@ -1477,15 +1512,6 @@ impl std::fmt::Display for LexicalErrorType {
             LexicalErrorType::Eof => write!(f, "unexpected EOF while parsing"),
             LexicalErrorType::AssignmentError => write!(f, "invalid assignment target"),
             LexicalErrorType::OtherError(msg) => write!(f, "{msg}"),
-            LexicalErrorType::UnclosedStringError => {
-                write!(f, "missing closing quote in string literal")
-            }
-            LexicalErrorType::MissingUnicodeLbrace => {
-                write!(f, "Missing `{{` in Unicode escape sequence")
-            }
-            LexicalErrorType::MissingUnicodeRbrace => {
-                write!(f, "Missing `}}` in Unicode escape sequence")
-            }
         }
     }
 }
@@ -2280,7 +2306,9 @@ f"{(lambda x:{x})}"
 
     #[test]
     fn test_fstring_error() {
-        use FStringErrorType::{SingleRbrace, UnterminatedString, UnterminatedTripleQuotedString};
+        use FStringErrorType::{
+            SingleRbrace, UnclosedLbrace, UnterminatedString, UnterminatedTripleQuotedString,
+        };
 
         assert_eq!(lex_fstring_error("f'}'"), SingleRbrace);
         assert_eq!(lex_fstring_error("f'{{}'"), SingleRbrace);
@@ -2291,6 +2319,18 @@ f"{(lambda x:{x})}"
         assert_eq!(lex_fstring_error("f'{3:}}>10}'"), SingleRbrace);
         assert_eq!(lex_fstring_error(r"f'\{foo}\}'"), SingleRbrace);
 
+        assert_eq!(lex_fstring_error("f'{'"), UnclosedLbrace);
+        assert_eq!(lex_fstring_error("f'{foo!r'"), UnclosedLbrace);
+        assert_eq!(lex_fstring_error("f'{foo='"), UnclosedLbrace);
+        assert_eq!(
+            lex_fstring_error(
+                r#"f"{"
+"#
+            ),
+            UnclosedLbrace
+        );
+        assert_eq!(lex_fstring_error(r#"f"""{""""#), UnclosedLbrace);
+
         assert_eq!(lex_fstring_error(r#"f""#), UnterminatedString);
         assert_eq!(lex_fstring_error(r"f'"), UnterminatedString);
 
@@ -2305,4 +2345,25 @@ f"{(lambda x:{x})}"
             UnterminatedTripleQuotedString
         );
     }
+
+    #[test]
+    fn test_fstring_error_location() {
+        assert_debug_snapshot!(lex_error("f'{'"), @r###"
+        LexicalError {
+            error: FStringError(
+                UnclosedLbrace,
+            ),
+            location: 4,
+        }
+        "###);
+
+        assert_debug_snapshot!(lex_error("f'{'α"), @r###"
+        LexicalError {
+            error: FStringError(
+                UnclosedLbrace,
+            ),
+            location: 6,
+        }
+        "###);
+    }
 }

crates/ruff_python_parser/src/lib.rs

@@ -109,227 +109,30 @@
 //! [parsing]: https://en.wikipedia.org/wiki/Parsing
 //! [lexer]: crate::lexer
 
-use std::cell::Cell;
-
-pub use error::{FStringErrorType, ParseError, ParseErrorType};
-use lexer::{lex, lex_starts_at};
-pub use parser::Program;
-use ruff_python_ast::{Expr, Mod, ModModule, PySourceType, Suite};
-use ruff_text_size::TextSize;
+pub use parser::{
+    parse, parse_expression, parse_expression_starts_at, parse_program, parse_starts_at,
+    parse_suite, parse_tokens, ParseError, ParseErrorType,
+};
+use ruff_python_ast::{Mod, PySourceType, Suite};
+pub use string::FStringErrorType;
 pub use string_token_flags::StringKind;
 pub use token::{Tok, TokenKind};
 
 use crate::lexer::LexResult;
 
-mod error;
-mod lalrpop;
+mod context;
+mod function;
+mod invalid;
+// Skip flattening lexer to distinguish from full ruff_python_parser
 pub mod lexer;
 mod parser;
 mod soft_keywords;
 mod string;
 mod string_token_flags;
 mod token;
-mod token_set;
 mod token_source;
 pub mod typing;
 
-thread_local! {
-    static NEW_PARSER: Cell<bool> = Cell::new(std::env::var("NEW_PARSER").is_ok());
-}
-
-/// Controls whether the current thread uses the new hand written or the old lalrpop based parser.
-///
-/// Uses the new hand written parser if `use_new_parser` is true.
-///
-/// Defaults to use the new handwritten parser if the environment variable `NEW_PARSER` is set.
-pub fn set_new_parser(use_new_parser: bool) {
-    NEW_PARSER.set(use_new_parser);
-}
-
-/// Parse a full Python program usually consisting of multiple lines.
-///
-/// This is a convenience function that can be used to parse a full Python program without having to
-/// specify the [`Mode`] or the location. It is probably what you want to use most of the time.
-///
-/// # Example
-///
-/// For example, parsing a simple function definition and a call to that function:
-///
-/// ```
-/// use ruff_python_parser as parser;
-/// let source = r#"
-/// def foo():
-///    return 42
-///
-/// print(foo())
-/// "#;
-/// let program = parser::parse_program(source);
-/// assert!(program.is_ok());
-/// ```
-pub fn parse_program(source: &str) -> Result<ModModule, ParseError> {
-    let lexer = lex(source, Mode::Module);
-    match parse_tokens(lexer.collect(), source, Mode::Module)? {
-        Mod::Module(m) => Ok(m),
-        Mod::Expression(_) => unreachable!("Mode::Module doesn't return other variant"),
-    }
-}
-
-pub fn parse_suite(source: &str) -> Result<Suite, ParseError> {
-    parse_program(source).map(|m| m.body)
-}
-
-/// Parses a single Python expression.
-///
-/// This convenience function can be used to parse a single expression without having to
-/// specify the Mode or the location.
-///
-/// # Example
-///
-/// For example, parsing a single expression denoting the addition of two numbers:
-///
-///  ```
-/// use ruff_python_parser as parser;
-/// let expr = parser::parse_expression("1 + 2");
-///
-/// assert!(expr.is_ok());
-///
-/// ```
-pub fn parse_expression(source: &str) -> Result<Expr, ParseError> {
-    let lexer = lex(source, Mode::Expression).collect();
-    match parse_tokens(lexer, source, Mode::Expression)? {
-        Mod::Expression(expression) => Ok(*expression.body),
-        Mod::Module(_m) => unreachable!("Mode::Expression doesn't return other variant"),
-    }
-}
-
-/// Parses a Python expression from a given location.
-///
-/// This function allows to specify the location of the expression in the source code, other than
-/// that, it behaves exactly like [`parse_expression`].
-///
-/// # Example
-///
-/// Parsing a single expression denoting the addition of two numbers, but this time specifying a different,
-/// somewhat silly, location:
-///
-/// ```
-/// use ruff_python_parser::{parse_expression_starts_at};
-/// # use ruff_text_size::TextSize;
-///
-/// let expr = parse_expression_starts_at("1 + 2", TextSize::from(400));
-/// assert!(expr.is_ok());
-/// ```
-pub fn parse_expression_starts_at(source: &str, offset: TextSize) -> Result<Expr, ParseError> {
-    let lexer = lex_starts_at(source, Mode::Module, offset).collect();
-    match parse_tokens(lexer, source, Mode::Expression)? {
-        Mod::Expression(expression) => Ok(*expression.body),
-        Mod::Module(_m) => unreachable!("Mode::Expression doesn't return other variant"),
-    }
-}
-
-/// Parse the given Python source code using the specified [`Mode`].
-///
-/// This function is the most general function to parse Python code. Based on the [`Mode`] supplied,
-/// it can be used to parse a single expression, a full Python program, an interactive expression
-/// or a Python program containing IPython escape commands.
-///
-/// # Example
-///
-/// If we want to parse a simple expression, we can use the [`Mode::Expression`] mode during
-/// parsing:
-///
-/// ```
-/// use ruff_python_parser::{Mode, parse};
-///
-/// let expr = parse("1 + 2", Mode::Expression);
-/// assert!(expr.is_ok());
-/// ```
-///
-/// Alternatively, we can parse a full Python program consisting of multiple lines:
-///
-/// ```
-/// use ruff_python_parser::{Mode, parse};
-///
-/// let source = r#"
-/// class Greeter:
-///
-///   def greet(self):
-///    print("Hello, world!")
-/// "#;
-/// let program = parse(source, Mode::Module);
-/// assert!(program.is_ok());
-/// ```
-///
-/// Additionally, we can parse a Python program containing IPython escapes:
-///
-/// ```
-/// use ruff_python_parser::{Mode, parse};
-///
-/// let source = r#"
-/// %timeit 1 + 2
-/// ?str.replace
-/// !ls
-/// "#;
-/// let program = parse(source, Mode::Ipython);
-/// assert!(program.is_ok());
-/// ```
-pub fn parse(source: &str, mode: Mode) -> Result<Mod, ParseError> {
-    let lxr = lexer::lex(source, mode);
-    parse_tokens(lxr.collect(), source, mode)
-}
-
-/// Parse the given Python source code using the specified [`Mode`] and [`TextSize`].
-///
-/// This function allows to specify the location of the the source code, other than
-/// that, it behaves exactly like [`parse`].
-///
-/// # Example
-///
-/// ```
-/// # use ruff_text_size::TextSize;
-/// use ruff_python_parser::{Mode, parse_starts_at};
-///
-/// let source = r#"
-/// def fib(i):
-///    a, b = 0, 1
-///    for _ in range(i):
-///       a, b = b, a + b
-///    return a
-///
-/// print(fib(42))
-/// "#;
-/// let program = parse_starts_at(source, Mode::Module, TextSize::from(0));
-/// assert!(program.is_ok());
-/// ```
-pub fn parse_starts_at(source: &str, mode: Mode, offset: TextSize) -> Result<Mod, ParseError> {
-    let lxr = lexer::lex_starts_at(source, mode, offset);
-    parse_tokens(lxr.collect(), source, mode)
-}
-
-/// Parse an iterator of [`LexResult`]s using the specified [`Mode`].
-///
-/// This could allow you to perform some preprocessing on the tokens before parsing them.
-///
-/// # Example
-///
-/// As an example, instead of parsing a string, we can parse a list of tokens after we generate
-/// them using the [`lexer::lex`] function:
-///
-/// ```
-/// use ruff_python_parser::{lexer::lex, Mode, parse_tokens};
-///
-/// let source = "1 + 2";
-/// let expr = parse_tokens(lex(source, Mode::Expression).collect(), source, Mode::Expression);
-/// assert!(expr.is_ok());
-/// ```
-pub fn parse_tokens(tokens: Vec<LexResult>, source: &str, mode: Mode) -> Result<Mod, ParseError> {
-    if NEW_PARSER.get() {
-        crate::parser::parse_tokens(tokens, source, mode)
-    } else {
-        crate::lalrpop::parse_tokens(tokens, source, mode)
-    }
-}
-
 /// Collect tokens up to and including the first error.
 pub fn tokenize(contents: &str, mode: Mode) -> Vec<LexResult> {
     let mut tokens: Vec<LexResult> = allocate_tokens_vec(contents);
@@ -447,3 +250,28 @@ impl std::fmt::Display for ModeParseError {
         write!(f, r#"mode must be "exec", "eval", "ipython", or "single""#)
     }
 }
+
+#[rustfmt::skip]
+#[allow(unreachable_pub)]
+#[allow(clippy::type_complexity)]
+#[allow(clippy::extra_unused_lifetimes)]
+#[allow(clippy::needless_lifetimes)]
+#[allow(clippy::unused_self)]
+#[allow(clippy::cast_sign_loss)]
+#[allow(clippy::default_trait_access)]
+#[allow(clippy::let_unit_value)]
+#[allow(clippy::just_underscores_and_digits)]
+#[allow(clippy::no_effect_underscore_binding)]
+#[allow(clippy::trivially_copy_pass_by_ref)]
+#[allow(clippy::option_option)]
+#[allow(clippy::unnecessary_wraps)]
+#[allow(clippy::uninlined_format_args)]
+#[allow(clippy::cloned_instead_of_copied)]
+mod python {
+
+    #[cfg(feature = "lalrpop")]
+    include!(concat!(env!("OUT_DIR"), "/src/python.rs"));
+
+    #[cfg(not(feature = "lalrpop"))]
+    include!("python.rs");
+}

crates/ruff_python_parser/src/parser.rs

@@ -1,8 +1,576 @@
+//! Contains the interface to the Python `ruff_python_parser`.
+//!
+//! Functions in this module can be used to parse Python code into an [Abstract Syntax Tree]
+//! (AST) that is then transformed into bytecode.
+//!
+//! There are three ways to parse Python code corresponding to the different [`Mode`]s
+//! defined in the [`mode`] module.
+//!
+//! All functions return a [`Result`](std::result::Result) containing the parsed AST or
+//! a [`ParseError`] if parsing failed.
+//!
+//! [Abstract Syntax Tree]: https://en.wikipedia.org/wiki/Abstract_syntax_tree
+//! [`Mode`]: crate::mode
+
+use itertools::Itertools;
+pub(super) use lalrpop_util::ParseError as LalrpopError;
+
+use ruff_python_ast::{
+    Expr, ExprAttribute, ExprAwait, ExprBinOp, ExprBoolOp, ExprBooleanLiteral, ExprBytesLiteral,
+    ExprCall, ExprCompare, ExprDict, ExprDictComp, ExprEllipsisLiteral, ExprFString, ExprGenerator,
+    ExprIf, ExprIpyEscapeCommand, ExprLambda, ExprList, ExprListComp, ExprName, ExprNamed,
+    ExprNoneLiteral, ExprNumberLiteral, ExprSet, ExprSetComp, ExprSlice, ExprStarred,
+    ExprStringLiteral, ExprSubscript, ExprTuple, ExprUnaryOp, ExprYield, ExprYieldFrom, Mod,
+    ModModule, Suite,
+};
+use ruff_text_size::{Ranged, TextRange, TextSize};
+
+use crate::lexer::{lex, lex_starts_at, LexResult};
+use crate::token_source::TokenSource;
+use crate::{
+    lexer::{self, LexicalError, LexicalErrorType},
+    python,
+    token::Tok,
+    tokenize_all, Mode,
+};
+
+/// Parse a full Python program usually consisting of multiple lines.
+///
+/// This is a convenience function that can be used to parse a full Python program without having to
+/// specify the [`Mode`] or the location. It is probably what you want to use most of the time.
+///
+/// # Example
+///
+/// For example, parsing a simple function definition and a call to that function:
+///
+/// ```
+/// use ruff_python_parser as parser;
+/// let source = r#"
+/// def foo():
+///    return 42
+///
+/// print(foo())
+/// "#;
+/// let program = parser::parse_program(source);
+/// assert!(program.is_ok());
+/// ```
+pub fn parse_program(source: &str) -> Result<ModModule, ParseError> {
+    match parse_tokens(tokenize_all(source, Mode::Module), source, Mode::Module)? {
+        Mod::Module(m) => Ok(m),
+        Mod::Expression(_) => unreachable!("Mode::Module doesn't return other variant"),
+    }
+}
+
+pub fn parse_suite(source: &str) -> Result<Suite, ParseError> {
+    parse_program(source).map(|m| m.body)
+}
+
+/// Parses a single Python expression.
+///
+/// This convenience function can be used to parse a single expression without having to
+/// specify the Mode or the location.
+///
+/// # Example
+///
+/// For example, parsing a single expression denoting the addition of two numbers:
+///
+///  ```
+/// use ruff_python_parser as parser;
+/// let expr = parser::parse_expression("1 + 2");
+///
+/// assert!(expr.is_ok());
+///
+/// ```
+pub fn parse_expression(source: &str) -> Result<Expr, ParseError> {
+    let lexer = lex(source, Mode::Expression);
+    match parse_tokens(lexer.collect(), source, Mode::Expression)? {
+        Mod::Expression(expression) => Ok(*expression.body),
+        Mod::Module(_m) => unreachable!("Mode::Expression doesn't return other variant"),
+    }
+}
+
+/// Parses a Python expression from a given location.
+///
+/// This function allows to specify the location of the expression in the source code, other than
+/// that, it behaves exactly like [`parse_expression`].
+///
+/// # Example
+///
+/// Parsing a single expression denoting the addition of two numbers, but this time specifying a different,
+/// somewhat silly, location:
+///
+/// ```
+/// use ruff_python_parser::{parse_expression_starts_at};
+/// # use ruff_text_size::TextSize;
+///
+/// let expr = parse_expression_starts_at("1 + 2", TextSize::from(400));
+/// assert!(expr.is_ok());
+/// ```
+pub fn parse_expression_starts_at(source: &str, offset: TextSize) -> Result<Expr, ParseError> {
+    let lexer = lex_starts_at(source, Mode::Module, offset);
+    match parse_tokens(lexer.collect(), source, Mode::Expression)? {
+        Mod::Expression(expression) => Ok(*expression.body),
+        Mod::Module(_m) => unreachable!("Mode::Expression doesn't return other variant"),
+    }
+}
+
+/// Parse the given Python source code using the specified [`Mode`].
+///
+/// This function is the most general function to parse Python code. Based on the [`Mode`] supplied,
+/// it can be used to parse a single expression, a full Python program, an interactive expression
+/// or a Python program containing IPython escape commands.
+///
+/// # Example
+///
+/// If we want to parse a simple expression, we can use the [`Mode::Expression`] mode during
+/// parsing:
+///
+/// ```
+/// use ruff_python_parser::{Mode, parse};
+///
+/// let expr = parse("1 + 2", Mode::Expression);
+/// assert!(expr.is_ok());
+/// ```
+///
+/// Alternatively, we can parse a full Python program consisting of multiple lines:
+///
+/// ```
+/// use ruff_python_parser::{Mode, parse};
+///
+/// let source = r#"
+/// class Greeter:
+///
+///   def greet(self):
+///    print("Hello, world!")
+/// "#;
+/// let program = parse(source, Mode::Module);
+/// assert!(program.is_ok());
+/// ```
+///
+/// Additionally, we can parse a Python program containing IPython escapes:
+///
+/// ```
+/// use ruff_python_parser::{Mode, parse};
+///
+/// let source = r#"
+/// %timeit 1 + 2
+/// ?str.replace
+/// !ls
+/// "#;
+/// let program = parse(source, Mode::Ipython);
+/// assert!(program.is_ok());
+/// ```
+pub fn parse(source: &str, mode: Mode) -> Result<Mod, ParseError> {
+    parse_starts_at(source, mode, TextSize::default())
+}
+
+/// Parse the given Python source code using the specified [`Mode`] and [`TextSize`].
+///
+/// This function allows to specify the location of the the source code, other than
+/// that, it behaves exactly like [`parse`].
+///
+/// # Example
+///
+/// ```
+/// # use ruff_text_size::TextSize;
+/// use ruff_python_parser::{Mode, parse_starts_at};
+///
+/// let source = r#"
+/// def fib(i):
+///    a, b = 0, 1
+///    for _ in range(i):
+///       a, b = b, a + b
+///    return a
+///
+/// print(fib(42))
+/// "#;
+/// let program = parse_starts_at(source, Mode::Module, TextSize::from(0));
+/// assert!(program.is_ok());
+/// ```
+pub fn parse_starts_at(source: &str, mode: Mode, offset: TextSize) -> Result<Mod, ParseError> {
+    let lxr = lexer::lex_starts_at(source, mode, offset);
+    parse_tokens(lxr.collect(), source, mode)
+}
+
+/// Parse an iterator of [`LexResult`]s using the specified [`Mode`].
+///
+/// This could allow you to perform some preprocessing on the tokens before parsing them.
+///
+/// # Example
+///
+/// As an example, instead of parsing a string, we can parse a list of tokens after we generate
+/// them using the [`lexer::lex`] function:
+///
+/// ```
+/// use ruff_python_parser::{lexer::lex, Mode, parse_tokens};
+///
+/// let source = "1 + 2";
+/// let expr = parse_tokens(lex(source, Mode::Expression).collect(), source, Mode::Expression);
+/// assert!(expr.is_ok());
+/// ```
+pub fn parse_tokens(tokens: Vec<LexResult>, source: &str, mode: Mode) -> Result<Mod, ParseError> {
+    let marker_token = (Tok::start_marker(mode), TextRange::default());
+    let lexer = std::iter::once(Ok(marker_token)).chain(TokenSource::new(tokens));
+    python::TopParser::new()
+        .parse(
+            source,
+            mode,
+            lexer.map_ok(|(t, range)| (range.start(), t, range.end())),
+        )
+        .map_err(parse_error_from_lalrpop)
+}
+
+/// Represents represent errors that occur during parsing and are
+/// returned by the `parse_*` functions.
+
+#[derive(Debug, PartialEq)]
+pub struct ParseError {
+    pub error: ParseErrorType,
+    pub offset: TextSize,
+}
+
+impl std::ops::Deref for ParseError {
+    type Target = ParseErrorType;
+
+    fn deref(&self) -> &Self::Target {
+        &self.error
+    }
+}
+
+impl std::error::Error for ParseError {
+    fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
+        Some(&self.error)
+    }
+}
+
+impl std::fmt::Display for ParseError {
+    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+        write!(
+            f,
+            "{} at byte offset {}",
+            &self.error,
+            u32::from(self.offset)
+        )
+    }
+}
+
+/// Represents the different types of errors that can occur during parsing.
+#[derive(Debug, PartialEq)]
+pub enum ParseErrorType {
+    /// Parser encountered an unexpected end of input
+    Eof,
+    /// Parser encountered an extra token
+    ExtraToken(Tok),
+    /// Parser encountered an invalid token
+    InvalidToken,
+    /// Parser encountered an unexpected token
+    UnrecognizedToken(Tok, Option<String>),
+    // Maps to `User` type from `lalrpop-util`
+    /// Parser encountered an error during lexing.
+    Lexical(LexicalErrorType),
+}
+
+impl std::error::Error for ParseErrorType {}
+
+// Convert `lalrpop_util::ParseError` to our internal type
+fn parse_error_from_lalrpop(err: LalrpopError<TextSize, Tok, LexicalError>) -> ParseError {
+    match err {
+        // TODO: Are there cases where this isn't an EOF?
+        LalrpopError::InvalidToken { location } => ParseError {
+            error: ParseErrorType::Eof,
+            offset: location,
+        },
+        LalrpopError::ExtraToken { token } => ParseError {
+            error: ParseErrorType::ExtraToken(token.1),
+            offset: token.0,
+        },
+        LalrpopError::User { error } => ParseError {
+            offset: error.location(),
+            error: ParseErrorType::Lexical(error.into_error()),
+        },
+        LalrpopError::UnrecognizedToken { token, expected } => {
+            // Hacky, but it's how CPython does it. See PyParser_AddToken,
+            // in particular "Only one possible expected token" comment.
+            let expected = (expected.len() == 1).then(|| expected[0].clone());
+            ParseError {
+                error: ParseErrorType::UnrecognizedToken(token.1, expected),
+                offset: token.0,
+            }
+        }
+        LalrpopError::UnrecognizedEof { location, expected } => {
+            // This could be an initial indentation error that we should ignore
+            let indent_error = expected == ["Indent"];
+            if indent_error {
+                ParseError {
+                    error: ParseErrorType::Lexical(LexicalErrorType::IndentationError),
+                    offset: location,
+                }
+            } else {
+                ParseError {
+                    error: ParseErrorType::Eof,
+                    offset: location,
+                }
+            }
+        }
+    }
+}
+
+impl std::fmt::Display for ParseErrorType {
+    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+        match *self {
+            ParseErrorType::Eof => write!(f, "Got unexpected EOF"),
+            ParseErrorType::ExtraToken(ref tok) => write!(f, "Got extraneous token: {tok:?}"),
+            ParseErrorType::InvalidToken => write!(f, "Got invalid token"),
+            ParseErrorType::UnrecognizedToken(ref tok, ref expected) => {
+                if *tok == Tok::Indent {
+                    write!(f, "unexpected indent")
+                } else if expected.as_deref() == Some("Indent") {
+                    write!(f, "expected an indented block")
+                } else {
+                    write!(f, "invalid syntax. Got unexpected token {tok}")
+                }
+            }
+            ParseErrorType::Lexical(ref error) => write!(f, "{error}"),
+        }
+    }
+}
+
+impl ParseErrorType {
+    /// Returns true if the error is an indentation error.
+    pub fn is_indentation_error(&self) -> bool {
+        match self {
+            ParseErrorType::Lexical(LexicalErrorType::IndentationError) => true,
+            ParseErrorType::UnrecognizedToken(token, expected) => {
+                *token == Tok::Indent || expected.clone() == Some("Indent".to_owned())
+            }
+            _ => false,
+        }
+    }
+
+    /// Returns true if the error is a tab error.
+    pub fn is_tab_error(&self) -> bool {
+        matches!(
+            self,
+            ParseErrorType::Lexical(LexicalErrorType::TabError | LexicalErrorType::TabsAfterSpaces)
+        )
+    }
+}
+
+impl From<LexicalError> for ParseError {
+    fn from(error: LexicalError) -> Self {
+        ParseError {
+            offset: error.location(),
+            error: ParseErrorType::Lexical(error.into_error()),
+        }
+    }
+}
+
+/// An expression that may be parenthesized.
+#[derive(Clone, Debug)]
+pub(super) struct ParenthesizedExpr {
+    /// The range of the expression, including any parentheses.
+    pub(super) range: TextRange,
+    /// The underlying expression.
+    pub(super) expr: Expr,
+}
+
+impl ParenthesizedExpr {
+    /// Returns `true` if the expression is parenthesized.
+    pub(super) fn is_parenthesized(&self) -> bool {
+        self.range.start() != self.expr.range().start()
+    }
+}
+
+impl Ranged for ParenthesizedExpr {
+    fn range(&self) -> TextRange {
+        self.range
+    }
+}
+impl From<Expr> for ParenthesizedExpr {
+    fn from(expr: Expr) -> Self {
+        ParenthesizedExpr {
+            range: expr.range(),
+            expr,
+        }
+    }
+}
+impl From<ParenthesizedExpr> for Expr {
+    fn from(parenthesized_expr: ParenthesizedExpr) -> Self {
+        parenthesized_expr.expr
+    }
+}
+impl From<ExprIpyEscapeCommand> for ParenthesizedExpr {
+    fn from(payload: ExprIpyEscapeCommand) -> Self {
+        Expr::IpyEscapeCommand(payload).into()
+    }
+}
+impl From<ExprBoolOp> for ParenthesizedExpr {
+    fn from(payload: ExprBoolOp) -> Self {
+        Expr::BoolOp(payload).into()
+    }
+}
+impl From<ExprNamed> for ParenthesizedExpr {
+    fn from(payload: ExprNamed) -> Self {
+        Expr::Named(payload).into()
+    }
+}
+impl From<ExprBinOp> for ParenthesizedExpr {
+    fn from(payload: ExprBinOp) -> Self {
+        Expr::BinOp(payload).into()
+    }
+}
+impl From<ExprUnaryOp> for ParenthesizedExpr {
+    fn from(payload: ExprUnaryOp) -> Self {
+        Expr::UnaryOp(payload).into()
+    }
+}
+impl From<ExprLambda> for ParenthesizedExpr {
+    fn from(payload: ExprLambda) -> Self {
+        Expr::Lambda(payload).into()
+    }
+}
+impl From<ExprIf> for ParenthesizedExpr {
+    fn from(payload: ExprIf) -> Self {
+        Expr::If(payload).into()
+    }
+}
+impl From<ExprDict> for ParenthesizedExpr {
+    fn from(payload: ExprDict) -> Self {
+        Expr::Dict(payload).into()
+    }
+}
+impl From<ExprSet> for ParenthesizedExpr {
+    fn from(payload: ExprSet) -> Self {
+        Expr::Set(payload).into()
+    }
+}
+impl From<ExprListComp> for ParenthesizedExpr {
+    fn from(payload: ExprListComp) -> Self {
+        Expr::ListComp(payload).into()
+    }
+}
+impl From<ExprSetComp> for ParenthesizedExpr {
+    fn from(payload: ExprSetComp) -> Self {
+        Expr::SetComp(payload).into()
+    }
+}
+impl From<ExprDictComp> for ParenthesizedExpr {
+    fn from(payload: ExprDictComp) -> Self {
+        Expr::DictComp(payload).into()
+    }
+}
+impl From<ExprGenerator> for ParenthesizedExpr {
+    fn from(payload: ExprGenerator) -> Self {
+        Expr::Generator(payload).into()
+    }
+}
+impl From<ExprAwait> for ParenthesizedExpr {
+    fn from(payload: ExprAwait) -> Self {
+        Expr::Await(payload).into()
+    }
+}
+impl From<ExprYield> for ParenthesizedExpr {
+    fn from(payload: ExprYield) -> Self {
+        Expr::Yield(payload).into()
+    }
+}
+impl From<ExprYieldFrom> for ParenthesizedExpr {
+    fn from(payload: ExprYieldFrom) -> Self {
+        Expr::YieldFrom(payload).into()
+    }
+}
+impl From<ExprCompare> for ParenthesizedExpr {
+    fn from(payload: ExprCompare) -> Self {
+        Expr::Compare(payload).into()
+    }
+}
+impl From<ExprCall> for ParenthesizedExpr {
+    fn from(payload: ExprCall) -> Self {
+        Expr::Call(payload).into()
+    }
+}
+impl From<ExprFString> for ParenthesizedExpr {
+    fn from(payload: ExprFString) -> Self {
+        Expr::FString(payload).into()
+    }
+}
+impl From<ExprStringLiteral> for ParenthesizedExpr {
+    fn from(payload: ExprStringLiteral) -> Self {
+        Expr::StringLiteral(payload).into()
+    }
+}
+impl From<ExprBytesLiteral> for ParenthesizedExpr {
+    fn from(payload: ExprBytesLiteral) -> Self {
+        Expr::BytesLiteral(payload).into()
+    }
+}
+impl From<ExprNumberLiteral> for ParenthesizedExpr {
+    fn from(payload: ExprNumberLiteral) -> Self {
+        Expr::NumberLiteral(payload).into()
+    }
+}
+impl From<ExprBooleanLiteral> for ParenthesizedExpr {
+    fn from(payload: ExprBooleanLiteral) -> Self {
+        Expr::BooleanLiteral(payload).into()
+    }
+}
+impl From<ExprNoneLiteral> for ParenthesizedExpr {
+    fn from(payload: ExprNoneLiteral) -> Self {
+        Expr::NoneLiteral(payload).into()
+    }
+}
+impl From<ExprEllipsisLiteral> for ParenthesizedExpr {
+    fn from(payload: ExprEllipsisLiteral) -> Self {
+        Expr::EllipsisLiteral(payload).into()
+    }
+}
+impl From<ExprAttribute> for ParenthesizedExpr {
+    fn from(payload: ExprAttribute) -> Self {
+        Expr::Attribute(payload).into()
+    }
+}
+impl From<ExprSubscript> for ParenthesizedExpr {
+    fn from(payload: ExprSubscript) -> Self {
+        Expr::Subscript(payload).into()
+    }
+}
+impl From<ExprStarred> for ParenthesizedExpr {
+    fn from(payload: ExprStarred) -> Self {
+        Expr::Starred(payload).into()
+    }
+}
+impl From<ExprName> for ParenthesizedExpr {
+    fn from(payload: ExprName) -> Self {
+        Expr::Name(payload).into()
+    }
+}
+impl From<ExprList> for ParenthesizedExpr {
+    fn from(payload: ExprList) -> Self {
+        Expr::List(payload).into()
+    }
+}
+impl From<ExprTuple> for ParenthesizedExpr {
+    fn from(payload: ExprTuple) -> Self {
+        Expr::Tuple(payload).into()
+    }
+}
+impl From<ExprSlice> for ParenthesizedExpr {
+    fn from(payload: ExprSlice) -> Self {
+        Expr::Slice(payload).into()
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use insta::assert_debug_snapshot;
 
-    use crate::{lexer, parse, parse_expression, parse_suite, parse_tokens, Mode};
+    use super::*;
+
+    #[cfg(target_pointer_width = "64")]
+    #[test]
+    fn size_assertions() {
+        assert_eq!(std::mem::size_of::<ParenthesizedExpr>(), 72);
+    }
 
     #[test]
     fn test_parse_empty() {
@@ -274,6 +842,7 @@ with ((yield from a)): pass
         for source in [
             "with 0,: pass",
             "with 0 as x,: pass",
+            "with 0 as *x: pass",
             "with *a: pass",
             "with *a as x: pass",
             "with (*a): pass",
@@ -868,11 +1437,11 @@ a = 1
 %timeit a == 1
     "
         .trim();
-        let lxr = lexer::lex(source, Mode::Ipython);
+        let lxr = lexer::lex_starts_at(source, Mode::Ipython, TextSize::default());
         let parse_err = parse_tokens(lxr.collect(), source, Mode::Module).unwrap_err();
         assert_eq!(
             parse_err.to_string(),
-            "IPython escape commands are only allowed in `Mode::Ipython` at byte range 6..20"
+            "IPython escape commands are only allowed in `Mode::Ipython` at byte offset 6"
                 .to_string()
         );
     }

crates/ruff_python_parser/src/parser/helpers.rs

@@ -1,148 +0,0 @@
-use std::hash::BuildHasherDefault;
-
-use ast::CmpOp;
-use ruff_python_ast::{self as ast, Expr, ExprContext};
-use rustc_hash::FxHashSet;
-
-use crate::{ParseError, ParseErrorType, TokenKind};
-
-/// Set the `ctx` for `Expr::Id`, `Expr::Attribute`, `Expr::Subscript`, `Expr::Starred`,
-/// `Expr::Tuple` and `Expr::List`. If `expr` is either `Expr::Tuple` or `Expr::List`,
-/// recursively sets the `ctx` for their elements.
-pub(super) fn set_expr_ctx(expr: &mut Expr, new_ctx: ExprContext) {
-    match expr {
-        Expr::Name(ast::ExprName { ctx, .. })
-        | Expr::Attribute(ast::ExprAttribute { ctx, .. })
-        | Expr::Subscript(ast::ExprSubscript { ctx, .. }) => *ctx = new_ctx,
-        Expr::Starred(ast::ExprStarred { value, ctx, .. }) => {
-            *ctx = new_ctx;
-            set_expr_ctx(value, new_ctx);
-        }
-        Expr::UnaryOp(ast::ExprUnaryOp { operand, .. }) => {
-            set_expr_ctx(operand, new_ctx);
-        }
-        Expr::List(ast::ExprList { elts, ctx, .. })
-        | Expr::Tuple(ast::ExprTuple { elts, ctx, .. }) => {
-            *ctx = new_ctx;
-            elts.iter_mut()
-                .for_each(|element| set_expr_ctx(element, new_ctx));
-        }
-        _ => {}
-    }
-}
-
-/// Check if the given expression is itself or contains an expression that is
-/// valid on the left hand side of an assignment. For example, identifiers,
-/// starred expressions, attribute expressions, subscript expressions,
-/// list and tuple unpacking are valid assignment targets.
-pub(super) fn is_valid_assignment_target(expr: &Expr) -> bool {
-    match expr {
-        Expr::Starred(ast::ExprStarred { value, .. }) => is_valid_assignment_target(value),
-        Expr::List(ast::ExprList { elts, .. }) | Expr::Tuple(ast::ExprTuple { elts, .. }) => {
-            elts.iter().all(is_valid_assignment_target)
-        }
-        Expr::Name(_) | Expr::Attribute(_) | Expr::Subscript(_) => true,
-        _ => false,
-    }
-}
-
-/// Check if the given expression is itself or contains an expression that is
-/// valid on the left hand side of an augmented assignment. For example, identifiers,
-/// attribute and subscript expressions are valid augmented assignment targets.
-pub(super) fn is_valid_aug_assignment_target(expr: &Expr) -> bool {
-    matches!(
-        expr,
-        Expr::Name(_) | Expr::Attribute(_) | Expr::Subscript(_)
-    )
-}
-
-/// Check if the given expression is itself or contains an expression that is
-/// valid as a target of a `del` statement.
-pub(super) fn is_valid_del_target(expr: &Expr) -> bool {
-    // https://github.com/python/cpython/blob/d864b0094f9875c5613cbb0b7f7f3ca8f1c6b606/Parser/action_helpers.c#L1150-L1180
-    match expr {
-        Expr::List(ast::ExprList { elts, .. }) | Expr::Tuple(ast::ExprTuple { elts, .. }) => {
-            elts.iter().all(is_valid_del_target)
-        }
-        Expr::Name(_) | Expr::Attribute(_) | Expr::Subscript(_) => true,
-        _ => false,
-    }
-}
-
-/// Converts a [`TokenKind`] array of size 2 to its correspondent [`CmpOp`].
-pub(super) fn token_kind_to_cmp_op(kind: [TokenKind; 2]) -> Result<CmpOp, ()> {
-    Ok(match kind {
-        [TokenKind::Is, TokenKind::Not] => CmpOp::IsNot,
-        [TokenKind::Is, _] => CmpOp::Is,
-        [TokenKind::In, _] => CmpOp::In,
-        [TokenKind::EqEqual, _] => CmpOp::Eq,
-        [TokenKind::Less, _] => CmpOp::Lt,
-        [TokenKind::Greater, _] => CmpOp::Gt,
-        [TokenKind::NotEqual, _] => CmpOp::NotEq,
-        [TokenKind::LessEqual, _] => CmpOp::LtE,
-        [TokenKind::GreaterEqual, _] => CmpOp::GtE,
-        [TokenKind::Not, TokenKind::In] => CmpOp::NotIn,
-        _ => return Err(()),
-    })
-}
-
-// Perform validation of function/lambda parameters in a function definition.
-pub(super) fn validate_parameters(parameters: &ast::Parameters) -> Result<(), ParseError> {
-    let mut all_arg_names = FxHashSet::with_capacity_and_hasher(
-        parameters.posonlyargs.len()
-            + parameters.args.len()
-            + usize::from(parameters.vararg.is_some())
-            + parameters.kwonlyargs.len()
-            + usize::from(parameters.kwarg.is_some()),
-        BuildHasherDefault::default(),
-    );
-
-    let posonlyargs = parameters.posonlyargs.iter();
-    let args = parameters.args.iter();
-    let kwonlyargs = parameters.kwonlyargs.iter();
-
-    let vararg: Option<&ast::Parameter> = parameters.vararg.as_deref();
-    let kwarg: Option<&ast::Parameter> = parameters.kwarg.as_deref();
-
-    for arg in posonlyargs
-        .chain(args)
-        .chain(kwonlyargs)
-        .map(|arg| &arg.parameter)
-        .chain(vararg)
-        .chain(kwarg)
-    {
-        let range = arg.range;
-        let arg_name = arg.name.as_str();
-        if !all_arg_names.insert(arg_name) {
-            return Err(ParseError {
-                error: ParseErrorType::DuplicateArgumentError(arg_name.to_string()),
-                location: range,
-            });
-        }
-    }
-
-    Ok(())
-}
-
-pub(super) fn validate_arguments(arguments: &ast::Arguments) -> Result<(), ParseError> {
-    let mut all_arg_names = FxHashSet::with_capacity_and_hasher(
-        arguments.keywords.len(),
-        BuildHasherDefault::default(),
-    );
-
-    for (name, range) in arguments
-        .keywords
-        .iter()
-        .filter_map(|argument| argument.arg.as_ref().map(|arg| (arg, argument.range)))
-    {
-        let arg_name = name.as_str();
-        if !all_arg_names.insert(arg_name) {
-            return Err(ParseError {
-                error: ParseErrorType::DuplicateKeywordArgumentError(arg_name.to_string()),
-                location: range,
-            });
-        }
-    }
-
-    Ok(())
-}

crates/ruff_python_parser/src/parser/pattern.rs

@@ -1,671 +0,0 @@
-use ruff_python_ast::{self as ast, Expr, ExprContext, Number, Operator, Pattern, Singleton};
-use ruff_text_size::{Ranged, TextSize};
-
-use crate::parser::progress::ParserProgress;
-use crate::parser::{Parser, SequenceMatchPatternParentheses};
-use crate::token_set::TokenSet;
-use crate::{ParseErrorType, Tok, TokenKind};
-
-use super::RecoveryContextKind;
-
-/// The set of tokens that can start a literal pattern.
-const LITERAL_PATTERN_START_SET: TokenSet = TokenSet::new([
-    TokenKind::None,
-    TokenKind::True,
-    TokenKind::False,
-    TokenKind::String,
-    TokenKind::Int,
-    TokenKind::Float,
-    TokenKind::Complex,
-]);
-
-/// The set of tokens that can start a pattern.
-const PATTERN_START_SET: TokenSet = TokenSet::new([
-    // Star pattern
-    TokenKind::Star,
-    // Capture pattern
-    // Wildcard pattern ('_' is a name token)
-    // Value pattern (name or attribute)
-    // Class pattern
-    TokenKind::Name,
-    // Group pattern
-    TokenKind::Lpar,
-    // Sequence pattern
-    TokenKind::Lsqb,
-    // Mapping pattern
-    TokenKind::Lbrace,
-])
-.union(LITERAL_PATTERN_START_SET);
-
-/// The set of tokens that can start a mapping pattern.
-const MAPPING_PATTERN_START_SET: TokenSet = TokenSet::new([
-    // Double star pattern
-    TokenKind::DoubleStar,
-    // Value pattern
-    TokenKind::Name,
-])
-.union(LITERAL_PATTERN_START_SET);
-
-impl<'src> Parser<'src> {
-    /// Returns `true` if the current token is a valid start of a pattern.
-    pub(super) fn at_pattern_start(&self) -> bool {
-        self.at_ts(PATTERN_START_SET)
-    }
-
-    /// Returns `true` if the current token is a valid start of a mapping pattern.
-    pub(super) fn at_mapping_pattern_start(&self) -> bool {
-        self.at_ts(MAPPING_PATTERN_START_SET)
-    }
-
-    /// Entry point to start parsing a pattern.
-    ///
-    /// See: <https://docs.python.org/3/reference/compound_stmts.html#grammar-token-python-grammar-patterns>
-    pub(super) fn parse_match_patterns(&mut self) -> Pattern {
-        let start = self.node_start();
-        let pattern = self.parse_match_pattern();
-
-        if self.at(TokenKind::Comma) {
-            Pattern::MatchSequence(self.parse_sequence_match_pattern(pattern, start, None))
-        } else {
-            pattern
-        }
-    }
-
-    /// Parses an `or_pattern` or an `as_pattern`.
-    ///
-    /// See: <https://docs.python.org/3/reference/compound_stmts.html#grammar-token-python-grammar-pattern>
-    fn parse_match_pattern(&mut self) -> Pattern {
-        let start = self.node_start();
-        let mut lhs = self.parse_match_pattern_lhs();
-
-        // Or pattern
-        if self.at(TokenKind::Vbar) {
-            let mut patterns = vec![lhs];
-            let mut progress = ParserProgress::default();
-
-            while self.eat(TokenKind::Vbar) {
-                progress.assert_progressing(self);
-                let pattern = self.parse_match_pattern_lhs();
-                patterns.push(pattern);
-            }
-
-            lhs = Pattern::MatchOr(ast::PatternMatchOr {
-                range: self.node_range(start),
-                patterns,
-            });
-        }
-
-        // As pattern
-        if self.eat(TokenKind::As) {
-            let ident = self.parse_identifier();
-            lhs = Pattern::MatchAs(ast::PatternMatchAs {
-                range: self.node_range(start),
-                name: Some(ident),
-                pattern: Some(Box::new(lhs)),
-            });
-        }
-
-        lhs
-    }
-
-    /// Parses a pattern.
-    ///
-    /// See: <https://docs.python.org/3/reference/compound_stmts.html#grammar-token-python-grammar-closed_pattern>
-    fn parse_match_pattern_lhs(&mut self) -> Pattern {
-        let start = self.node_start();
-        let mut lhs = match self.current_token_kind() {
-            TokenKind::Lbrace => Pattern::MatchMapping(self.parse_match_pattern_mapping()),
-            TokenKind::Star => Pattern::MatchStar(self.parse_match_pattern_star()),
-            TokenKind::Lpar | TokenKind::Lsqb => self.parse_delimited_match_pattern(),
-            _ => self.parse_match_pattern_literal(),
-        };
-
-        if self.at(TokenKind::Lpar) {
-            lhs = Pattern::MatchClass(self.parse_match_pattern_class(lhs, start));
-        }
-
-        if self.at(TokenKind::Plus) || self.at(TokenKind::Minus) {
-            let (operator_token, _) = self.next_token();
-            let operator = if matches!(operator_token, Tok::Plus) {
-                Operator::Add
-            } else {
-                Operator::Sub
-            };
-
-            let lhs_value = if let Pattern::MatchValue(lhs) = lhs {
-                if !matches!(&*lhs.value, Expr::NumberLiteral(_) | Expr::UnaryOp(_)) {
-                    self.add_error(
-                        ParseErrorType::OtherError("invalid lhs pattern".to_string()),
-                        &lhs,
-                    );
-                }
-                lhs.value
-            } else {
-                self.add_error(
-                    ParseErrorType::OtherError("invalid lhs pattern".to_string()),
-                    &lhs,
-                );
-
-                // In case it's not a valid LHS pattern, we'll use an empty `Expr::Name`
-                // to indicate that.
-                Box::new(Expr::Name(ast::ExprName {
-                    id: String::new(),
-                    ctx: ExprContext::Invalid,
-                    range: lhs.range(),
-                }))
-            };
-
-            let rhs_pattern = self.parse_match_pattern_lhs();
-            let rhs_value = if let Pattern::MatchValue(rhs) = rhs_pattern {
-                if !matches!(
-                    &*rhs.value,
-                    Expr::NumberLiteral(ast::ExprNumberLiteral {
-                        value: ast::Number::Complex { .. },
-                        ..
-                    })
-                ) {
-                    self.add_error(
-                        ParseErrorType::OtherError(
-                            "imaginary number required in complex literal".to_string(),
-                        ),
-                        &rhs,
-                    );
-                }
-                rhs.value
-            } else {
-                self.add_error(
-                    ParseErrorType::OtherError("invalid rhs pattern".to_string()),
-                    rhs_pattern.range(),
-                );
-
-                // In case it's not a valid RHS pattern, we'll use an empty `Expr::Name`
-                // to indicate that.
-                Box::new(Expr::Name(ast::ExprName {
-                    id: String::new(),
-                    ctx: ExprContext::Invalid,
-                    range: rhs_pattern.range(),
-                }))
-            };
-
-            let range = self.node_range(start);
-
-            return Pattern::MatchValue(ast::PatternMatchValue {
-                value: Box::new(Expr::BinOp(ast::ExprBinOp {
-                    left: lhs_value,
-                    op: operator,
-                    right: rhs_value,
-                    range,
-                })),
-                range,
-            });
-        }
-
-        lhs
-    }
-
-    /// Parses a mapping pattern.
-    ///
-    /// # Panics
-    ///
-    /// If the parser isn't positioned at a `{` token.
-    ///
-    /// See: <https://docs.python.org/3/reference/compound_stmts.html#mapping-patterns>
-    fn parse_match_pattern_mapping(&mut self) -> ast::PatternMatchMapping {
-        let start = self.node_start();
-        self.bump(TokenKind::Lbrace);
-
-        let mut keys = vec![];
-        let mut patterns = vec![];
-        let mut rest = None;
-
-        self.parse_comma_separated_list(RecoveryContextKind::MatchPatternMapping, |parser| {
-            if parser.eat(TokenKind::DoubleStar) {
-                rest = Some(parser.parse_identifier());
-            } else {
-                let key = match parser.parse_match_pattern_lhs() {
-                    Pattern::MatchValue(ast::PatternMatchValue { value, .. }) => *value,
-                    Pattern::MatchSingleton(ast::PatternMatchSingleton { value, range }) => {
-                        match value {
-                            Singleton::None => Expr::NoneLiteral(ast::ExprNoneLiteral { range }),
-                            Singleton::True => {
-                                Expr::BooleanLiteral(ast::ExprBooleanLiteral { value: true, range })
-                            }
-                            Singleton::False => Expr::BooleanLiteral(ast::ExprBooleanLiteral {
-                                value: false,
-                                range,
-                            }),
-                        }
-                    }
-                    pattern => {
-                        parser.add_error(
-                            ParseErrorType::OtherError("invalid mapping pattern key".to_string()),
-                            &pattern,
-                        );
-                        Expr::Name(ast::ExprName {
-                            id: String::new(),
-                            ctx: ExprContext::Invalid,
-                            range: pattern.range(),
-                        })
-                    }
-                };
-                keys.push(key);
-
-                parser.expect(TokenKind::Colon);
-
-                patterns.push(parser.parse_match_pattern());
-            }
-        });
-
-        // TODO(dhruvmanila): There can't be any other pattern after a `**` pattern.
-        // TODO(dhruvmanila): Duplicate literal keys should raise a SyntaxError.
-
-        self.expect(TokenKind::Rbrace);
-
-        ast::PatternMatchMapping {
-            range: self.node_range(start),
-            keys,
-            patterns,
-            rest,
-        }
-    }
-
-    /// Parses a star pattern.
-    ///
-    /// # Panics
-    ///
-    /// If the parser isn't positioned at a `*` token.
-    ///
-    /// See: <https://docs.python.org/3/reference/compound_stmts.html#grammar-token-python-grammar-star_pattern>
-    fn parse_match_pattern_star(&mut self) -> ast::PatternMatchStar {
-        let start = self.node_start();
-        self.bump(TokenKind::Star);
-
-        let ident = self.parse_identifier();
-
-        ast::PatternMatchStar {
-            range: self.node_range(start),
-            name: if ident.is_valid() && ident.id == "_" {
-                None
-            } else {
-                Some(ident)
-            },
-        }
-    }
-
-    /// Entry point to start parsing a sequence pattern.
-    ///
-    /// # Panics
-    ///
-    /// If the parser isn't positioned at a `(` or `[` token.
-    ///
-    /// See: <https://docs.python.org/3/reference/compound_stmts.html#sequence-patterns>
-    fn parse_delimited_match_pattern(&mut self) -> Pattern {
-        let start = self.node_start();
-        let parentheses = if self.eat(TokenKind::Lpar) {
-            SequenceMatchPatternParentheses::Tuple
-        } else {
-            self.bump(TokenKind::Lsqb);
-            SequenceMatchPatternParentheses::List
-        };
-
-        if matches!(
-            self.current_token_kind(),
-            TokenKind::Newline | TokenKind::Colon
-        ) {
-            self.add_error(
-                ParseErrorType::OtherError(format!(
-                    "missing `{closing}`",
-                    closing = if parentheses.is_list() { "]" } else { ")" }
-                )),
-                self.current_token_range(),
-            );
-        }
-
-        if self.eat(parentheses.closing_kind()) {
-            return Pattern::MatchSequence(ast::PatternMatchSequence {
-                patterns: vec![],
-                range: self.node_range(start),
-            });
-        }
-
-        let mut pattern = self.parse_match_pattern();
-
-        if parentheses.is_list() || self.at(TokenKind::Comma) {
-            pattern = Pattern::MatchSequence(self.parse_sequence_match_pattern(
-                pattern,
-                start,
-                Some(parentheses),
-            ));
-        } else {
-            self.expect(parentheses.closing_kind());
-        }
-
-        pattern
-    }
-
-    /// Parses the rest of a sequence pattern, given the first element.
-    ///
-    /// If the `parentheses` is `None`, it is an [open sequence pattern].
-    ///
-    /// See: <https://docs.python.org/3/reference/compound_stmts.html#sequence-patterns>
-    ///
-    /// [open sequence pattern]: https://docs.python.org/3/reference/compound_stmts.html#grammar-token-python-grammar-open_sequence_pattern
-    fn parse_sequence_match_pattern(
-        &mut self,
-        first_element: Pattern,
-        start: TextSize,
-        parentheses: Option<SequenceMatchPatternParentheses>,
-    ) -> ast::PatternMatchSequence {
-        if parentheses.is_some_and(|parentheses| {
-            self.at(parentheses.closing_kind()) || self.peek_nth(1) == parentheses.closing_kind()
-        }) {
-            // The comma is optional if it is a single-element sequence
-            self.eat(TokenKind::Comma);
-        } else {
-            self.expect(TokenKind::Comma);
-        }
-
-        let mut patterns = vec![first_element];
-
-        self.parse_comma_separated_list(
-            RecoveryContextKind::SequenceMatchPattern(parentheses),
-            |parser| patterns.push(parser.parse_match_pattern()),
-        );
-
-        if let Some(parentheses) = parentheses {
-            self.expect(parentheses.closing_kind());
-        }
-
-        ast::PatternMatchSequence {
-            range: self.node_range(start),
-            patterns,
-        }
-    }
-
-    /// Parses a literal pattern.
-    ///
-    /// See: <https://docs.python.org/3/reference/compound_stmts.html#grammar-token-python-grammar-literal_pattern>
-    fn parse_match_pattern_literal(&mut self) -> Pattern {
-        let start = self.node_start();
-        match self.current_token_kind() {
-            TokenKind::None => {
-                self.bump(TokenKind::None);
-                Pattern::MatchSingleton(ast::PatternMatchSingleton {
-                    value: Singleton::None,
-                    range: self.node_range(start),
-                })
-            }
-            TokenKind::True => {
-                self.bump(TokenKind::True);
-                Pattern::MatchSingleton(ast::PatternMatchSingleton {
-                    value: Singleton::True,
-                    range: self.node_range(start),
-                })
-            }
-            TokenKind::False => {
-                self.bump(TokenKind::False);
-                Pattern::MatchSingleton(ast::PatternMatchSingleton {
-                    value: Singleton::False,
-                    range: self.node_range(start),
-                })
-            }
-            TokenKind::String | TokenKind::FStringStart => {
-                let str = self.parse_strings();
-
-                Pattern::MatchValue(ast::PatternMatchValue {
-                    value: Box::new(str),
-                    range: self.node_range(start),
-                })
-            }
-            TokenKind::Complex => {
-                let (Tok::Complex { real, imag }, _) = self.bump(TokenKind::Complex) else {
-                    unreachable!()
-                };
-                let range = self.node_range(start);
-
-                Pattern::MatchValue(ast::PatternMatchValue {
-                    value: Box::new(Expr::NumberLiteral(ast::ExprNumberLiteral {
-                        value: Number::Complex { real, imag },
-                        range,
-                    })),
-                    range,
-                })
-            }
-            TokenKind::Int => {
-                let (Tok::Int { value }, _) = self.bump(TokenKind::Int) else {
-                    unreachable!()
-                };
-                let range = self.node_range(start);
-
-                Pattern::MatchValue(ast::PatternMatchValue {
-                    value: Box::new(Expr::NumberLiteral(ast::ExprNumberLiteral {
-                        value: Number::Int(value),
-                        range,
-                    })),
-                    range,
-                })
-            }
-            TokenKind::Float => {
-                let (Tok::Float { value }, _) = self.bump(TokenKind::Float) else {
-                    unreachable!()
-                };
-                let range = self.node_range(start);
-
-                Pattern::MatchValue(ast::PatternMatchValue {
-                    value: Box::new(Expr::NumberLiteral(ast::ExprNumberLiteral {
-                        value: Number::Float(value),
-                        range,
-                    })),
-                    range,
-                })
-            }
-            TokenKind::Name if self.peek_nth(1) == TokenKind::Dot => {
-                let (Tok::Name { name }, _) = self.bump(TokenKind::Name) else {
-                    unreachable!()
-                };
-                let id = Expr::Name(ast::ExprName {
-                    id: name.to_string(),
-                    ctx: ExprContext::Load,
-                    range: self.node_range(start),
-                });
-
-                let attribute = self.parse_attr_expr_for_match_pattern(id, start);
-
-                Pattern::MatchValue(ast::PatternMatchValue {
-                    value: Box::new(attribute),
-                    range: self.node_range(start),
-                })
-            }
-            TokenKind::Name => {
-                let (Tok::Name { name }, _) = self.bump(TokenKind::Name) else {
-                    unreachable!()
-                };
-                let range = self.node_range(start);
-
-                Pattern::MatchAs(ast::PatternMatchAs {
-                    range,
-                    pattern: None,
-                    name: if name.contains('_') {
-                        None
-                    } else {
-                        Some(ast::Identifier {
-                            id: name.to_string(),
-                            range,
-                        })
-                    },
-                })
-            }
-            TokenKind::Minus
-                if matches!(
-                    self.peek_nth(1),
-                    TokenKind::Int | TokenKind::Float | TokenKind::Complex
-                ) =>
-            {
-                let parsed_expr = self.parse_lhs_expression();
-
-                let range = self.node_range(start);
-                Pattern::MatchValue(ast::PatternMatchValue {
-                    value: Box::new(parsed_expr.expr),
-                    range,
-                })
-            }
-            kind => {
-                // Upon encountering an unexpected token, return a `Pattern::MatchValue` containing
-                // an empty `Expr::Name`.
-                let invalid_node = if kind.is_keyword() {
-                    Expr::Name(self.parse_name())
-                } else {
-                    self.add_error(
-                        ParseErrorType::OtherError("Expected a pattern".to_string()),
-                        self.current_token_range(),
-                    );
-                    Expr::Name(ast::ExprName {
-                        range: self.missing_node_range(),
-                        id: String::new(),
-                        ctx: ExprContext::Invalid,
-                    })
-                };
-
-                Pattern::MatchValue(ast::PatternMatchValue {
-                    value: Box::new(invalid_node),
-                    range: self.missing_node_range(),
-                })
-            }
-        }
-    }
-
-    /// Parses an attribute expression until the current token is not a `.`.
-    fn parse_attr_expr_for_match_pattern(&mut self, mut lhs: Expr, start: TextSize) -> Expr {
-        while self.current_token_kind() == TokenKind::Dot {
-            lhs = Expr::Attribute(self.parse_attribute_expression(lhs, start));
-        }
-
-        lhs
-    }
-
-    /// Parses the [pattern arguments] in a class pattern.
-    ///
-    /// # Panics
-    ///
-    /// If the parser isn't positioned at a `(` token.
-    ///
-    /// See: <https://docs.python.org/3/reference/compound_stmts.html#class-patterns>
-    ///
-    /// [pattern arguments]: https://docs.python.org/3/reference/compound_stmts.html#grammar-token-python-grammar-pattern_arguments
-    fn parse_match_pattern_class(
-        &mut self,
-        cls: Pattern,
-        start: TextSize,
-    ) -> ast::PatternMatchClass {
-        let arguments_start = self.node_start();
-
-        self.bump(TokenKind::Lpar);
-
-        let mut patterns = vec![];
-        let mut keywords = vec![];
-        let mut has_seen_pattern = false;
-        let mut has_seen_keyword_pattern = false;
-
-        self.parse_comma_separated_list(
-            RecoveryContextKind::MatchPatternClassArguments,
-            |parser| {
-                let pattern_start = parser.node_start();
-                let pattern = parser.parse_match_pattern();
-
-                if parser.eat(TokenKind::Equal) {
-                    has_seen_pattern = false;
-                    has_seen_keyword_pattern = true;
-
-                    let value_pattern = parser.parse_match_pattern();
-
-                    // Key can only be an identifier
-                    if let Pattern::MatchAs(ast::PatternMatchAs {
-                        name: Some(attr), ..
-                    }) = pattern
-                    {
-                        keywords.push(ast::PatternKeyword {
-                            attr,
-                            pattern: value_pattern,
-                            range: parser.node_range(pattern_start),
-                        });
-                    } else {
-                        // In case it's not a valid keyword pattern, we'll add an empty identifier
-                        // to indicate that. This is to avoid dropping the parsed value pattern.
-                        keywords.push(ast::PatternKeyword {
-                            attr: ast::Identifier {
-                                id: String::new(),
-                                range: parser.missing_node_range(),
-                            },
-                            pattern: value_pattern,
-                            range: parser.node_range(pattern_start),
-                        });
-                        parser.add_error(
-                            ParseErrorType::OtherError("Invalid keyword pattern".to_string()),
-                            parser.node_range(pattern_start),
-                        );
-                    }
-                } else {
-                    has_seen_pattern = true;
-                    patterns.push(pattern);
-                }
-
-                if has_seen_keyword_pattern && has_seen_pattern {
-                    parser.add_error(
-                        ParseErrorType::OtherError(
-                            "pattern not allowed after keyword pattern".to_string(),
-                        ),
-                        parser.node_range(pattern_start),
-                    );
-                }
-            },
-        );
-
-        self.expect(TokenKind::Rpar);
-
-        let arguments_range = self.node_range(arguments_start);
-
-        let cls = match cls {
-            Pattern::MatchAs(ast::PatternMatchAs {
-                name: Some(ident), ..
-            }) => Box::new(Expr::Name(if ident.is_valid() {
-                ast::ExprName {
-                    range: ident.range(),
-                    id: ident.id,
-                    ctx: ExprContext::Load,
-                }
-            } else {
-                ast::ExprName {
-                    range: ident.range(),
-                    id: String::new(),
-                    ctx: ExprContext::Invalid,
-                }
-            })),
-            Pattern::MatchValue(ast::PatternMatchValue { value, range: _ })
-                if matches!(value.as_ref(), Expr::Attribute(_)) =>
-            {
-                value
-            }
-            pattern => {
-                self.add_error(
-                    ParseErrorType::OtherError("invalid value for a class pattern".to_string()),
-                    &pattern,
-                );
-                Box::new(Expr::Name(ast::ExprName {
-                    id: String::new(),
-                    ctx: ExprContext::Invalid,
-                    range: pattern.range(),
-                }))
-            }
-        };
-
-        ast::PatternMatchClass {
-            cls,
-            arguments: ast::PatternArguments {
-                patterns,
-                keywords,
-                range: arguments_range,
-            },
-            range: self.node_range(start),
-        }
-    }
-}

crates/ruff_python_parser/src/parser/progress.rs

@@ -1,36 +0,0 @@
-use crate::parser::Parser;
-use crate::TokenKind;
-use ruff_text_size::TextSize;
-
-/// Captures the progress of the parser and allows to test if the parsing is still making progress
-#[derive(Debug, Copy, Clone, Default)]
-pub(super) struct ParserProgress(Option<(TokenKind, TextSize)>);
-
-impl ParserProgress {
-    /// Returns true if the parser has passed this position
-    #[inline]
-    fn has_progressed(self, p: &Parser) -> bool {
-        match self.0 {
-            None => true,
-            Some(snapshot) => snapshot != (p.current_token_kind(), p.current_token_range().start()),
-        }
-    }
-
-    /// Asserts that the parsing is still making progress.
-    ///
-    /// # Panics
-    ///
-    /// Panics if the parser hasn't progressed since the last call.
-    #[inline]
-    pub(super) fn assert_progressing(&mut self, p: &Parser) {
-        assert!(
-            self.has_progressed(p),
-            "The parser is no longer progressing. Stuck at '{}' {:?}:{:?}",
-            p.src_text(p.current_token_range()),
-            p.current_token_kind(),
-            p.current_token_range(),
-        );
-
-        self.0 = Some((p.current_token_kind(), p.current_token_range().start()));
-    }
-}

crates/ruff_python_parser/src/parser/tests.rs

@@ -1,15 +0,0 @@
-mod parser;
-mod suite;
-
-// This is a sanity test for what looks like an ipython directive being
-// assigned to. Although this doesn't actually parse as an assignment
-// statement, but rather, a directive whose value is `foo = 42`.
-#[test]
-fn ok_ipy_escape_command() {
-    use crate::Mode;
-
-    let src = r"!foo = 42";
-    let tokens = crate::lexer::lex(src, Mode::Ipython).collect();
-    let ast = crate::parse_tokens(tokens, src, Mode::Ipython);
-    insta::assert_debug_snapshot!(ast);
-}