Store current TokenKind on Parser

## Summary

This PR removes the `allow_trailing_comma` parameter from list parsing
because it can be inferred using the recovery context kind.

Cargo.lock

@@ -2353,9 +2353,11 @@ dependencies = [
 name = "ruff_python_parser"
 version = "0.0.0"
 dependencies = [
+ "annotate-snippets 0.9.2",
  "anyhow",
  "bitflags 2.4.2",
  "bstr",
+ "drop_bomb",
  "insta",
  "is-macro",
  "itertools 0.12.1",
@@ -2363,6 +2365,7 @@ dependencies = [
  "lalrpop-util",
  "memchr",
  "ruff_python_ast",
+ "ruff_source_file",
  "ruff_text_size",
  "rustc-hash",
  "static_assertions",

crates/ruff/tests/format.rs

@@ -523,7 +523,7 @@ from module import =
     ----- stdout -----
 
     ----- stderr -----
-    error: Failed to parse main.py:2:20: Unexpected token '='
+    error: Failed to parse main.py:2:20: Unexpected token =
     "###);
 
     Ok(())

crates/ruff/tests/integration_test.rs

@@ -731,11 +731,11 @@ fn stdin_parse_error() {
     success: false
     exit_code: 1
     ----- stdout -----
-    -:1:17: E999 SyntaxError: Unexpected token '='
+    -:1:17: E999 SyntaxError: Unexpected token =
     Found 1 error.
 
     ----- stderr -----
-    error: Failed to parse at 1:17: Unexpected token '='
+    error: Failed to parse at 1:17: Unexpected token =
     "###);
 }
 

crates/ruff/tests/snapshots/show_settings__display_default_settings.snap

@@ -52,6 +52,7 @@ file_resolver.exclude = [
 file_resolver.extend_exclude = [
 	"crates/ruff_linter/resources/",
 	"crates/ruff_python_formatter/resources/",
+	"crates/ruff_python_parser/resources/",
 ]
 file_resolver.force_exclude = false
 file_resolver.include = [

crates/ruff_benchmark/benches/formatter.rs

@@ -7,7 +7,7 @@ use ruff_benchmark::{TestCase, TestFile, TestFileDownloadError};
 use ruff_python_formatter::{format_module_ast, PreviewMode, PyFormatOptions};
 use ruff_python_index::CommentRangesBuilder;
 use ruff_python_parser::lexer::lex;
-use ruff_python_parser::{allocate_tokens_vec, parse_tokens, Mode};
+use ruff_python_parser::{allocate_tokens_vec, parse_tokens, set_new_parser, Mode};
 
 #[cfg(target_os = "windows")]
 #[global_allocator]
@@ -42,6 +42,8 @@ fn create_test_cases() -> Result<Vec<TestCase>, TestFileDownloadError> {
 }
 
 fn benchmark_formatter(criterion: &mut Criterion) {
+    set_new_parser(true);
+
     let mut group = criterion.benchmark_group("formatter");
     let test_cases = create_test_cases().unwrap();
 

crates/ruff_benchmark/benches/lexer.rs

@@ -2,7 +2,7 @@ use ruff_benchmark::criterion::{
     criterion_group, criterion_main, measurement::WallTime, BenchmarkId, Criterion, Throughput,
 };
 use ruff_benchmark::{TestCase, TestFile, TestFileDownloadError};
-use ruff_python_parser::{lexer, Mode};
+use ruff_python_parser::{lexer, set_new_parser, Mode};
 
 #[cfg(target_os = "windows")]
 #[global_allocator]
@@ -37,6 +37,8 @@ fn create_test_cases() -> Result<Vec<TestCase>, TestFileDownloadError> {
 }
 
 fn benchmark_lexer(criterion: &mut Criterion<WallTime>) {
+    set_new_parser(true);
+
     let test_cases = create_test_cases().unwrap();
     let mut group = criterion.benchmark_group("lexer");
 

crates/ruff_benchmark/benches/linter.rs

@@ -10,7 +10,7 @@ use ruff_linter::settings::{flags, LinterSettings};
 use ruff_linter::source_kind::SourceKind;
 use ruff_linter::{registry::Rule, RuleSelector};
 use ruff_python_ast::PySourceType;
-use ruff_python_parser::{lexer, parse_program_tokens, Mode};
+use ruff_python_parser::{lexer, parse_program_tokens, set_new_parser, Mode};
 
 #[cfg(target_os = "windows")]
 #[global_allocator]
@@ -45,6 +45,8 @@ fn create_test_cases() -> Result<Vec<TestCase>, TestFileDownloadError> {
 }
 
 fn benchmark_linter(mut group: BenchmarkGroup, settings: &LinterSettings) {
+    set_new_parser(true);
+
     let test_cases = create_test_cases().unwrap();
 
     for case in test_cases {

crates/ruff_benchmark/benches/parser.rs

@@ -4,7 +4,7 @@ use ruff_benchmark::criterion::{
 use ruff_benchmark::{TestCase, TestFile, TestFileDownloadError};
 use ruff_python_ast::statement_visitor::{walk_stmt, StatementVisitor};
 use ruff_python_ast::Stmt;
-use ruff_python_parser::parse_suite;
+use ruff_python_parser::{parse_suite, set_new_parser};
 
 #[cfg(target_os = "windows")]
 #[global_allocator]
@@ -50,6 +50,8 @@ impl<'a> StatementVisitor<'a> for CountVisitor {
 }
 
 fn benchmark_parser(criterion: &mut Criterion<WallTime>) {
+    set_new_parser(true);
+
     let test_cases = create_test_cases().unwrap();
     let mut group = criterion.benchmark_group("parser");
 

crates/ruff_linter/resources/test/fixtures/pycodestyle/E302_first_line_docstring.py

@@ -1,4 +0,0 @@
-"""Test where the error is after the module's docstring."""
-
-def fn():
-    pass

crates/ruff_linter/resources/test/fixtures/pycodestyle/E302_first_line_expression.py

@@ -1,4 +0,0 @@
-"Test where the first line is a comment, " + "and the rule violation follows it."
-
-def fn():
-    pass

crates/ruff_linter/resources/test/fixtures/pycodestyle/E302_first_line_function.py

@@ -1,5 +0,0 @@
-def fn1():
-    pass
-
-def fn2():
-    pass

crates/ruff_linter/resources/test/fixtures/pycodestyle/E302_first_line_statement.py

@@ -1,4 +0,0 @@
-print("Test where the first line is a statement, and the rule violation follows it.")
-
-def fn():
-    pass

crates/ruff_linter/resources/test/fixtures/pycodestyle/E303_first_line_comment.py

@@ -1,6 +0,0 @@
-# Test where the first line is a comment, and the rule violation follows it.
-
-
-
-def fn():
-    pass

crates/ruff_linter/resources/test/fixtures/pycodestyle/E303_first_line_docstring.py

@@ -1,6 +0,0 @@
-"""Test where the error is after the module's docstring."""
-
-
-
-def fn():
-    pass

crates/ruff_linter/resources/test/fixtures/pycodestyle/E303_first_line_expression.py

@@ -1,6 +0,0 @@
-"Test where the first line is a comment, " + "and the rule violation follows it."
-
-
-
-def fn():
-    pass

crates/ruff_linter/resources/test/fixtures/pycodestyle/E303_first_line_statement.py

@@ -1,6 +0,0 @@
-print("Test where the first line is a statement, and the rule violation follows it.")
-
-
-
-def fn():
-    pass

crates/ruff_linter/src/checkers/ast/analyze/expression.rs

@@ -254,7 +254,7 @@ pub(crate) fn expression(expr: &Expr, checker: &mut Checker) {
                         }
                     }
                 }
-                ExprContext::Del => {}
+                _ => {}
             }
             if checker.enabled(Rule::SixPY3) {
                 flake8_2020::rules::name_or_attribute(checker, expr);

crates/ruff_linter/src/checkers/ast/mod.rs

@@ -986,6 +986,7 @@ impl<'a> Visitor<'a> for Checker<'a> {
                 ExprContext::Load => self.handle_node_load(expr),
                 ExprContext::Store => self.handle_node_store(id, expr),
                 ExprContext::Del => self.handle_node_delete(expr),
+                ExprContext::Invalid => {}
             },
             _ => {}
         }

crates/ruff_linter/src/logging.rs

@@ -194,7 +194,7 @@ impl DisplayParseError {
         // Translate the byte offset to a location in the originating source.
         let location =
             if let Some(jupyter_index) = source_kind.as_ipy_notebook().map(Notebook::index) {
-                let source_location = source_code.source_location(error.offset);
+                let source_location = source_code.source_location(error.location.start());
 
                 ErrorLocation::Cell(
                     jupyter_index
@@ -208,7 +208,7 @@ impl DisplayParseError {
                     },
                 )
             } else {
-                ErrorLocation::File(source_code.source_location(error.offset))
+                ErrorLocation::File(source_code.source_location(error.location.start()))
             };
 
         Self {
@@ -275,27 +275,7 @@ impl<'a> DisplayParseErrorType<'a> {
 
 impl Display for DisplayParseErrorType<'_> {
     fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
-        match self.0 {
-            ParseErrorType::Eof => write!(f, "Expected token but reached end of file."),
-            ParseErrorType::ExtraToken(ref tok) => write!(
-                f,
-                "Got extraneous token: {tok}",
-                tok = TruncateAtNewline(&tok)
-            ),
-            ParseErrorType::InvalidToken => write!(f, "Got invalid token"),
-            ParseErrorType::UnrecognizedToken(ref tok, ref expected) => {
-                if let Some(expected) = expected.as_ref() {
-                    write!(
-                        f,
-                        "Expected '{expected}', but got {tok}",
-                        tok = TruncateAtNewline(&tok)
-                    )
-                } else {
-                    write!(f, "Unexpected token {tok}", tok = TruncateAtNewline(&tok))
-                }
-            }
-            ParseErrorType::Lexical(ref error) => write!(f, "{error}"),
-        }
+        write!(f, "{}", TruncateAtNewline(&self.0))
     }
 }
 

crates/ruff_linter/src/rules/flake8_bugbear/rules/function_uses_loop_variable.rs

@@ -67,7 +67,7 @@ impl<'a> Visitor<'a> for LoadedNamesVisitor<'a> {
             Expr::Name(name) => match &name.ctx {
                 ExprContext::Load => self.loaded.push(name),
                 ExprContext::Store => self.stored.push(name),
-                ExprContext::Del => {}
+                _ => {}
             },
             _ => visitor::walk_expr(self, expr),
         }

crates/ruff_linter/src/rules/isort/snapshots/ruff_linter__rules__isort__tests__bom_unsorted.py.snap

@@ -14,5 +14,3 @@ bom_unsorted.py:1:1: I001 [*] Import block is un-sorted or un-formatted
 2   |-import bar
   1 |+import bar
   2 |+import foo
-
-

crates/ruff_linter/src/rules/pycodestyle/mod.rs

@@ -171,24 +171,6 @@ mod tests {
         Ok(())
     }
 
-    #[test_case(Rule::BlankLinesTopLevel, Path::new("E302_first_line_docstring.py"))]
-    #[test_case(Rule::BlankLinesTopLevel, Path::new("E302_first_line_expression.py"))]
-    #[test_case(Rule::BlankLinesTopLevel, Path::new("E302_first_line_function.py"))]
-    #[test_case(Rule::BlankLinesTopLevel, Path::new("E302_first_line_statement.py"))]
-    #[test_case(Rule::TooManyBlankLines, Path::new("E303_first_line_comment.py"))]
-    #[test_case(Rule::TooManyBlankLines, Path::new("E303_first_line_docstring.py"))]
-    #[test_case(Rule::TooManyBlankLines, Path::new("E303_first_line_expression.py"))]
-    #[test_case(Rule::TooManyBlankLines, Path::new("E303_first_line_statement.py"))]
-    fn blank_lines_first_line(rule_code: Rule, path: &Path) -> Result<()> {
-        let snapshot = format!("{}_{}", rule_code.noqa_code(), path.to_string_lossy());
-        let diagnostics = test_path(
-            Path::new("pycodestyle").join(path).as_path(),
-            &settings::LinterSettings::for_rule(rule_code),
-        )?;
-        assert_messages!(snapshot, diagnostics);
-        Ok(())
-    }
-
     #[test_case(Rule::BlankLineBetweenMethods, Path::new("E30.py"))]
     #[test_case(Rule::BlankLinesTopLevel, Path::new("E30.py"))]
     #[test_case(Rule::TooManyBlankLines, Path::new("E30.py"))]

crates/ruff_linter/src/rules/pycodestyle/rules/blank_lines.rs

@@ -696,7 +696,9 @@ impl<'a> BlankLinesChecker<'a> {
             state.class_status.update(&logical_line);
             state.fn_status.update(&logical_line);
 
-            self.check_line(&logical_line, &state, prev_indent_length, diagnostics);
+            if state.is_not_first_logical_line {
+                self.check_line(&logical_line, &state, prev_indent_length, diagnostics);
+            }
 
             match logical_line.kind {
                 LogicalLineKind::Class => {
@@ -816,8 +818,6 @@ impl<'a> BlankLinesChecker<'a> {
             && line.kind.is_class_function_or_decorator()
             // Blank lines in stub files are used to group definitions. Don't enforce blank lines.
             && !self.source_type.is_stub()
-            // Do not expect blank lines before the first logical line.
-            && state.is_not_first_logical_line
         {
             // E302
             let mut diagnostic = Diagnostic::new(

crates/ruff_linter/src/rules/pycodestyle/rules/errors.rs

@@ -81,7 +81,7 @@ pub(crate) fn syntax_error(
     parse_error: &ParseError,
     locator: &Locator,
 ) {
-    let rest = locator.after(parse_error.offset);
+    let rest = locator.after(parse_error.location.start());
 
     // Try to create a non-empty range so that the diagnostic can print a caret at the
     // right position. This requires that we retrieve the next character, if any, and take its length
@@ -95,6 +95,6 @@ pub(crate) fn syntax_error(
         SyntaxError {
             message: format!("{}", DisplayParseErrorType::new(&parse_error.error)),
         },
-        TextRange::at(parse_error.offset, len),
+        TextRange::at(parse_error.location.start(), len),
     ));
 }

crates/ruff_linter/src/rules/pycodestyle/snapshots/ruff_linter__rules__pycodestyle__tests__E302_E302_first_line_docstring.py.snap

@@ -1,19 +0,0 @@
----
-source: crates/ruff_linter/src/rules/pycodestyle/mod.rs
----
-E302_first_line_docstring.py:3:1: E302 [*] Expected 2 blank lines, found 1
-  |
-1 | """Test where the error is after the module's docstring."""
-2 | 
-3 | def fn():
-  | ^^^ E302
-4 |     pass
-  |
-  = help: Add missing blank line(s)
-
-ℹ Safe fix
-1 1 | """Test where the error is after the module's docstring."""
-2 2 | 
-  3 |+
-3 4 | def fn():
-4 5 |     pass

crates/ruff_linter/src/rules/pycodestyle/snapshots/ruff_linter__rules__pycodestyle__tests__E302_E302_first_line_expression.py.snap

@@ -1,19 +0,0 @@
----
-source: crates/ruff_linter/src/rules/pycodestyle/mod.rs
----
-E302_first_line_expression.py:3:1: E302 [*] Expected 2 blank lines, found 1
-  |
-1 | "Test where the first line is a comment, " + "and the rule violation follows it."
-2 | 
-3 | def fn():
-  | ^^^ E302
-4 |     pass
-  |
-  = help: Add missing blank line(s)
-
-ℹ Safe fix
-1 1 | "Test where the first line is a comment, " + "and the rule violation follows it."
-2 2 | 
-  3 |+
-3 4 | def fn():
-4 5 |     pass

crates/ruff_linter/src/rules/pycodestyle/snapshots/ruff_linter__rules__pycodestyle__tests__E302_E302_first_line_function.py.snap

@@ -1,20 +0,0 @@
----
-source: crates/ruff_linter/src/rules/pycodestyle/mod.rs
----
-E302_first_line_function.py:4:1: E302 [*] Expected 2 blank lines, found 1
-  |
-2 |     pass
-3 | 
-4 | def fn2():
-  | ^^^ E302
-5 |     pass
-  |
-  = help: Add missing blank line(s)
-
-ℹ Safe fix
-1 1 | def fn1():
-2 2 |     pass
-3 3 | 
-  4 |+
-4 5 | def fn2():
-5 6 |     pass

crates/ruff_linter/src/rules/pycodestyle/snapshots/ruff_linter__rules__pycodestyle__tests__E302_E302_first_line_statement.py.snap

@@ -1,19 +0,0 @@
----
-source: crates/ruff_linter/src/rules/pycodestyle/mod.rs
----
-E302_first_line_statement.py:3:1: E302 [*] Expected 2 blank lines, found 1
-  |
-1 | print("Test where the first line is a statement, and the rule violation follows it.")
-2 | 
-3 | def fn():
-  | ^^^ E302
-4 |     pass
-  |
-  = help: Add missing blank line(s)
-
-ℹ Safe fix
-1 1 | print("Test where the first line is a statement, and the rule violation follows it.")
-2 2 | 
-  3 |+
-3 4 | def fn():
-4 5 |     pass

crates/ruff_linter/src/rules/pycodestyle/snapshots/ruff_linter__rules__pycodestyle__tests__E303_E303_first_line_comment.py.snap

@@ -1,18 +0,0 @@
----
-source: crates/ruff_linter/src/rules/pycodestyle/mod.rs
----
-E303_first_line_comment.py:5:1: E303 [*] Too many blank lines (3)
-  |
-5 | def fn():
-  | ^^^ E303
-6 |     pass
-  |
-  = help: Remove extraneous blank line(s)
-
-ℹ Safe fix
-1 1 | # Test where the first line is a comment, and the rule violation follows it.
-2 2 | 
-3 3 | 
-4   |-
-5 4 | def fn():
-6 5 |     pass

crates/ruff_linter/src/rules/pycodestyle/snapshots/ruff_linter__rules__pycodestyle__tests__E303_E303_first_line_docstring.py.snap

@@ -1,18 +0,0 @@
----
-source: crates/ruff_linter/src/rules/pycodestyle/mod.rs
----
-E303_first_line_docstring.py:5:1: E303 [*] Too many blank lines (3)
-  |
-5 | def fn():
-  | ^^^ E303
-6 |     pass
-  |
-  = help: Remove extraneous blank line(s)
-
-ℹ Safe fix
-1 1 | """Test where the error is after the module's docstring."""
-2 2 | 
-3 3 | 
-4   |-
-5 4 | def fn():
-6 5 |     pass

crates/ruff_linter/src/rules/pycodestyle/snapshots/ruff_linter__rules__pycodestyle__tests__E303_E303_first_line_expression.py.snap

@@ -1,18 +0,0 @@
----
-source: crates/ruff_linter/src/rules/pycodestyle/mod.rs
----
-E303_first_line_expression.py:5:1: E303 [*] Too many blank lines (3)
-  |
-5 | def fn():
-  | ^^^ E303
-6 |     pass
-  |
-  = help: Remove extraneous blank line(s)
-
-ℹ Safe fix
-1 1 | "Test where the first line is a comment, " + "and the rule violation follows it."
-2 2 | 
-3 3 | 
-4   |-
-5 4 | def fn():
-6 5 |     pass

crates/ruff_linter/src/rules/pycodestyle/snapshots/ruff_linter__rules__pycodestyle__tests__E303_E303_first_line_statement.py.snap

@@ -1,18 +0,0 @@
----
-source: crates/ruff_linter/src/rules/pycodestyle/mod.rs
----
-E303_first_line_statement.py:5:1: E303 [*] Too many blank lines (3)
-  |
-5 | def fn():
-  | ^^^ E303
-6 |     pass
-  |
-  = help: Remove extraneous blank line(s)
-
-ℹ Safe fix
-1 1 | print("Test where the first line is a statement, and the rule violation follows it.")
-2 2 | 
-3 3 | 
-4   |-
-5 4 | def fn():
-6 5 |     pass

crates/ruff_linter/src/rules/pycodestyle/snapshots/ruff_linter__rules__pycodestyle__tests__E999_E999.py.snap

@@ -8,5 +8,3 @@ E999.py:3:1: E999 SyntaxError: unindent does not match any outer indentation lev
   | ^ E999
 4 | 
   |
-
-

crates/ruff_linter/src/rules/pyupgrade/snapshots/ruff_linter__rules__pyupgrade__tests__UP027.py.snap

@@ -110,5 +110,3 @@ UP027.py:10:17: UP027 [*] Replace unpacked list comprehension with a generator e
 14 14 | 
 15 15 | # Should not change
 16 16 | foo = [fn(x) for x in items]
-
-

crates/ruff_linter/src/rules/ruff/rules/invalid_formatter_suppression_comment.rs

@@ -141,15 +141,15 @@ impl<'src, 'loc> UselessSuppressionComments<'src, 'loc> {
         if comment.kind == SuppressionKind::Off || comment.kind == SuppressionKind::On {
             if let Some(
                 AnyNodeRef::StmtClassDef(StmtClassDef {
-                                             name,
-                                             decorator_list,
-                                             ..
-                                         })
+                    name,
+                    decorator_list,
+                    ..
+                })
                 | AnyNodeRef::StmtFunctionDef(StmtFunctionDef {
-                                                  name,
-                                                  decorator_list,
-                                                  ..
-                                              }),
+                    name,
+                    decorator_list,
+                    ..
+                }),
             ) = comment.enclosing
             {
                 if comment.line_position.is_own_line() && comment.range.start() < name.start() {
@@ -196,7 +196,7 @@ impl<'src, 'loc> UselessSuppressionComments<'src, 'loc> {
         self.captured.sort_by_key(|(t, _)| t.start());
     }
 
-    fn ignored_comments(&self) -> impl Iterator<Item=(TextRange, IgnoredReason)> + '_ {
+    fn ignored_comments(&self) -> impl Iterator<Item = (TextRange, IgnoredReason)> + '_ {
         self.captured.iter().map(|(r, i)| (*r, *i))
     }
 }
@@ -276,8 +276,7 @@ const fn is_valid_enclosing_node(node: AnyNodeRef) -> bool {
         | AnyNodeRef::StmtIpyEscapeCommand(_)
         | AnyNodeRef::ExceptHandlerExceptHandler(_)
         | AnyNodeRef::MatchCase(_)
-        | AnyNodeRef::ElifElseClause(_)
-        | AnyNodeRef::StmtCrement(_) => true,
+        | AnyNodeRef::ElifElseClause(_) => true,
 
         AnyNodeRef::ExprBoolOp(_)
         | AnyNodeRef::ExprNamed(_)

crates/ruff_python_ast/src/comparable.rs

@@ -234,6 +234,7 @@ pub enum ComparablePattern<'a> {
     MatchStar(PatternMatchStar<'a>),
     MatchAs(PatternMatchAs<'a>),
     MatchOr(PatternMatchOr<'a>),
+    Invalid,
 }
 
 impl<'a> From<&'a ast::Pattern> for ComparablePattern<'a> {
@@ -864,6 +865,7 @@ pub enum ComparableExpr<'a> {
     Tuple(ExprTuple<'a>),
     Slice(ExprSlice<'a>),
     IpyEscapeCommand(ExprIpyEscapeCommand<'a>),
+    Invalid,
 }
 
 impl<'a> From<&'a Box<ast::Expr>> for Box<ComparableExpr<'a>> {
@@ -1543,9 +1545,6 @@ impl<'a> From<&'a ast::Stmt> for ComparableStmt<'a> {
             ast::Stmt::Pass(_) => Self::Pass,
             ast::Stmt::Break(_) => Self::Break,
             ast::Stmt::Continue(_) => Self::Continue,
-            ast::Stmt::Crement(_) => {
-                todo!()
-            }
         }
     }
 }

crates/ruff_python_ast/src/helpers.rs

@@ -432,7 +432,6 @@ pub fn any_over_stmt(stmt: &Stmt, func: &dyn Fn(&Expr) -> bool) -> bool {
         Stmt::AugAssign(ast::StmtAugAssign { target, value, .. }) => {
             any_over_expr(target, func) || any_over_expr(value, func)
         }
-        Stmt::Crement(ast::StmtCrement { target, .. }) => any_over_expr(target, func),
         Stmt::AnnAssign(ast::StmtAnnAssign {
             target,
             annotation,
@@ -1603,7 +1602,7 @@ mod tests {
     fn any_over_stmt_type_alias() {
         let seen = RefCell::new(Vec::new());
         let name = Expr::Name(ExprName {
-            id: "x".to_string(),
+            id: "x".into(),
             range: TextRange::default(),
             ctx: ExprContext::Load,
         });

crates/ruff_python_ast/src/node.rs

@@ -36,7 +36,6 @@ pub enum AnyNode {
     StmtTypeAlias(ast::StmtTypeAlias),
     StmtAssign(ast::StmtAssign),
     StmtAugAssign(ast::StmtAugAssign),
-    StmtCrement(ast::StmtCrement),
     StmtAnnAssign(ast::StmtAnnAssign),
     StmtFor(ast::StmtFor),
     StmtWhile(ast::StmtWhile),
@@ -131,7 +130,6 @@ impl AnyNode {
             AnyNode::StmtTypeAlias(node) => Some(Stmt::TypeAlias(node)),
             AnyNode::StmtAssign(node) => Some(Stmt::Assign(node)),
             AnyNode::StmtAugAssign(node) => Some(Stmt::AugAssign(node)),
-            AnyNode::StmtCrement(node) => Some(Stmt::Crement(node)),
             AnyNode::StmtAnnAssign(node) => Some(Stmt::AnnAssign(node)),
             AnyNode::StmtFor(node) => Some(Stmt::For(node)),
             AnyNode::StmtWhile(node) => Some(Stmt::While(node)),
@@ -264,7 +262,6 @@ impl AnyNode {
             | AnyNode::StmtTypeAlias(_)
             | AnyNode::StmtAssign(_)
             | AnyNode::StmtAugAssign(_)
-            | AnyNode::StmtCrement(_)
             | AnyNode::StmtAnnAssign(_)
             | AnyNode::StmtFor(_)
             | AnyNode::StmtWhile(_)
@@ -330,7 +327,6 @@ impl AnyNode {
             | AnyNode::StmtTypeAlias(_)
             | AnyNode::StmtAssign(_)
             | AnyNode::StmtAugAssign(_)
-            | AnyNode::StmtCrement(_)
             | AnyNode::StmtAnnAssign(_)
             | AnyNode::StmtFor(_)
             | AnyNode::StmtWhile(_)
@@ -436,7 +432,6 @@ impl AnyNode {
             | AnyNode::StmtTypeAlias(_)
             | AnyNode::StmtAssign(_)
             | AnyNode::StmtAugAssign(_)
-            | AnyNode::StmtCrement(_)
             | AnyNode::StmtAnnAssign(_)
             | AnyNode::StmtFor(_)
             | AnyNode::StmtWhile(_)
@@ -527,7 +522,6 @@ impl AnyNode {
             | AnyNode::StmtTypeAlias(_)
             | AnyNode::StmtAssign(_)
             | AnyNode::StmtAugAssign(_)
-            | AnyNode::StmtCrement(_)
             | AnyNode::StmtAnnAssign(_)
             | AnyNode::StmtFor(_)
             | AnyNode::StmtWhile(_)
@@ -643,7 +637,6 @@ impl AnyNode {
             Self::StmtTypeAlias(node) => AnyNodeRef::StmtTypeAlias(node),
             Self::StmtAssign(node) => AnyNodeRef::StmtAssign(node),
             Self::StmtAugAssign(node) => AnyNodeRef::StmtAugAssign(node),
-            Self::StmtCrement(node) => AnyNodeRef::StmtCrement(node),
             Self::StmtAnnAssign(node) => AnyNodeRef::StmtAnnAssign(node),
             Self::StmtFor(node) => AnyNodeRef::StmtFor(node),
             Self::StmtWhile(node) => AnyNodeRef::StmtWhile(node),
@@ -1132,48 +1125,6 @@ impl AstNode for ast::StmtAugAssign {
         visitor.visit_expr(value);
     }
 }
-impl AstNode for ast::StmtCrement {
-    fn cast(kind: AnyNode) -> Option<Self>
-    where
-        Self: Sized,
-    {
-        if let AnyNode::StmtCrement(node) = kind {
-            Some(node)
-        } else {
-            None
-        }
-    }
-
-    fn cast_ref(kind: AnyNodeRef) -> Option<&Self> {
-        if let AnyNodeRef::StmtCrement(node) = kind {
-            Some(node)
-        } else {
-            None
-        }
-    }
-
-    fn as_any_node_ref(&self) -> AnyNodeRef {
-        AnyNodeRef::from(self)
-    }
-
-    fn into_any_node(self) -> AnyNode {
-        AnyNode::from(self)
-    }
-
-    fn visit_preorder<'a, V>(&'a self, visitor: &mut V)
-    where
-        V: PreorderVisitor<'a> + ?Sized,
-    {
-        let ast::StmtCrement {
-            target,
-            op: _,
-            range: _,
-        } = self;
-
-        visitor.visit_expr(target);
-        // TODO(konstin): visitor.visit_operator(op);
-    }
-}
 impl AstNode for ast::StmtAnnAssign {
     fn cast(kind: AnyNode) -> Option<Self>
     where
@@ -4587,7 +4538,6 @@ impl From<Stmt> for AnyNode {
             Stmt::Break(node) => AnyNode::StmtBreak(node),
             Stmt::Continue(node) => AnyNode::StmtContinue(node),
             Stmt::IpyEscapeCommand(node) => AnyNode::StmtIpyEscapeCommand(node),
-            Stmt::Crement(node) => AnyNode::StmtCrement(node),
         }
     }
 }
@@ -4726,12 +4676,6 @@ impl From<ast::StmtAugAssign> for AnyNode {
     }
 }
 
-impl From<ast::StmtCrement> for AnyNode {
-    fn from(node: ast::StmtCrement) -> Self {
-        AnyNode::StmtCrement(node)
-    }
-}
-
 impl From<ast::StmtAnnAssign> for AnyNode {
     fn from(node: ast::StmtAnnAssign) -> Self {
         AnyNode::StmtAnnAssign(node)
@@ -5307,7 +5251,6 @@ impl Ranged for AnyNode {
             AnyNode::StringLiteral(node) => node.range(),
             AnyNode::BytesLiteral(node) => node.range(),
             AnyNode::ElifElseClause(node) => node.range(),
-            AnyNode::StmtCrement(node) => node.range(),
         }
     }
 }
@@ -5323,7 +5266,6 @@ pub enum AnyNodeRef<'a> {
     StmtTypeAlias(&'a ast::StmtTypeAlias),
     StmtAssign(&'a ast::StmtAssign),
     StmtAugAssign(&'a ast::StmtAugAssign),
-    StmtCrement(&'a ast::StmtCrement),
     StmtAnnAssign(&'a ast::StmtAnnAssign),
     StmtFor(&'a ast::StmtFor),
     StmtWhile(&'a ast::StmtWhile),
@@ -5502,7 +5444,6 @@ impl<'a> AnyNodeRef<'a> {
             AnyNodeRef::StringLiteral(node) => NonNull::from(*node).cast(),
             AnyNodeRef::BytesLiteral(node) => NonNull::from(*node).cast(),
             AnyNodeRef::ElifElseClause(node) => NonNull::from(*node).cast(),
-            AnyNodeRef::StmtCrement(node) => NonNull::from(*node).cast(),
         }
     }
 
@@ -5605,7 +5546,6 @@ impl<'a> AnyNodeRef<'a> {
             AnyNodeRef::StringLiteral(_) => NodeKind::StringLiteral,
             AnyNodeRef::BytesLiteral(_) => NodeKind::BytesLiteral,
             AnyNodeRef::ElifElseClause(_) => NodeKind::ElifElseClause,
-            AnyNodeRef::StmtCrement(_) => NodeKind::StmtCrement,
         }
     }
 
@@ -5618,7 +5558,6 @@ impl<'a> AnyNodeRef<'a> {
             | AnyNodeRef::StmtTypeAlias(_)
             | AnyNodeRef::StmtAssign(_)
             | AnyNodeRef::StmtAugAssign(_)
-            | AnyNodeRef::StmtCrement(_)
             | AnyNodeRef::StmtAnnAssign(_)
             | AnyNodeRef::StmtFor(_)
             | AnyNodeRef::StmtWhile(_)
@@ -5751,7 +5690,6 @@ impl<'a> AnyNodeRef<'a> {
             | AnyNodeRef::StmtTypeAlias(_)
             | AnyNodeRef::StmtAssign(_)
             | AnyNodeRef::StmtAugAssign(_)
-            | AnyNodeRef::StmtCrement(_)
             | AnyNodeRef::StmtAnnAssign(_)
             | AnyNodeRef::StmtFor(_)
             | AnyNodeRef::StmtWhile(_)
@@ -5816,7 +5754,6 @@ impl<'a> AnyNodeRef<'a> {
             | AnyNodeRef::StmtTypeAlias(_)
             | AnyNodeRef::StmtAssign(_)
             | AnyNodeRef::StmtAugAssign(_)
-            | AnyNodeRef::StmtCrement(_)
             | AnyNodeRef::StmtAnnAssign(_)
             | AnyNodeRef::StmtFor(_)
             | AnyNodeRef::StmtWhile(_)
@@ -5922,7 +5859,6 @@ impl<'a> AnyNodeRef<'a> {
             | AnyNodeRef::StmtTypeAlias(_)
             | AnyNodeRef::StmtAssign(_)
             | AnyNodeRef::StmtAugAssign(_)
-            | AnyNodeRef::StmtCrement(_)
             | AnyNodeRef::StmtAnnAssign(_)
             | AnyNodeRef::StmtFor(_)
             | AnyNodeRef::StmtWhile(_)
@@ -6013,7 +5949,6 @@ impl<'a> AnyNodeRef<'a> {
             | AnyNodeRef::StmtTypeAlias(_)
             | AnyNodeRef::StmtAssign(_)
             | AnyNodeRef::StmtAugAssign(_)
-            | AnyNodeRef::StmtCrement(_)
             | AnyNodeRef::StmtAnnAssign(_)
             | AnyNodeRef::StmtFor(_)
             | AnyNodeRef::StmtWhile(_)
@@ -6123,7 +6058,6 @@ impl<'a> AnyNodeRef<'a> {
             AnyNodeRef::StmtTypeAlias(node) => node.visit_preorder(visitor),
             AnyNodeRef::StmtAssign(node) => node.visit_preorder(visitor),
             AnyNodeRef::StmtAugAssign(node) => node.visit_preorder(visitor),
-            AnyNodeRef::StmtCrement(node) => node.visit_preorder(visitor),
             AnyNodeRef::StmtAnnAssign(node) => node.visit_preorder(visitor),
             AnyNodeRef::StmtFor(node) => node.visit_preorder(visitor),
             AnyNodeRef::StmtWhile(node) => node.visit_preorder(visitor),
@@ -6438,12 +6372,6 @@ impl<'a> From<&'a ast::StmtAugAssign> for AnyNodeRef<'a> {
     }
 }
 
-impl<'a> From<&'a ast::StmtCrement> for AnyNodeRef<'a> {
-    fn from(node: &'a ast::StmtCrement) -> Self {
-        AnyNodeRef::StmtCrement(node)
-    }
-}
-
 impl<'a> From<&'a ast::StmtAnnAssign> for AnyNodeRef<'a> {
     fn from(node: &'a ast::StmtAnnAssign) -> Self {
         AnyNodeRef::StmtAnnAssign(node)
@@ -6909,7 +6837,6 @@ impl<'a> From<&'a Stmt> for AnyNodeRef<'a> {
             Stmt::Break(node) => AnyNodeRef::StmtBreak(node),
             Stmt::Continue(node) => AnyNodeRef::StmtContinue(node),
             Stmt::IpyEscapeCommand(node) => AnyNodeRef::StmtIpyEscapeCommand(node),
-            Stmt::Crement(node) => AnyNodeRef::StmtCrement(node),
         }
     }
 }
@@ -7146,7 +7073,6 @@ impl Ranged for AnyNodeRef<'_> {
             AnyNodeRef::FString(node) => node.range(),
             AnyNodeRef::StringLiteral(node) => node.range(),
             AnyNodeRef::BytesLiteral(node) => node.range(),
-            AnyNodeRef::StmtCrement(node) => node.range(),
         }
     }
 }
@@ -7247,5 +7173,4 @@ pub enum NodeKind {
     FString,
     StringLiteral,
     BytesLiteral,
-    StmtCrement,
 }

crates/ruff_python_ast/src/nodes.rs

@@ -1,6 +1,7 @@
 #![allow(clippy::derive_partial_eq_without_eq)]
 
 use std::cell::OnceCell;
+
 use std::fmt;
 use std::fmt::Debug;
 use std::ops::Deref;
@@ -101,8 +102,6 @@ pub enum Stmt {
     // Jupyter notebook specific
     #[is(name = "ipy_escape_command_stmt")]
     IpyEscapeCommand(StmtIpyEscapeCommand),
-    #[is(name = "crement_stmt")]
-    Crement(StmtCrement),
 }
 
 /// An AST node used to represent a IPython escape command at the statement level.
@@ -299,26 +298,6 @@ impl From<StmtAugAssign> for Stmt {
     }
 }
 
-#[derive(Clone, Debug, PartialEq)]
-pub enum CrementKind {
-    Increment,
-    Decrement,
-}
-
-/// `++` or `--`
-#[derive(Clone, Debug, PartialEq)]
-pub struct StmtCrement {
-    pub range: TextRange,
-    pub target: Box<Expr>,
-    pub op: CrementKind,
-}
-
-impl From<StmtCrement> for Stmt {
-    fn from(payload: StmtCrement) -> Self {
-        Stmt::Crement(payload)
-    }
-}
-
 /// See also [AnnAssign](https://docs.python.org/3/library/ast.html#ast.AnnAssign)
 #[derive(Clone, Debug, PartialEq)]
 pub struct StmtAnnAssign {
@@ -969,12 +948,19 @@ impl Ranged for FStringExpressionElement {
     }
 }
 
+/// An `FStringLiteralElement` with an empty `value` is an invalid f-string element.
 #[derive(Clone, Debug, PartialEq)]
 pub struct FStringLiteralElement {
     pub range: TextRange,
     pub value: Box<str>,
 }
 
+impl FStringLiteralElement {
+    pub fn is_valid(&self) -> bool {
+        !self.value.is_empty()
+    }
+}
+
 impl Ranged for FStringLiteralElement {
     fn range(&self) -> TextRange {
         self.range
@@ -1523,6 +1509,9 @@ bitflags! {
         /// The string has an `r` or `R` prefix, meaning it is a raw string.
         /// It is invalid to set this flag if `U_PREFIX` is also set.
         const R_PREFIX = 1 << 3;
+
+        /// The string was deemed invalid by the parser.
+        const INVALID = 1 << 4;
     }
 }
 
@@ -1554,6 +1543,12 @@ impl StringLiteralFlags {
         self
     }
 
+    #[must_use]
+    pub fn with_invalid(mut self) -> Self {
+        self.0 |= StringLiteralFlagsInner::INVALID;
+        self
+    }
+
     pub const fn prefix(self) -> &'static str {
         if self.0.contains(StringLiteralFlagsInner::U_PREFIX) {
             debug_assert!(!self.0.contains(StringLiteralFlagsInner::R_PREFIX));
@@ -1648,6 +1643,15 @@ impl StringLiteral {
     pub fn as_str(&self) -> &str {
         self
     }
+
+    /// Creates an invalid string literal with the given range.
+    pub fn invalid(range: TextRange) -> Self {
+        Self {
+            range,
+            value: "".into(),
+            flags: StringLiteralFlags::default().with_invalid(),
+        }
+    }
 }
 
 impl From<StringLiteral> for Expr {
@@ -1856,6 +1860,9 @@ bitflags! {
 
         /// The bytestring has an `r` or `R` prefix, meaning it is a raw bytestring.
         const R_PREFIX = 1 << 3;
+
+        /// The bytestring was deemed invalid by the parser.
+        const INVALID = 1 << 4;
     }
 }
 
@@ -1883,6 +1890,12 @@ impl BytesLiteralFlags {
         self
     }
 
+    #[must_use]
+    pub fn with_invalid(mut self) -> Self {
+        self.0 |= BytesLiteralFlagsInner::INVALID;
+        self
+    }
+
     /// Does the bytestring have an `r` or `R` prefix?
     pub const fn is_raw(self) -> bool {
         self.0.contains(BytesLiteralFlagsInner::R_PREFIX)
@@ -1942,6 +1955,15 @@ impl BytesLiteral {
     pub fn as_slice(&self) -> &[u8] {
         self
     }
+
+    /// Creates a new invalid bytes literal with the given range.
+    pub fn invalid(range: TextRange) -> Self {
+        Self {
+            range,
+            value: Box::new([]),
+            flags: BytesLiteralFlags::default().with_invalid(),
+        }
+    }
 }
 
 impl From<BytesLiteral> for Expr {
@@ -2141,6 +2163,7 @@ pub enum ExprContext {
     Load,
     Store,
     Del,
+    Invalid,
 }
 impl ExprContext {
     #[inline]
@@ -3566,10 +3589,17 @@ impl IpyEscapeKind {
     }
 }
 
+/// An `Identifier` with an empty `id` is invalid.
+///
+/// For example, in the following code `id` will be empty.
+/// ```python
+/// def 1():
+///     ...
+/// ```
 #[derive(Clone, Debug, PartialEq, Eq, Hash)]
 pub struct Identifier {
-    id: String,
-    range: TextRange,
+    pub id: String,
+    pub range: TextRange,
 }
 
 impl Identifier {
@@ -3580,6 +3610,10 @@ impl Identifier {
             range,
         }
     }
+
+    pub fn is_valid(&self) -> bool {
+        !self.id.is_empty()
+    }
 }
 
 impl Identifier {
@@ -3715,11 +3749,6 @@ impl Ranged for crate::nodes::StmtAugAssign {
         self.range
     }
 }
-impl Ranged for crate::nodes::StmtCrement {
-    fn range(&self) -> TextRange {
-        self.range
-    }
-}
 impl Ranged for crate::nodes::StmtAnnAssign {
     fn range(&self) -> TextRange {
         self.range
@@ -3843,7 +3872,6 @@ impl Ranged for crate::Stmt {
             Self::Break(node) => node.range(),
             Self::Continue(node) => node.range(),
             Stmt::IpyEscapeCommand(node) => node.range(),
-            Stmt::Crement(node) => node.range(),
         }
     }
 }

crates/ruff_python_ast/src/visitor.rs

@@ -203,14 +203,6 @@ pub fn walk_stmt<'a, V: Visitor<'a> + ?Sized>(visitor: &mut V, stmt: &'a Stmt) {
             visitor.visit_operator(op);
             visitor.visit_expr(target);
         }
-        Stmt::Crement(ast::StmtCrement {
-            target,
-            op: _,
-            range: _,
-        }) => {
-            // TODO(konstin): visitor.visit_operator(op);
-            visitor.visit_expr(target);
-        }
         Stmt::AnnAssign(ast::StmtAnnAssign {
             target,
             annotation,

crates/ruff_python_ast/src/visitor/preorder.rs

@@ -209,7 +209,6 @@ where
             Stmt::TypeAlias(stmt) => stmt.visit_preorder(visitor),
             Stmt::Assign(stmt) => stmt.visit_preorder(visitor),
             Stmt::AugAssign(stmt) => stmt.visit_preorder(visitor),
-            Stmt::Crement(stmt) => stmt.visit_preorder(visitor),
             Stmt::AnnAssign(stmt) => stmt.visit_preorder(visitor),
             Stmt::For(stmt) => stmt.visit_preorder(visitor),
             Stmt::While(stmt) => stmt.visit_preorder(visitor),

crates/ruff_python_ast/src/visitor/transformer.rs

@@ -306,7 +306,6 @@ pub fn walk_stmt<V: Transformer + ?Sized>(visitor: &V, stmt: &mut Stmt) {
         Stmt::Nonlocal(_) => {}
         Stmt::Expr(ast::StmtExpr { value, range: _ }) => visitor.visit_expr(value),
         Stmt::Pass(_) | Stmt::Break(_) | Stmt::Continue(_) | Stmt::IpyEscapeCommand(_) => {}
-        Stmt::Crement(_) => {}
     }
 }
 

crates/ruff_python_codegen/src/generator.rs

@@ -4,10 +4,10 @@ use std::ops::Deref;
 
 use ruff_python_ast::str::Quote;
 use ruff_python_ast::{
-    self as ast, Alias, ArgOrKeyword, BoolOp, CmpOp, Comprehension, ConversionFlag, CrementKind,
-    DebugText, ExceptHandler, Expr, Identifier, MatchCase, Operator, Parameter, Parameters,
-    Pattern, Singleton, Stmt, Suite, TypeParam, TypeParamParamSpec, TypeParamTypeVar,
-    TypeParamTypeVarTuple, WithItem,
+    self as ast, Alias, ArgOrKeyword, BoolOp, CmpOp, Comprehension, ConversionFlag, DebugText,
+    ExceptHandler, Expr, Identifier, MatchCase, Operator, Parameter, Parameters, Pattern,
+    Singleton, Stmt, Suite, TypeParam, TypeParamParamSpec, TypeParamTypeVar, TypeParamTypeVarTuple,
+    WithItem,
 };
 use ruff_python_ast::{ParameterWithDefault, TypeParams};
 use ruff_python_literal::escape::{AsciiEscape, Escape, UnicodeEscape};
@@ -345,20 +345,6 @@ impl<'a> Generator<'a> {
                     self.unparse_expr(value, precedence::AUG_ASSIGN);
                 });
             }
-            Stmt::Crement(ast::StmtCrement {
-                target,
-                op,
-                range: _,
-            }) => {
-                statement!({
-                    self.unparse_expr(target, precedence::AUG_ASSIGN);
-                    self.p(" ");
-                    self.p(match op {
-                        CrementKind::Increment => "+= 1",
-                        CrementKind::Decrement => "-= 1",
-                    });
-                });
-            }
             Stmt::AnnAssign(ast::StmtAnnAssign {
                 target,
                 annotation,

crates/ruff_python_formatter/src/lib.rs

@@ -135,7 +135,7 @@ pub fn format_module_source(
     let source_type = options.source_type();
     let (tokens, comment_ranges) =
         tokens_and_ranges(source, source_type).map_err(|err| ParseError {
-            offset: err.location(),
+            location: err.location(),
             error: ParseErrorType::Lexical(err.into_error()),
         })?;
     let module = parse_tokens(tokens, source, source_type.as_mode())?;

crates/ruff_python_formatter/src/range.rs

@@ -73,7 +73,7 @@ pub fn format_range(
 
     let (tokens, comment_ranges) =
         tokens_and_ranges(source, options.source_type()).map_err(|err| ParseError {
-            offset: err.location(),
+            location: err.location(),
             error: ParseErrorType::Lexical(err.into_error()),
         })?;
 
@@ -706,8 +706,7 @@ impl Format<PyFormatContext<'_>> for FormatEnclosingNode<'_> {
             | AnyNodeRef::TypeParamTypeVar(_)
             | AnyNodeRef::TypeParamTypeVarTuple(_)
             | AnyNodeRef::TypeParamParamSpec(_)
-            | AnyNodeRef::BytesLiteral(_)
-            | AnyNodeRef::StmtCrement(_) => {
+            | AnyNodeRef::BytesLiteral(_) => {
                 panic!("Range formatting only supports formatting logical lines")
             }
         }

crates/ruff_python_formatter/src/statement/mod.rs

@@ -45,19 +45,6 @@ impl FormatRule<Stmt, PyFormatContext<'_>> for FormatStmt {
             Stmt::Delete(x) => x.format().fmt(f),
             Stmt::Assign(x) => x.format().fmt(f),
             Stmt::AugAssign(x) => x.format().fmt(f),
-            Stmt::Crement(x) => {
-                x.target.format().fmt(f)?;
-                use ruff_formatter::prelude::*;
-                match x.op {
-                    ruff_python_ast::CrementKind::Increment => {
-                        ruff_formatter::write!(f, [space(), text("+="), space(), text("1")])?;
-                    }
-                    ruff_python_ast::CrementKind::Decrement => {
-                        ruff_formatter::write!(f, [space(), text("-="), space(), text("1")])?;
-                    }
-                }
-                Ok(())
-            }
             Stmt::AnnAssign(x) => x.format().fmt(f),
             Stmt::For(x) => x.format().fmt(f),
             Stmt::While(x) => x.format().fmt(f),

crates/ruff_python_formatter/tests/snapshots/format@empty_multiple_trailing_newlines.py.snap

@@ -13,6 +13,3 @@ input_file: crates/ruff_python_formatter/resources/test/fixtures/ruff/empty_mult
 ```python
 
 ```
-
-
-

crates/ruff_python_formatter/tests/snapshots/format@empty_trailing_newline.py.snap

@@ -11,6 +11,3 @@ input_file: crates/ruff_python_formatter/resources/test/fixtures/ruff/empty_trai
 ```python
 
 ```
-
-
-

crates/ruff_python_parser/Cargo.toml

@@ -19,6 +19,7 @@ ruff_text_size = { path = "../ruff_text_size" }
 
 anyhow = { workspace = true }
 bitflags = { workspace = true }
+drop_bomb = { workspace = true }
 bstr = { workspace = true }
 is-macro = { workspace = true }
 itertools = { workspace = true }
@@ -30,7 +31,10 @@ unicode-ident = { workspace = true }
 unicode_names2 = { workspace = true }
 
 [dev-dependencies]
-insta = { workspace = true }
+ruff_source_file = { path = "../ruff_source_file" }
+
+annotate-snippets = { workspace = true }
+insta = { workspace = true, features = ["glob"] }
 
 [build-dependencies]
 anyhow = { workspace = true }

crates/ruff_python_parser/build.rs

@@ -5,7 +5,7 @@ use std::path::{Path, PathBuf};
 use tiny_keccak::{Hasher, Sha3};
 
 fn main() {
-    const SOURCE: &str = "src/python.lalrpop";
+    const SOURCE: &str = "src/lalrpop/python.lalrpop";
     println!("cargo:rerun-if-changed={SOURCE}");
 
     let target;
@@ -14,12 +14,12 @@ fn main() {
     #[cfg(feature = "lalrpop")]
     {
         let out_dir = PathBuf::from(std::env::var_os("OUT_DIR").unwrap());
-        target = out_dir.join("src/python.rs");
+        target = out_dir.join("src/lalrpop/python.rs");
     }
     #[cfg(not(feature = "lalrpop"))]
     {
-        target = PathBuf::from("src/python.rs");
-        error = "python.lalrpop and src/python.rs doesn't match. This is a ruff_python_parser bug. Please report it unless you are editing ruff_python_parser. Run `lalrpop src/python.lalrpop` to build ruff_python_parser again.";
+        target = PathBuf::from("src/lalrpop/python.rs");
+        error = "python.lalrpop and src/lalrpop/python.rs doesn't match. This is a ruff_python_parser bug. Please report it unless you are editing ruff_python_parser. Run `lalrpop src/lalrpop/python.lalrpop` to build ruff_python_parser again.";
     }
 
     let Some(message) = requires_lalrpop(SOURCE, &target) else {

crates/ruff_python_parser/resources/.editorconfig

@@ -0,0 +1,6 @@
+# Check http://editorconfig.org for more information
+# This is the main config file for this project:
+root = true
+
+[*.py]
+insert_final_newline = false

crates/ruff_python_parser/resources/invalid/expressions/call_invalid_argument_ordering.py

@@ -0,0 +1,6 @@
+f(b=20, c)
+
+f(**b, *c)
+
+# Duplicate keyword argument
+f(a=20, a=30)

crates/ruff_python_parser/resources/invalid/expressions/emoji_identifiers.py

@@ -0,0 +1,7 @@
+a = (🐶
+    # comment 🐶
+)
+
+a = (🐶 +
+    # comment
+🐶)

crates/ruff_python_parser/resources/invalid/expressions/emoji_statement.py

@@ -0,0 +1 @@
+👍

crates/ruff_python_parser/resources/invalid/expressions/lambda_default_parameters.py

@@ -0,0 +1,2 @@
+# TODO(micha): The offset of the generated error message is off by one.
+lambda a, b=20, c: 1

crates/ruff_python_parser/resources/invalid/expressions/lambda_duplicate_parameters.py

@@ -0,0 +1,9 @@
+lambda a, a: 1
+
+lambda a, *, a: 1
+
+lambda a, a=20: 1
+
+lambda a, *a: 1
+
+lambda a, *, **a: 1

crates/ruff_python_parser/resources/invalid/expressions/unclosed_fstring_lbrace.py

@@ -0,0 +1,8 @@
+# TODO(dhruvmanila): Remove the dummy test case and uncomment the others when this is fixed. See PR #10372
+x +
+
+# f'{'
+# f'{foo!r'
+# f'{foo='
+# f"{"
+# f"""{"""

crates/ruff_python_parser/resources/invalid/statements/assignment_keyword_target.py

@@ -0,0 +1,7 @@
+a = pass = c
+
+a + b
+
+a = b = pass = c
+
+a + b

crates/ruff_python_parser/resources/invalid/statements/assignment_missing_target.py

@@ -0,0 +1,7 @@
+a = = c
+
+a + b
+
+
+
+

crates/ruff_python_parser/resources/invalid/statements/assignment_missing_value.py

@@ -0,0 +1,3 @@
+a =
+
+a + b

crates/ruff_python_parser/resources/invalid/statements/function_default_parameters.py

@@ -0,0 +1,2 @@
+# TODO(micha): The range of the generated error message is off by one.
+def f(a, b=20, c): pass

crates/ruff_python_parser/resources/invalid/statements/function_duplicate_parameters.py

@@ -0,0 +1,12 @@
+def f(a, a): pass
+
+def f2(a, *, a): pass
+
+def f3(a, a=20): pass
+
+def f4(a, *a): pass
+
+def f5(a, *, **a): pass
+
+# TODO(micha): This is inconsistent. All other examples only highlight the argument name.
+def f6(a, a: str): pass

crates/ruff_python_parser/resources/invalid/statements/function_type_parameters.py

@@ -0,0 +1,24 @@
+# FIXME: The type param related error message and the parser recovery are looking pretty good **except**
+# that the lexer never recovers from the unclosed `[`, resulting in it lexing `NonLogicalNewline` tokens instead of `Newline` tokens.
+# That's because the parser has no way of feeding the error recovery back to the lexer,
+# so they don't agree on the state of the world which can lead to all kind of errors further down in the file.
+# This is not just a problem with parentheses but also with the transformation made by the
+# `SoftKeywordTransformer` because the `Parser` and `Transfomer` may not agree if they're
+# currently in a position where the `type` keyword is allowed or not.
+# That roughly means that any kind of recovery can lead to unrelated syntax errors
+# on following lines.
+
+def unclosed[A, *B(test: name):
+    pass
+
+    a + b
+
+def keyword[A, await](): ...
+
+def not_a_type_param[A, |, B](): ...
+
+def multiple_commas[A,,B](): ...
+
+def multiple_trailing_commas[A,,](): ...
+
+def multiple_commas_and_recovery[A,,100](): ...

crates/ruff_python_parser/resources/invalid/statements/if_elif.py

@@ -0,0 +1,7 @@
+if True:
+    pass
+elif False:
+    pass
+elf:
+    pass
+

crates/ruff_python_parser/resources/invalid/statements/if_extra_closing_parentheses.py

@@ -0,0 +1,3 @@
+# FIXME(micha): This creates two syntax errors instead of just one (and overlapping ones)
+if True)):
+    pass

crates/ruff_python_parser/resources/invalid/statements/if_extra_indent.py

@@ -0,0 +1,8 @@
+# Improving the recovery would require changing the lexer to emit an extra dedent token after `a + b`.
+if True:
+    pass
+        a + b
+
+    pass
+
+a = 10

crates/ruff_python_parser/resources/invalid/statements/if_missing_body.py

@@ -0,0 +1,6 @@
+if True:
+
+
+a + b
+
+if False: # This if statement has neither an indent nor a newline.

crates/ruff_python_parser/resources/invalid/statements/if_missing_colon.py

@@ -0,0 +1,4 @@
+if True
+    pass
+
+a = 10

crates/ruff_python_parser/resources/invalid/statements/import_from.py

@@ -0,0 +1,11 @@
+from abc import a, b,
+
+a + b
+
+from abc import ,,
+
+from abc import
+
+from abc import (a, b, c
+
+a + b

crates/ruff_python_parser/resources/invalid/statements/invalid_assignment_targets.py

@@ -0,0 +1,42 @@
+# Regression test: https://github.com/astral-sh/ruff/issues/6895
+# First we test, broadly, that various kinds of assignments are now
+# rejected by the parser. e.g., `5 = 3`, `5 += 3`, `(5): int = 3`.
+
+5 = 3
+
+5 += 3
+
+(5): int = 3
+
+# Now we exhaustively test all possible cases where assignment can fail.
+x or y = 42
+(x := 5) = 42
+x + y = 42
+-x = 42
+(lambda _: 1) = 42
+a if b else c = 42
+{"a": 5} = 42
+{a} = 42
+[x for x in xs] = 42
+{x for x in xs} = 42
+{x: x * 2 for x in xs} = 42
+(x for x in xs) = 42
+await x = 42
+(yield x) = 42
+(yield from xs) = 42
+a < b < c = 42
+foo() = 42
+
+f"{quux}" = 42
+f"{foo} and {bar}" = 42
+
+"foo" = 42
+b"foo" = 42
+123 = 42
+True = 42
+None = 42
+... = 42
+*foo() = 42
+[x, foo(), y] = [42, 42, 42]
+[[a, b], [[42]], d] = [[1, 2], [[3]], 4]
+(x, foo(), y) = (42, 42, 42)

crates/ruff_python_parser/resources/invalid/statements/invalid_augmented_assignment_target.py

@@ -0,0 +1,34 @@
+# This is similar to `./invalid_assignment_targets.py`, but for augmented
+# assignment targets.
+
+x or y += 42
+(x := 5) += 42
+x + y += 42
+-x += 42
+(lambda _: 1) += 42
+a if b else c += 42
+{"a": 5} += 42
+{a} += 42
+[x for x in xs] += 42
+{x for x in xs} += 42
+{x: x * 2 for x in xs} += 42
+(x for x in xs) += 42
+await x += 42
+(yield x) += 42
+(yield from xs) += 42
+a < b < c += 42
+foo() += 42
+
+f"{quux}" += 42
+f"{foo} and {bar}" += 42
+
+"foo" += 42
+b"foo" += 42
+123 += 42
+True += 42
+None += 42
+... += 42
+*foo() += 42
+[x, foo(), y] += [42, 42, 42]
+[[a, b], [[42]], d] += [[1, 2], [[3]], 4]
+(x, foo(), y) += (42, 42, 42)

crates/ruff_python_parser/resources/invalid/statements/named_expression_statement.py

@@ -0,0 +1,3 @@
+# This test previously passed before the assignment operator checking
+# above, but we include it here for good measure.
+(5 := 3)

crates/ruff_python_parser/resources/valid/expressions/dictionary_comprehension.py

@@ -0,0 +1 @@
+x = {y for y in (1, 2, 3)}

crates/ruff_python_parser/resources/valid/expressions/lambda.py

@@ -0,0 +1,11 @@
+lambda: 1
+
+lambda a, b, c: 1
+
+lambda a, b=20, c=30: 1
+
+lambda *, a, b, c: 1
+
+lambda *, a, b=20, c=30: 1
+
+lambda a, b, c, *, d, e: 0

crates/ruff_python_parser/resources/valid/expressions/list_comprehension.py

@@ -0,0 +1 @@
+x = [y for y in (1, 2, 3)]

crates/ruff_python_parser/resources/valid/expressions/named_expression.py

@@ -0,0 +1,4 @@
+if x := 1:
+    pass
+
+(x := 5)

crates/ruff_python_parser/resources/valid/statement/annotated_assignment.py

@@ -0,0 +1 @@
+x: int = 1

crates/ruff_python_parser/resources/valid/statement/assignment.py

@@ -0,0 +1,40 @@
+x = (1, 2, 3)
+
+(x, y) = (1, 2, 3)
+
+[x, y] = (1, 2, 3)
+
+x.y = (1, 2, 3)
+
+x[y] = (1, 2, 3)
+
+(x, *y) = (1, 2, 3)
+
+
+# This last group of tests checks that assignments we expect to be parsed
+# (including some interesting ones) continue to be parsed successfully.
+
+*foo = 42
+
+[x, y, z] = [1, 2, 3]
+
+(x, y, z) = (1, 2, 3)
+x[0] = 42
+
+# This is actually a type error, not a syntax error. So check that it
+# doesn't fail parsing.
+
+5[0] = 42
+x[1:2] = [42]
+
+# This is actually a type error, not a syntax error. So check that it
+# doesn't fail parsing.
+5[1:2] = [42]
+
+foo.bar = 42
+
+# This is actually an attribute error, not a syntax error. So check that
+# it doesn't fail parsing.
+"foo".y = 42
+
+foo = 42

crates/ruff_python_parser/resources/valid/statement/augmented_assignment.py

@@ -0,0 +1,18 @@
+x += 1
+x.y += (1, 2, 3)
+x[y] += (1, 2, 3)
+
+# All possible augmented assignment tokens
+x += 1
+x -= 1
+x *= 1
+x /= 1
+x //= 1
+x %= 1
+x **= 1
+x &= 1
+x |= 1
+x ^= 1
+x <<= 1
+x >>= 1
+x @= 1

crates/ruff_python_parser/resources/valid/statement/delete.py

@@ -0,0 +1,3 @@
+del x
+del x.y
+del x[y]

crates/ruff_python_parser/resources/valid/statement/for.py

@@ -0,0 +1,2 @@
+for x in (1, 2, 3):
+    pass

crates/ruff_python_parser/resources/valid/statement/function.py

@@ -0,0 +1,38 @@
+def no_parameters():
+    pass
+
+
+def positional_parameters(a, b, c):
+    pass
+
+
+def positional_parameters_with_default_values(a, b=20, c=30):
+    pass
+
+
+def keyword_only_parameters(*, a, b, c):
+    pass
+
+
+def keyword_only_parameters_with_defaults(*, a, b=20, c=30):
+    pass
+
+
+def positional_and_keyword_parameters(a, b, c, *, d, e, f):
+    pass
+
+
+def positional_and_keyword_parameters_with_defaults(a, b, c, *, d, e=20, f=30):
+    pass
+
+
+def positional_and_keyword_parameters_with_defaults_and_varargs(
+    a, b, c, *args, d, e=20, f=30
+):
+    pass
+
+
+def positional_and_keyword_parameters_with_defaults_and_varargs_and_kwargs(
+    a, b, c, *args, d, e=20, f=30, **kwargs
+):
+    pass

crates/ruff_python_parser/resources/valid/statement/with.py

@@ -0,0 +1,2 @@
+with 1 as x:
+    pass

crates/ruff_python_parser/src/error.rs

@@ -0,0 +1,221 @@
+use std::fmt;
+
+use ruff_text_size::TextRange;
+
+use crate::{
+    lexer::{LexicalError, LexicalErrorType},
+    Tok, TokenKind,
+};
+
+/// Represents represent errors that occur during parsing and are
+/// returned by the `parse_*` functions.
+#[derive(Debug, PartialEq)]
+pub struct ParseError {
+    pub error: ParseErrorType,
+    pub location: TextRange,
+}
+
+impl std::ops::Deref for ParseError {
+    type Target = ParseErrorType;
+
+    fn deref(&self) -> &Self::Target {
+        &self.error
+    }
+}
+
+impl std::error::Error for ParseError {
+    fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
+        Some(&self.error)
+    }
+}
+
+impl fmt::Display for ParseError {
+    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+        write!(f, "{} at byte range {:?}", &self.error, self.location)
+    }
+}
+
+impl From<LexicalError> for ParseError {
+    fn from(error: LexicalError) -> Self {
+        ParseError {
+            location: error.location(),
+            error: ParseErrorType::Lexical(error.into_error()),
+        }
+    }
+}
+
+impl ParseError {
+    pub fn error(self) -> ParseErrorType {
+        self.error
+    }
+}
+
+/// Represents the different types of errors that can occur during parsing of an f-string.
+#[derive(Debug, Clone, PartialEq)]
+pub enum FStringErrorType {
+    /// Expected a right brace after an opened left brace.
+    UnclosedLbrace,
+    /// An invalid conversion flag was encountered.
+    InvalidConversionFlag,
+    /// A single right brace was encountered.
+    SingleRbrace,
+    /// Unterminated string.
+    UnterminatedString,
+    /// Unterminated triple-quoted string.
+    UnterminatedTripleQuotedString,
+    /// A lambda expression without parentheses was encountered.
+    LambdaWithoutParentheses,
+}
+
+impl std::fmt::Display for FStringErrorType {
+    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+        use FStringErrorType::{
+            InvalidConversionFlag, LambdaWithoutParentheses, SingleRbrace, UnclosedLbrace,
+            UnterminatedString, UnterminatedTripleQuotedString,
+        };
+        match self {
+            UnclosedLbrace => write!(f, "expecting '}}'"),
+            InvalidConversionFlag => write!(f, "invalid conversion character"),
+            SingleRbrace => write!(f, "single '}}' is not allowed"),
+            UnterminatedString => write!(f, "unterminated string"),
+            UnterminatedTripleQuotedString => write!(f, "unterminated triple-quoted string"),
+            LambdaWithoutParentheses => {
+                write!(f, "lambda expressions are not allowed without parentheses")
+            }
+        }
+    }
+}
+
+/// Represents the different types of errors that can occur during parsing.
+#[derive(Debug, PartialEq)]
+pub enum ParseErrorType {
+    /// An unexpected error occurred.
+    OtherError(String),
+    /// An empty slice was found during parsing, e.g `l[]`.
+    EmptySlice,
+    /// An invalid expression was found in the assignment `target`.
+    InvalidAssignmentTarget,
+    /// An invalid expression was found in the named assignment `target`.
+    InvalidNamedAssignmentTarget,
+    /// An invalid expression was found in the augmented assignment `target`.
+    InvalidAugmentedAssignmentTarget,
+    /// An invalid expression was found in the delete `target`.
+    InvalidDeleteTarget,
+    /// Multiple simple statements were found in the same line without a `;` separating them.
+    SimpleStmtsInSameLine,
+    /// An unexpected indentation was found during parsing.
+    UnexpectedIndentation,
+    /// The statement being parsed cannot be `async`.
+    StmtIsNotAsync(TokenKind),
+    /// A parameter was found after a vararg
+    ParamFollowsVarKeywordParam,
+    /// A positional argument follows a keyword argument.
+    PositionalArgumentError,
+    /// An iterable argument unpacking `*args` follows keyword argument unpacking `**kwargs`.
+    UnpackedArgumentError,
+    /// A non-default argument follows a default argument.
+    DefaultArgumentError,
+    /// A simple statement and a compound statement was found in the same line.
+    SimpleStmtAndCompoundStmtInSameLine,
+    /// An invalid `match` case pattern was found.
+    InvalidMatchPatternLiteral { pattern: TokenKind },
+    /// The parser expected a specific token that was not found.
+    ExpectedToken {
+        expected: TokenKind,
+        found: TokenKind,
+    },
+    /// A duplicate argument was found in a function definition.
+    DuplicateArgumentError(String),
+    /// A keyword argument was repeated.
+    DuplicateKeywordArgumentError(String),
+    /// An f-string error containing the [`FStringErrorType`].
+    FStringError(FStringErrorType),
+    /// Parser encountered an error during lexing.
+    Lexical(LexicalErrorType),
+
+    // RustPython specific.
+    /// Parser encountered an extra token
+    ExtraToken(Tok),
+    /// Parser encountered an invalid token
+    InvalidToken,
+    /// Parser encountered an unexpected token
+    UnrecognizedToken(Tok, Option<String>),
+    /// Parser encountered an unexpected end of input
+    Eof,
+}
+
+impl std::error::Error for ParseErrorType {}
+
+impl std::fmt::Display for ParseErrorType {
+    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+        match self {
+            ParseErrorType::OtherError(msg) => write!(f, "{msg}"),
+            ParseErrorType::ExpectedToken { found, expected } => {
+                write!(f, "expected {expected:?}, found {found:?}")
+            }
+            ParseErrorType::Lexical(ref lex_error) => write!(f, "{lex_error}"),
+            ParseErrorType::SimpleStmtsInSameLine => {
+                write!(f, "use `;` to separate simple statements")
+            }
+            ParseErrorType::SimpleStmtAndCompoundStmtInSameLine => write!(
+                f,
+                "compound statements not allowed in the same line as simple statements"
+            ),
+            ParseErrorType::StmtIsNotAsync(kind) => {
+                write!(f, "`{kind:?}` statement cannot be async")
+            }
+            ParseErrorType::UnpackedArgumentError => {
+                write!(
+                    f,
+                    "iterable argument unpacking follows keyword argument unpacking"
+                )
+            }
+            ParseErrorType::PositionalArgumentError => {
+                write!(f, "positional argument follows keyword argument unpacking")
+            }
+            ParseErrorType::EmptySlice => write!(f, "slice cannot be empty"),
+            ParseErrorType::ParamFollowsVarKeywordParam => {
+                write!(f, "parameters cannot follow var-keyword parameter")
+            }
+            ParseErrorType::DefaultArgumentError => {
+                write!(f, "non-default argument follows default argument")
+            }
+            ParseErrorType::InvalidMatchPatternLiteral { pattern } => {
+                write!(f, "invalid pattern `{pattern:?}`")
+            }
+            ParseErrorType::UnexpectedIndentation => write!(f, "unexpected indentation"),
+            ParseErrorType::InvalidAssignmentTarget => write!(f, "invalid assignment target"),
+            ParseErrorType::InvalidNamedAssignmentTarget => {
+                write!(f, "invalid named assignment target")
+            }
+            ParseErrorType::InvalidAugmentedAssignmentTarget => {
+                write!(f, "invalid augmented assignment target")
+            }
+            ParseErrorType::InvalidDeleteTarget => {
+                write!(f, "invalid delete target")
+            }
+            ParseErrorType::DuplicateArgumentError(arg_name) => {
+                write!(f, "duplicate argument '{arg_name}' in function definition")
+            }
+            ParseErrorType::DuplicateKeywordArgumentError(arg_name) => {
+                write!(f, "keyword argument repeated: {arg_name}")
+            }
+            ParseErrorType::FStringError(ref fstring_error) => {
+                write!(f, "f-string: {fstring_error}")
+            }
+            // RustPython specific.
+            ParseErrorType::Eof => write!(f, "Got unexpected EOF"),
+            ParseErrorType::ExtraToken(ref tok) => write!(f, "Got extraneous token: {tok:?}"),
+            ParseErrorType::InvalidToken => write!(f, "Got invalid token"),
+            ParseErrorType::UnrecognizedToken(ref tok, ref expected) => {
+                if *tok == Tok::Indent {
+                    write!(f, "Unexpected indent")
+                } else if expected.as_deref() == Some("Indent") {
+                    write!(f, "Expected an indented block")
+                } else {
+                    write!(f, "Unexpected token {tok}")
+                }
+            }
+        }
+    }
+}

crates/ruff_python_parser/src/invalid.rs

@@ -1,699 +0,0 @@
-/*!
-Defines some helper routines for rejecting invalid Python programs.
-
-These routines are named in a way that supports qualified use. For example,
-`invalid::assignment_targets`.
-*/
-
-use {ruff_python_ast::Expr, ruff_text_size::TextSize};
-
-use crate::lexer::{LexicalError, LexicalErrorType};
-
-/// Returns an error for invalid assignment targets.
-///
-/// # Errors
-///
-/// This returns an error when any of the given expressions are themselves
-/// or contain an expression that is invalid on the left hand side of an
-/// assignment. For example, all literal expressions are invalid assignment
-/// targets.
-pub(crate) fn assignment_targets(targets: &[Expr]) -> Result<(), LexicalError> {
-    for t in targets {
-        assignment_target(t)?;
-    }
-    Ok(())
-}
-
-/// Returns an error if the given target is invalid for the left hand side of
-/// an assignment.
-///
-/// # Errors
-///
-/// This returns an error when the given expression is itself or contains an
-/// expression that is invalid on the left hand side of an assignment. For
-/// example, all literal expressions are invalid assignment targets.
-pub(crate) fn assignment_target(target: &Expr) -> Result<(), LexicalError> {
-    // Allowing a glob import here because of its limited scope.
-    #[allow(clippy::enum_glob_use)]
-    use self::Expr::*;
-
-    let err = |location: TextSize| -> LexicalError {
-        let error = LexicalErrorType::AssignmentError;
-        LexicalError::new(error, location)
-    };
-    match *target {
-        BoolOp(ref e) => Err(err(e.range.start())),
-        Named(ref e) => Err(err(e.range.start())),
-        BinOp(ref e) => Err(err(e.range.start())),
-        UnaryOp(ref e) => Err(err(e.range.start())),
-        Lambda(ref e) => Err(err(e.range.start())),
-        If(ref e) => Err(err(e.range.start())),
-        Dict(ref e) => Err(err(e.range.start())),
-        Set(ref e) => Err(err(e.range.start())),
-        ListComp(ref e) => Err(err(e.range.start())),
-        SetComp(ref e) => Err(err(e.range.start())),
-        DictComp(ref e) => Err(err(e.range.start())),
-        Generator(ref e) => Err(err(e.range.start())),
-        Await(ref e) => Err(err(e.range.start())),
-        Yield(ref e) => Err(err(e.range.start())),
-        YieldFrom(ref e) => Err(err(e.range.start())),
-        Compare(ref e) => Err(err(e.range.start())),
-        Call(ref e) => Err(err(e.range.start())),
-        // FString is recursive, but all its forms are invalid as an
-        // assignment target, so we can reject it without exploring it.
-        FString(ref e) => Err(err(e.range.start())),
-        StringLiteral(ref e) => Err(err(e.range.start())),
-        BytesLiteral(ref e) => Err(err(e.range.start())),
-        NumberLiteral(ref e) => Err(err(e.range.start())),
-        BooleanLiteral(ref e) => Err(err(e.range.start())),
-        NoneLiteral(ref e) => Err(err(e.range.start())),
-        EllipsisLiteral(ref e) => Err(err(e.range.start())),
-        // This isn't in the Python grammar but is Jupyter notebook specific.
-        // It seems like this should be an error. It does also seem like the
-        // parser prevents this from ever appearing as an assignment target
-        // anyway. ---AG
-        IpyEscapeCommand(ref e) => Err(err(e.range.start())),
-        // The only nested expressions allowed as an assignment target
-        // are star exprs, lists and tuples.
-        Starred(ref e) => assignment_target(&e.value),
-        List(ref e) => assignment_targets(&e.elts),
-        Tuple(ref e) => assignment_targets(&e.elts),
-        // Subscript is recursive and can be invalid, but aren't syntax errors.
-        // For example, `5[1] = 42` is a type error.
-        Subscript(_) => Ok(()),
-        // Similar to Subscript, e.g., `5[1:2] = [42]` is a type error.
-        Slice(_) => Ok(()),
-        // Similar to Subscript, e.g., `"foo".y = 42` is an attribute error.
-        Attribute(_) => Ok(()),
-        // These are always valid as assignment targets.
-        Name(_) => Ok(()),
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use crate::parse_suite;
-
-    // First we test, broadly, that various kinds of assignments are now
-    // rejected by the parser. e.g., `5 = 3`, `5 += 3`, `(5): int = 3`.
-
-    // Regression test: https://github.com/astral-sh/ruff/issues/6895
-    #[test]
-    fn err_literal_assignment() {
-        let ast = parse_suite(r"5 = 3");
-        insta::assert_debug_snapshot!(ast, @r###"
-        Err(
-            ParseError {
-                error: Lexical(
-                    AssignmentError,
-                ),
-                offset: 0,
-            },
-        )
-        "###);
-    }
-
-    // This test previously passed before the assignment operator checking
-    // above, but we include it here for good measure.
-    #[test]
-    fn err_assignment_expr() {
-        let ast = parse_suite(r"(5 := 3)");
-        insta::assert_debug_snapshot!(ast, @r###"
-        Err(
-            ParseError {
-                error: UnrecognizedToken(
-                    ColonEqual,
-                    None,
-                ),
-                offset: 3,
-            },
-        )
-        "###);
-    }
-
-    #[test]
-    fn err_literal_augment_assignment() {
-        let ast = parse_suite(r"5 += 3");
-        insta::assert_debug_snapshot!(ast, @r###"
-        Err(
-            ParseError {
-                error: Lexical(
-                    AssignmentError,
-                ),
-                offset: 0,
-            },
-        )
-        "###);
-    }
-
-    #[test]
-    fn err_literal_annotation_assignment() {
-        let ast = parse_suite(r"(5): int = 3");
-        insta::assert_debug_snapshot!(ast, @r###"
-        Err(
-            ParseError {
-                error: Lexical(
-                    AssignmentError,
-                ),
-                offset: 1,
-            },
-        )
-        "###);
-    }
-
-    // Now we exhaustively test all possible cases where assignment can fail.
-
-    #[test]
-    fn err_bool_op() {
-        let ast = parse_suite(r"x or y = 42");
-        insta::assert_debug_snapshot!(ast, @r###"
-        Err(
-            ParseError {
-                error: Lexical(
-                    AssignmentError,
-                ),
-                offset: 0,
-            },
-        )
-        "###);
-    }
-
-    #[test]
-    fn err_named_expr() {
-        let ast = parse_suite(r"(x := 5) = 42");
-        insta::assert_debug_snapshot!(ast, @r###"
-        Err(
-            ParseError {
-                error: Lexical(
-                    AssignmentError,
-                ),
-                offset: 1,
-            },
-        )
-        "###);
-    }
-
-    #[test]
-    fn err_bin_op() {
-        let ast = parse_suite(r"x + y = 42");
-        insta::assert_debug_snapshot!(ast, @r###"
-        Err(
-            ParseError {
-                error: Lexical(
-                    AssignmentError,
-                ),
-                offset: 0,
-            },
-        )
-        "###);
-    }
-
-    #[test]
-    fn err_unary_op() {
-        let ast = parse_suite(r"-x = 42");
-        insta::assert_debug_snapshot!(ast, @r###"
-        Err(
-            ParseError {
-                error: Lexical(
-                    AssignmentError,
-                ),
-                offset: 0,
-            },
-        )
-        "###);
-    }
-
-    #[test]
-    fn err_lambda() {
-        let ast = parse_suite(r"(lambda _: 1) = 42");
-        insta::assert_debug_snapshot!(ast, @r###"
-        Err(
-            ParseError {
-                error: Lexical(
-                    AssignmentError,
-                ),
-                offset: 1,
-            },
-        )
-        "###);
-    }
-
-    #[test]
-    fn err_if_exp() {
-        let ast = parse_suite(r"a if b else c = 42");
-        insta::assert_debug_snapshot!(ast, @r###"
-        Err(
-            ParseError {
-                error: Lexical(
-                    AssignmentError,
-                ),
-                offset: 0,
-            },
-        )
-        "###);
-    }
-
-    #[test]
-    fn err_dict() {
-        let ast = parse_suite(r"{'a':5} = 42");
-        insta::assert_debug_snapshot!(ast, @r###"
-        Err(
-            ParseError {
-                error: Lexical(
-                    AssignmentError,
-                ),
-                offset: 0,
-            },
-        )
-        "###);
-    }
-
-    #[test]
-    fn err_set() {
-        let ast = parse_suite(r"{a} = 42");
-        insta::assert_debug_snapshot!(ast, @r###"
-        Err(
-            ParseError {
-                error: Lexical(
-                    AssignmentError,
-                ),
-                offset: 0,
-            },
-        )
-        "###);
-    }
-
-    #[test]
-    fn err_list_comp() {
-        let ast = parse_suite(r"[x for x in xs] = 42");
-        insta::assert_debug_snapshot!(ast, @r###"
-        Err(
-            ParseError {
-                error: Lexical(
-                    AssignmentError,
-                ),
-                offset: 0,
-            },
-        )
-        "###);
-    }
-
-    #[test]
-    fn err_set_comp() {
-        let ast = parse_suite(r"{x for x in xs} = 42");
-        insta::assert_debug_snapshot!(ast, @r###"
-        Err(
-            ParseError {
-                error: Lexical(
-                    AssignmentError,
-                ),
-                offset: 0,
-            },
-        )
-        "###);
-    }
-
-    #[test]
-    fn err_dict_comp() {
-        let ast = parse_suite(r"{x: x*2 for x in xs} = 42");
-        insta::assert_debug_snapshot!(ast, @r###"
-        Err(
-            ParseError {
-                error: Lexical(
-                    AssignmentError,
-                ),
-                offset: 0,
-            },
-        )
-        "###);
-    }
-
-    #[test]
-    fn err_generator_exp() {
-        let ast = parse_suite(r"(x for x in xs) = 42");
-        insta::assert_debug_snapshot!(ast, @r###"
-        Err(
-            ParseError {
-                error: Lexical(
-                    AssignmentError,
-                ),
-                offset: 0,
-            },
-        )
-        "###);
-    }
-
-    #[test]
-    fn err_await() {
-        let ast = parse_suite(r"await x = 42");
-        insta::assert_debug_snapshot!(ast, @r###"
-        Err(
-            ParseError {
-                error: Lexical(
-                    AssignmentError,
-                ),
-                offset: 0,
-            },
-        )
-        "###);
-    }
-
-    #[test]
-    fn err_yield() {
-        let ast = parse_suite(r"(yield x) = 42");
-        insta::assert_debug_snapshot!(ast, @r###"
-        Err(
-            ParseError {
-                error: Lexical(
-                    AssignmentError,
-                ),
-                offset: 1,
-            },
-        )
-        "###);
-    }
-
-    #[test]
-    fn err_yield_from() {
-        let ast = parse_suite(r"(yield from xs) = 42");
-        insta::assert_debug_snapshot!(ast, @r###"
-        Err(
-            ParseError {
-                error: Lexical(
-                    AssignmentError,
-                ),
-                offset: 1,
-            },
-        )
-        "###);
-    }
-
-    #[test]
-    fn err_compare() {
-        let ast = parse_suite(r"a < b < c = 42");
-        insta::assert_debug_snapshot!(ast, @r###"
-        Err(
-            ParseError {
-                error: Lexical(
-                    AssignmentError,
-                ),
-                offset: 0,
-            },
-        )
-        "###);
-    }
-
-    #[test]
-    fn err_call() {
-        let ast = parse_suite(r"foo() = 42");
-        insta::assert_debug_snapshot!(ast, @r###"
-        Err(
-            ParseError {
-                error: Lexical(
-                    AssignmentError,
-                ),
-                offset: 0,
-            },
-        )
-        "###);
-    }
-
-    #[test]
-    fn err_formatted_value() {
-        // N.B. It looks like the parser can't generate a top-level
-        // FormattedValue, where as the official Python AST permits
-        // representing a single f-string containing just a variable as a
-        // FormattedValue directly.
-        //
-        // Bottom line is that because of this, this test is (at present)
-        // duplicative with the `fstring` test. That is, in theory these tests
-        // could fail independently, but in practice their failure or success
-        // is coupled.
-        //
-        // See: https://docs.python.org/3/library/ast.html#ast.FormattedValue
-        let ast = parse_suite(r#"f"{quux}" = 42"#);
-        insta::assert_debug_snapshot!(ast, @r###"
-        Err(
-            ParseError {
-                error: Lexical(
-                    AssignmentError,
-                ),
-                offset: 0,
-            },
-        )
-        "###);
-    }
-
-    #[test]
-    fn err_fstring() {
-        let ast = parse_suite(r#"f"{foo} and {bar}" = 42"#);
-        insta::assert_debug_snapshot!(ast, @r###"
-        Err(
-            ParseError {
-                error: Lexical(
-                    AssignmentError,
-                ),
-                offset: 0,
-            },
-        )
-        "###);
-    }
-
-    #[test]
-    fn err_string_literal() {
-        let ast = parse_suite(r#""foo" = 42"#);
-        insta::assert_debug_snapshot!(ast, @r###"
-        Err(
-            ParseError {
-                error: Lexical(
-                    AssignmentError,
-                ),
-                offset: 0,
-            },
-        )
-        "###);
-    }
-
-    #[test]
-    fn err_bytes_literal() {
-        let ast = parse_suite(r#"b"foo" = 42"#);
-        insta::assert_debug_snapshot!(ast, @r###"
-        Err(
-            ParseError {
-                error: Lexical(
-                    AssignmentError,
-                ),
-                offset: 0,
-            },
-        )
-        "###);
-    }
-
-    #[test]
-    fn err_number_literal() {
-        let ast = parse_suite(r"123 = 42");
-        insta::assert_debug_snapshot!(ast, @r###"
-        Err(
-            ParseError {
-                error: Lexical(
-                    AssignmentError,
-                ),
-                offset: 0,
-            },
-        )
-        "###);
-    }
-
-    #[test]
-    fn err_boolean_literal() {
-        let ast = parse_suite(r"True = 42");
-        insta::assert_debug_snapshot!(ast, @r###"
-        Err(
-            ParseError {
-                error: Lexical(
-                    AssignmentError,
-                ),
-                offset: 0,
-            },
-        )
-        "###);
-    }
-
-    #[test]
-    fn err_none_literal() {
-        let ast = parse_suite(r"None = 42");
-        insta::assert_debug_snapshot!(ast, @r###"
-        Err(
-            ParseError {
-                error: Lexical(
-                    AssignmentError,
-                ),
-                offset: 0,
-            },
-        )
-        "###);
-    }
-
-    #[test]
-    fn err_ellipsis_literal() {
-        let ast = parse_suite(r"... = 42");
-        insta::assert_debug_snapshot!(ast, @r###"
-        Err(
-            ParseError {
-                error: Lexical(
-                    AssignmentError,
-                ),
-                offset: 0,
-            },
-        )
-        "###);
-    }
-
-    #[test]
-    fn err_starred() {
-        let ast = parse_suite(r"*foo() = 42");
-        insta::assert_debug_snapshot!(ast, @r###"
-        Err(
-            ParseError {
-                error: Lexical(
-                    AssignmentError,
-                ),
-                offset: 1,
-            },
-        )
-        "###);
-    }
-
-    #[test]
-    fn err_list() {
-        let ast = parse_suite(r"[x, foo(), y] = [42, 42, 42]");
-        insta::assert_debug_snapshot!(ast, @r###"
-        Err(
-            ParseError {
-                error: Lexical(
-                    AssignmentError,
-                ),
-                offset: 4,
-            },
-        )
-        "###);
-    }
-
-    #[test]
-    fn err_list_nested() {
-        let ast = parse_suite(r"[[a, b], [[42]], d] = [[1, 2], [[3]], 4]");
-        insta::assert_debug_snapshot!(ast, @r###"
-        Err(
-            ParseError {
-                error: Lexical(
-                    AssignmentError,
-                ),
-                offset: 11,
-            },
-        )
-        "###);
-    }
-
-    #[test]
-    fn err_tuple() {
-        let ast = parse_suite(r"(x, foo(), y) = (42, 42, 42)");
-        insta::assert_debug_snapshot!(ast, @r###"
-        Err(
-            ParseError {
-                error: Lexical(
-                    AssignmentError,
-                ),
-                offset: 4,
-            },
-        )
-        "###);
-    }
-
-    // This last group of tests checks that assignments we expect to be parsed
-    // (including some interesting ones) continue to be parsed successfully.
-
-    #[test]
-    fn ok_starred() {
-        let ast = parse_suite(r"*foo = 42");
-        insta::assert_debug_snapshot!(ast);
-    }
-
-    #[test]
-    fn ok_list() {
-        let ast = parse_suite(r"[x, y, z] = [1, 2, 3]");
-        insta::assert_debug_snapshot!(ast);
-    }
-
-    #[test]
-    fn ok_tuple() {
-        let ast = parse_suite(r"(x, y, z) = (1, 2, 3)");
-        insta::assert_debug_snapshot!(ast);
-    }
-
-    #[test]
-    fn ok_subscript_normal() {
-        let ast = parse_suite(r"x[0] = 42");
-        insta::assert_debug_snapshot!(ast);
-    }
-
-    // This is actually a type error, not a syntax error. So check that it
-    // doesn't fail parsing.
-    #[test]
-    fn ok_subscript_weird() {
-        let ast = parse_suite(r"5[0] = 42");
-        insta::assert_debug_snapshot!(ast);
-    }
-
-    #[test]
-    fn ok_slice_normal() {
-        let ast = parse_suite(r"x[1:2] = [42]");
-        insta::assert_debug_snapshot!(ast);
-    }
-
-    // This is actually a type error, not a syntax error. So check that it
-    // doesn't fail parsing.
-    #[test]
-    fn ok_slice_weird() {
-        let ast = parse_suite(r"5[1:2] = [42]");
-        insta::assert_debug_snapshot!(ast);
-    }
-
-    #[test]
-    fn ok_attribute_normal() {
-        let ast = parse_suite(r"foo.bar = 42");
-        insta::assert_debug_snapshot!(ast);
-    }
-
-    // This is actually an attribute error, not a syntax error. So check that
-    // it doesn't fail parsing.
-    #[test]
-    fn ok_attribute_weird() {
-        let ast = parse_suite(r#""foo".y = 42"#);
-        insta::assert_debug_snapshot!(ast);
-    }
-
-    #[test]
-    fn ok_name() {
-        let ast = parse_suite(r"foo = 42");
-        insta::assert_debug_snapshot!(ast);
-    }
-
-    // This is a sanity test for what looks like an ipython directive being
-    // assigned to. Although this doesn't actually parse as an assignment
-    // statement, but rather, a directive whose value is `foo = 42`.
-    #[test]
-    fn ok_ipy_escape_command() {
-        use crate::Mode;
-
-        let src = r"!foo = 42";
-        let tokens = crate::lexer::lex(src, Mode::Ipython);
-        let ast = crate::parse_tokens(tokens.collect(), src, Mode::Ipython);
-        insta::assert_debug_snapshot!(ast);
-    }
-
-    #[test]
-    fn ok_assignment_expr() {
-        let ast = parse_suite(r"(x := 5)");
-        insta::assert_debug_snapshot!(ast);
-    }
-}

crates/ruff_python_parser/src/lalrpop/context.rs

@@ -1,18 +1,18 @@
 use ruff_python_ast::{self as ast, Expr, ExprContext};
 
-pub(crate) fn set_context(expr: Expr, ctx: ExprContext) -> Expr {
+pub(super) fn set_context(expr: Expr, ctx: ExprContext) -> Expr {
     match expr {
         Expr::Name(ast::ExprName { id, range, .. }) => ast::ExprName { range, id, ctx }.into(),
         Expr::Tuple(ast::ExprTuple {
             elts,
             range,
-            parenthesized: is_parenthesized,
+            parenthesized,
             ctx: _,
         }) => ast::ExprTuple {
             elts: elts.into_iter().map(|elt| set_context(elt, ctx)).collect(),
             range,
             ctx,
-            parenthesized: is_parenthesized,
+            parenthesized,
         }
         .into(),
 
@@ -55,7 +55,7 @@ pub(crate) fn set_context(expr: Expr, ctx: ExprContext) -> Expr {
 
 #[cfg(test)]
 mod tests {
-    use crate::parser::parse_suite;
+    use crate::parse_suite;
 
     #[test]
     fn test_assign_name() {

crates/ruff_python_parser/src/lalrpop/function.rs

@@ -0,0 +1,138 @@
+use std::hash::BuildHasherDefault;
+// Contains functions that perform validation and parsing of arguments and parameters.
+// Checks apply both to functions and to lambdas.
+use crate::lexer::{LexicalError, LexicalErrorType};
+use ruff_python_ast::{self as ast};
+use ruff_text_size::{Ranged, TextRange, TextSize};
+use rustc_hash::FxHashSet;
+
+pub(crate) struct ArgumentList {
+    pub(crate) args: Vec<ast::Expr>,
+    pub(crate) keywords: Vec<ast::Keyword>,
+}
+
+// Perform validation of function/lambda arguments in a function definition.
+pub(super) fn validate_arguments(arguments: &ast::Parameters) -> Result<(), LexicalError> {
+    let mut all_arg_names = FxHashSet::with_capacity_and_hasher(
+        arguments.posonlyargs.len()
+            + arguments.args.len()
+            + usize::from(arguments.vararg.is_some())
+            + arguments.kwonlyargs.len()
+            + usize::from(arguments.kwarg.is_some()),
+        BuildHasherDefault::default(),
+    );
+
+    let posonlyargs = arguments.posonlyargs.iter();
+    let args = arguments.args.iter();
+    let kwonlyargs = arguments.kwonlyargs.iter();
+
+    let vararg: Option<&ast::Parameter> = arguments.vararg.as_deref();
+    let kwarg: Option<&ast::Parameter> = arguments.kwarg.as_deref();
+
+    for arg in posonlyargs
+        .chain(args)
+        .chain(kwonlyargs)
+        .map(|arg| &arg.parameter)
+        .chain(vararg)
+        .chain(kwarg)
+    {
+        let range = arg.range;
+        let arg_name = arg.name.as_str();
+        if !all_arg_names.insert(arg_name) {
+            return Err(LexicalError::new(
+                LexicalErrorType::DuplicateArgumentError(arg_name.to_string().into_boxed_str()),
+                range,
+            ));
+        }
+    }
+
+    Ok(())
+}
+
+pub(super) fn validate_pos_params(
+    args: &(
+        Vec<ast::ParameterWithDefault>,
+        Vec<ast::ParameterWithDefault>,
+    ),
+) -> Result<(), LexicalError> {
+    let (posonlyargs, args) = args;
+    #[allow(clippy::skip_while_next)]
+    let first_invalid = posonlyargs
+        .iter()
+        .chain(args.iter()) // for all args
+        .skip_while(|arg| arg.default.is_none()) // starting with args without default
+        .skip_while(|arg| arg.default.is_some()) // and then args with default
+        .next(); // there must not be any more args without default
+    if let Some(invalid) = first_invalid {
+        return Err(LexicalError::new(
+            LexicalErrorType::DefaultArgumentError,
+            invalid.parameter.range(),
+        ));
+    }
+    Ok(())
+}
+
+type FunctionArgument = (
+    Option<(TextSize, TextSize, Option<ast::Identifier>)>,
+    ast::Expr,
+);
+
+// Parse arguments as supplied during a function/lambda *call*.
+pub(super) fn parse_arguments(
+    function_arguments: Vec<FunctionArgument>,
+) -> Result<ArgumentList, LexicalError> {
+    let mut args = vec![];
+    let mut keywords = vec![];
+
+    let mut keyword_names = FxHashSet::with_capacity_and_hasher(
+        function_arguments.len(),
+        BuildHasherDefault::default(),
+    );
+    let mut double_starred = false;
+    for (name, value) in function_arguments {
+        if let Some((start, end, name)) = name {
+            // Check for duplicate keyword arguments in the call.
+            if let Some(keyword_name) = &name {
+                if !keyword_names.insert(keyword_name.to_string()) {
+                    return Err(LexicalError::new(
+                        LexicalErrorType::DuplicateKeywordArgumentError(
+                            keyword_name.to_string().into_boxed_str(),
+                        ),
+                        TextRange::new(start, end),
+                    ));
+                }
+            } else {
+                double_starred = true;
+            }
+
+            keywords.push(ast::Keyword {
+                arg: name,
+                value,
+                range: TextRange::new(start, end),
+            });
+        } else {
+            // Positional arguments mustn't follow keyword arguments.
+            if !keywords.is_empty() && !is_starred(&value) {
+                return Err(LexicalError::new(
+                    LexicalErrorType::PositionalArgumentError,
+                    value.range(),
+                ));
+                // Allow starred arguments after keyword arguments but
+                // not after double-starred arguments.
+            } else if double_starred {
+                return Err(LexicalError::new(
+                    LexicalErrorType::UnpackedArgumentError,
+                    value.range(),
+                ));
+            }
+
+            args.push(value);
+        }
+    }
+    Ok(ArgumentList { args, keywords })
+}
+
+// Check if an expression is a starred expression.
+const fn is_starred(exp: &ast::Expr) -> bool {
+    exp.is_starred_expr()
+}

crates/ruff_python_parser/src/lalrpop/invalid.rs

@@ -0,0 +1,93 @@
+/*!
+Defines some helper routines for rejecting invalid Python programs.
+
+These routines are named in a way that supports qualified use. For example,
+`invalid::assignment_targets`.
+*/
+
+use ruff_python_ast::Expr;
+
+use ruff_text_size::TextRange;
+
+use crate::lexer::{LexicalError, LexicalErrorType};
+
+/// Returns an error for invalid assignment targets.
+///
+/// # Errors
+///
+/// This returns an error when any of the given expressions are themselves
+/// or contain an expression that is invalid on the left hand side of an
+/// assignment. For example, all literal expressions are invalid assignment
+/// targets.
+pub(crate) fn assignment_targets(targets: &[Expr]) -> Result<(), LexicalError> {
+    for t in targets {
+        assignment_target(t)?;
+    }
+    Ok(())
+}
+
+/// Returns an error if the given target is invalid for the left hand side of
+/// an assignment.
+///
+/// # Errors
+///
+/// This returns an error when the given expression is itself or contains an
+/// expression that is invalid on the left hand side of an assignment. For
+/// example, all literal expressions are invalid assignment targets.
+pub(crate) fn assignment_target(target: &Expr) -> Result<(), LexicalError> {
+    // Allowing a glob import here because of its limited scope.
+    #[allow(clippy::enum_glob_use)]
+    use self::Expr::*;
+
+    let err = |location: TextRange| -> LexicalError {
+        let error = LexicalErrorType::AssignmentError;
+        LexicalError::new(error, location)
+    };
+    match *target {
+        BoolOp(ref e) => Err(err(e.range)),
+        Named(ref e) => Err(err(e.range)),
+        BinOp(ref e) => Err(err(e.range)),
+        UnaryOp(ref e) => Err(err(e.range)),
+        Lambda(ref e) => Err(err(e.range)),
+        If(ref e) => Err(err(e.range)),
+        Dict(ref e) => Err(err(e.range)),
+        Set(ref e) => Err(err(e.range)),
+        ListComp(ref e) => Err(err(e.range)),
+        SetComp(ref e) => Err(err(e.range)),
+        DictComp(ref e) => Err(err(e.range)),
+        Generator(ref e) => Err(err(e.range)),
+        Await(ref e) => Err(err(e.range)),
+        Yield(ref e) => Err(err(e.range)),
+        YieldFrom(ref e) => Err(err(e.range)),
+        Compare(ref e) => Err(err(e.range)),
+        Call(ref e) => Err(err(e.range)),
+        // FString is recursive, but all its forms are invalid as an
+        // assignment target, so we can reject it without exploring it.
+        FString(ref e) => Err(err(e.range)),
+        StringLiteral(ref e) => Err(err(e.range)),
+        BytesLiteral(ref e) => Err(err(e.range)),
+        NumberLiteral(ref e) => Err(err(e.range)),
+        BooleanLiteral(ref e) => Err(err(e.range)),
+        NoneLiteral(ref e) => Err(err(e.range)),
+        EllipsisLiteral(ref e) => Err(err(e.range)),
+        // This isn't in the Python grammar but is Jupyter notebook specific.
+        // It seems like this should be an error. It does also seem like the
+        // parser prevents this from ever appearing as an assignment target
+        // anyway. ---AG
+        IpyEscapeCommand(ref e) => Err(err(e.range)),
+        // The only nested expressions allowed as an assignment target
+        // are star exprs, lists and tuples.
+        Starred(ref e) => assignment_target(&e.value),
+        List(ref e) => assignment_targets(&e.elts),
+        Tuple(ref e) => assignment_targets(&e.elts),
+        // Subscript is recursive and can be invalid, but aren't syntax errors.
+        // For example, `5[1] = 42` is a type error.
+        Subscript(_) => Ok(()),
+        // Similar to Subscript, e.g., `5[1:2] = [42]` is a type error.
+        Slice(_) => Ok(()),
+        // Similar to Subscript, e.g., `"foo".y = 42` is an attribute error.
+        Attribute(_) => Ok(()),
+        // These are always valid as assignment targets.
+        Name(_) => Ok(()),
+    }
+}

crates/ruff_python_parser/src/lalrpop/mod.rs

@@ -0,0 +1,311 @@
+//! The LALRPOP based parser implementation.
+
+use itertools::Itertools;
+use lalrpop_util::ParseError as LalrpopError;
+
+use ruff_python_ast::{
+    Expr, ExprAttribute, ExprAwait, ExprBinOp, ExprBoolOp, ExprBooleanLiteral, ExprBytesLiteral,
+    ExprCall, ExprCompare, ExprDict, ExprDictComp, ExprEllipsisLiteral, ExprFString, ExprGenerator,
+    ExprIf, ExprIpyEscapeCommand, ExprLambda, ExprList, ExprListComp, ExprName, ExprNamed,
+    ExprNoneLiteral, ExprNumberLiteral, ExprSet, ExprSetComp, ExprSlice, ExprStarred,
+    ExprStringLiteral, ExprSubscript, ExprTuple, ExprUnaryOp, ExprYield, ExprYieldFrom, Mod,
+};
+use ruff_text_size::{Ranged, TextRange, TextSize};
+
+use crate::lexer::{LexResult, LexicalError, LexicalErrorType};
+use crate::{Mode, ParseError, ParseErrorType, Tok};
+
+mod context;
+mod function;
+mod invalid;
+
+#[rustfmt::skip]
+#[allow(unreachable_pub)]
+#[allow(clippy::type_complexity)]
+#[allow(clippy::extra_unused_lifetimes)]
+#[allow(clippy::needless_lifetimes)]
+#[allow(clippy::unused_self)]
+#[allow(clippy::cast_sign_loss)]
+#[allow(clippy::default_trait_access)]
+#[allow(clippy::let_unit_value)]
+#[allow(clippy::just_underscores_and_digits)]
+#[allow(clippy::no_effect_underscore_binding)]
+#[allow(clippy::trivially_copy_pass_by_ref)]
+#[allow(clippy::option_option)]
+#[allow(clippy::unnecessary_wraps)]
+#[allow(clippy::uninlined_format_args)]
+#[allow(clippy::cloned_instead_of_copied)]
+mod python {
+
+    #[cfg(feature = "lalrpop")]
+    include!(concat!(env!("OUT_DIR"), "/src/lalrpop/python.rs"));
+
+    #[cfg(not(feature = "lalrpop"))]
+    include!("python.rs");
+}
+
+pub(crate) fn parse_tokens(
+    tokens: Vec<LexResult>,
+    source: &str,
+    mode: Mode,
+) -> Result<Mod, ParseError> {
+    let marker_token = (Tok::start_marker(mode), TextRange::default());
+    let lexer = std::iter::once(Ok(marker_token)).chain(
+        tokens
+            .into_iter()
+            .filter_ok(|token| !matches!(token, (Tok::Comment(..) | Tok::NonLogicalNewline, _))),
+    );
+    python::TopParser::new()
+        .parse(
+            source,
+            mode,
+            lexer.map_ok(|(t, range)| (range.start(), t, range.end())),
+        )
+        .map_err(parse_error_from_lalrpop)
+}
+
+fn parse_error_from_lalrpop(err: LalrpopError<TextSize, Tok, LexicalError>) -> ParseError {
+    match err {
+        // TODO: Are there cases where this isn't an EOF?
+        LalrpopError::InvalidToken { location } => ParseError {
+            error: ParseErrorType::Eof,
+            location: TextRange::empty(location),
+        },
+        LalrpopError::ExtraToken { token } => ParseError {
+            error: ParseErrorType::ExtraToken(token.1),
+            location: TextRange::new(token.0, token.2),
+        },
+        LalrpopError::User { error } => ParseError {
+            location: error.location(),
+            error: ParseErrorType::Lexical(error.into_error()),
+        },
+        LalrpopError::UnrecognizedToken { token, expected } => {
+            // Hacky, but it's how CPython does it. See PyParser_AddToken,
+            // in particular "Only one possible expected token" comment.
+            let expected = (expected.len() == 1).then(|| expected[0].clone());
+            ParseError {
+                error: ParseErrorType::UnrecognizedToken(token.1, expected),
+                location: TextRange::new(token.0, token.2),
+            }
+        }
+        LalrpopError::UnrecognizedEof { location, expected } => {
+            // This could be an initial indentation error that we should ignore
+            let indent_error = expected == ["Indent"];
+            if indent_error {
+                ParseError {
+                    error: ParseErrorType::Lexical(LexicalErrorType::IndentationError),
+                    location: TextRange::empty(location),
+                }
+            } else {
+                ParseError {
+                    error: ParseErrorType::Eof,
+                    location: TextRange::empty(location),
+                }
+            }
+        }
+    }
+}
+
+/// An expression that may be parenthesized.
+#[derive(Clone, Debug)]
+struct ParenthesizedExpr {
+    /// The range of the expression, including any parentheses.
+    range: TextRange,
+    /// The underlying expression.
+    expr: Expr,
+}
+
+impl ParenthesizedExpr {
+    /// Returns `true` if the expression is parenthesized.
+    fn is_parenthesized(&self) -> bool {
+        self.range.start() != self.expr.range().start()
+    }
+}
+
+impl Ranged for ParenthesizedExpr {
+    fn range(&self) -> TextRange {
+        self.range
+    }
+}
+impl From<Expr> for ParenthesizedExpr {
+    fn from(expr: Expr) -> Self {
+        ParenthesizedExpr {
+            range: expr.range(),
+            expr,
+        }
+    }
+}
+impl From<ParenthesizedExpr> for Expr {
+    fn from(parenthesized_expr: ParenthesizedExpr) -> Self {
+        parenthesized_expr.expr
+    }
+}
+impl From<ExprIpyEscapeCommand> for ParenthesizedExpr {
+    fn from(payload: ExprIpyEscapeCommand) -> Self {
+        Expr::IpyEscapeCommand(payload).into()
+    }
+}
+impl From<ExprBoolOp> for ParenthesizedExpr {
+    fn from(payload: ExprBoolOp) -> Self {
+        Expr::BoolOp(payload).into()
+    }
+}
+impl From<ExprNamed> for ParenthesizedExpr {
+    fn from(payload: ExprNamed) -> Self {
+        Expr::Named(payload).into()
+    }
+}
+impl From<ExprBinOp> for ParenthesizedExpr {
+    fn from(payload: ExprBinOp) -> Self {
+        Expr::BinOp(payload).into()
+    }
+}
+impl From<ExprUnaryOp> for ParenthesizedExpr {
+    fn from(payload: ExprUnaryOp) -> Self {
+        Expr::UnaryOp(payload).into()
+    }
+}
+impl From<ExprLambda> for ParenthesizedExpr {
+    fn from(payload: ExprLambda) -> Self {
+        Expr::Lambda(payload).into()
+    }
+}
+impl From<ExprIf> for ParenthesizedExpr {
+    fn from(payload: ExprIf) -> Self {
+        Expr::If(payload).into()
+    }
+}
+impl From<ExprDict> for ParenthesizedExpr {
+    fn from(payload: ExprDict) -> Self {
+        Expr::Dict(payload).into()
+    }
+}
+impl From<ExprSet> for ParenthesizedExpr {
+    fn from(payload: ExprSet) -> Self {
+        Expr::Set(payload).into()
+    }
+}
+impl From<ExprListComp> for ParenthesizedExpr {
+    fn from(payload: ExprListComp) -> Self {
+        Expr::ListComp(payload).into()
+    }
+}
+impl From<ExprSetComp> for ParenthesizedExpr {
+    fn from(payload: ExprSetComp) -> Self {
+        Expr::SetComp(payload).into()
+    }
+}
+impl From<ExprDictComp> for ParenthesizedExpr {
+    fn from(payload: ExprDictComp) -> Self {
+        Expr::DictComp(payload).into()
+    }
+}
+impl From<ExprGenerator> for ParenthesizedExpr {
+    fn from(payload: ExprGenerator) -> Self {
+        Expr::Generator(payload).into()
+    }
+}
+impl From<ExprAwait> for ParenthesizedExpr {
+    fn from(payload: ExprAwait) -> Self {
+        Expr::Await(payload).into()
+    }
+}
+impl From<ExprYield> for ParenthesizedExpr {
+    fn from(payload: ExprYield) -> Self {
+        Expr::Yield(payload).into()
+    }
+}
+impl From<ExprYieldFrom> for ParenthesizedExpr {
+    fn from(payload: ExprYieldFrom) -> Self {
+        Expr::YieldFrom(payload).into()
+    }
+}
+impl From<ExprCompare> for ParenthesizedExpr {
+    fn from(payload: ExprCompare) -> Self {
+        Expr::Compare(payload).into()
+    }
+}
+impl From<ExprCall> for ParenthesizedExpr {
+    fn from(payload: ExprCall) -> Self {
+        Expr::Call(payload).into()
+    }
+}
+impl From<ExprFString> for ParenthesizedExpr {
+    fn from(payload: ExprFString) -> Self {
+        Expr::FString(payload).into()
+    }
+}
+impl From<ExprStringLiteral> for ParenthesizedExpr {
+    fn from(payload: ExprStringLiteral) -> Self {
+        Expr::StringLiteral(payload).into()
+    }
+}
+impl From<ExprBytesLiteral> for ParenthesizedExpr {
+    fn from(payload: ExprBytesLiteral) -> Self {
+        Expr::BytesLiteral(payload).into()
+    }
+}
+impl From<ExprNumberLiteral> for ParenthesizedExpr {
+    fn from(payload: ExprNumberLiteral) -> Self {
+        Expr::NumberLiteral(payload).into()
+    }
+}
+impl From<ExprBooleanLiteral> for ParenthesizedExpr {
+    fn from(payload: ExprBooleanLiteral) -> Self {
+        Expr::BooleanLiteral(payload).into()
+    }
+}
+impl From<ExprNoneLiteral> for ParenthesizedExpr {
+    fn from(payload: ExprNoneLiteral) -> Self {
+        Expr::NoneLiteral(payload).into()
+    }
+}
+impl From<ExprEllipsisLiteral> for ParenthesizedExpr {
+    fn from(payload: ExprEllipsisLiteral) -> Self {
+        Expr::EllipsisLiteral(payload).into()
+    }
+}
+impl From<ExprAttribute> for ParenthesizedExpr {
+    fn from(payload: ExprAttribute) -> Self {
+        Expr::Attribute(payload).into()
+    }
+}
+impl From<ExprSubscript> for ParenthesizedExpr {
+    fn from(payload: ExprSubscript) -> Self {
+        Expr::Subscript(payload).into()
+    }
+}
+impl From<ExprStarred> for ParenthesizedExpr {
+    fn from(payload: ExprStarred) -> Self {
+        Expr::Starred(payload).into()
+    }
+}
+impl From<ExprName> for ParenthesizedExpr {
+    fn from(payload: ExprName) -> Self {
+        Expr::Name(payload).into()
+    }
+}
+impl From<ExprList> for ParenthesizedExpr {
+    fn from(payload: ExprList) -> Self {
+        Expr::List(payload).into()
+    }
+}
+impl From<ExprTuple> for ParenthesizedExpr {
+    fn from(payload: ExprTuple) -> Self {
+        Expr::Tuple(payload).into()
+    }
+}
+impl From<ExprSlice> for ParenthesizedExpr {
+    fn from(payload: ExprSlice) -> Self {
+        Expr::Slice(payload).into()
+    }
+}
+
+#[cfg(target_pointer_width = "64")]
+mod size_assertions {
+    use static_assertions::assert_eq_size;
+
+    use super::ParenthesizedExpr;
+
+    assert_eq_size!(ParenthesizedExpr, [u8; 72]);
+}

crates/ruff_python_parser/src/lalrpop/python.lalrpop

@@ -5,16 +5,19 @@
 
 use ruff_text_size::{Ranged, TextLen, TextRange, TextSize};
 use ruff_python_ast::{self as ast, Int, IpyEscapeKind};
+use super::{
+  function::{ArgumentList, parse_arguments, validate_pos_params, validate_arguments},
+  context::set_context,
+  ParenthesizedExpr,
+  invalid,
+};
 use crate::{
     FStringErrorType,
     Mode,
     lexer::{LexicalError, LexicalErrorType},
-    function::{ArgumentList, parse_arguments, validate_pos_params, validate_arguments},
-    context::set_context,
     string::{StringType, concatenated_strings, parse_fstring_literal_element, parse_string_literal},
     string_token_flags::StringKind,
     token,
-    invalid,
 };
 use lalrpop_util::ParseError;
 
@@ -110,16 +113,6 @@ DelStatement: ast::Stmt = {
 };
 
 ExpressionStatement: ast::Stmt = {
-    <location:@L> <target:TestOrStarExprList> <op:Crement> <end_location:@R> =>? {
-        invalid::assignment_target(&target.expr)?;
-        Ok(ast::Stmt::Crement(
-            ast::StmtCrement {
-                target: Box::new(set_context(target.into(), ast::ExprContext::Store)),
-                op,
-                range: (location..end_location).into()
-            },
-        ))
-    },
     <location:@L> <expression:TestOrStarExprList> <suffix:AssignSuffix*> <end_location:@R> =>? {
         // Just an expression, no assignment:
         if suffix.is_empty() {
@@ -167,33 +160,33 @@ ExpressionStatement: ast::Stmt = {
     },
 };
 
-AssignSuffix: crate::parser::ParenthesizedExpr = {
+AssignSuffix: ParenthesizedExpr = {
     "=" <e:TestListOrYieldExpr> => e,
     "=" <e:IpyEscapeCommandExpr> => e
 };
 
-TestListOrYieldExpr: crate::parser::ParenthesizedExpr = {
+TestListOrYieldExpr: ParenthesizedExpr = {
     TestList,
     YieldExpr
 }
 
 #[inline]
-TestOrStarExprList: crate::parser::ParenthesizedExpr = {
+TestOrStarExprList: ParenthesizedExpr = {
     // as far as I can tell, these were the same
     TestList
 };
 
-TestOrStarExpr: crate::parser::ParenthesizedExpr = {
+TestOrStarExpr: ParenthesizedExpr = {
     Test<"all">,
     StarExpr,
 };
 
-NamedOrStarExpr: crate::parser::ParenthesizedExpr = {
+NamedOrStarExpr: ParenthesizedExpr = {
     NamedExpression,
     StarExpr,
 };
 
-TestOrStarNamedExpr: crate::parser::ParenthesizedExpr = {
+TestOrStarNamedExpr: ParenthesizedExpr = {
     NamedExpressionTest,
     StarExpr,
 };
@@ -214,11 +207,6 @@ AugAssign: ast::Operator = {
     "//=" => ast::Operator::FloorDiv,
 };
 
-Crement: ast::CrementKind = {
-    "++" => ast::CrementKind::Increment,
-    "--" => ast::CrementKind::Decrement,
-};
-
 FlowStatement: ast::Stmt = {
     <location:@L> "break" <end_location:@R> => {
 
@@ -355,20 +343,20 @@ IpyEscapeCommandStatement: ast::Stmt = {
         } else {
             Err(LexicalError::new(
                 LexicalErrorType::OtherError("IPython escape commands are only allowed in `Mode::Ipython`".to_string().into_boxed_str()),
-                location,
+                (location..end_location).into(),
             ))?
         }
     }
 }
 
-IpyEscapeCommandExpr: crate::parser::ParenthesizedExpr = {
+IpyEscapeCommandExpr: ParenthesizedExpr = {
     <location:@L> <c:ipy_escape_command> <end_location:@R> =>? {
         if mode == Mode::Ipython {
             // This should never occur as the lexer won't allow it.
             if !matches!(c.0, IpyEscapeKind::Magic | IpyEscapeKind::Shell) {
                 return Err(LexicalError::new(
                     LexicalErrorType::OtherError("IPython escape command expr is only allowed for % and !".to_string().into_boxed_str()),
-                    location,
+                    (location..end_location).into(),
                 ))?;
             }
             Ok(ast::ExprIpyEscapeCommand {
@@ -379,7 +367,7 @@ IpyEscapeCommandExpr: crate::parser::ParenthesizedExpr = {
         } else {
             Err(LexicalError::new(
                 LexicalErrorType::OtherError("IPython escape commands are only allowed in `Mode::Ipython`".to_string().into_boxed_str()),
-                location,
+                (location..end_location).into(),
             ))?
         }
     }
@@ -399,7 +387,7 @@ IpyHelpEndEscapeCommandStatement: ast::Stmt = {
                     let ast::Expr::NumberLiteral(ast::ExprNumberLiteral { value: ast::Number::Int(integer), .. }) = slice.as_ref() else {
                         return Err(LexicalError::new(
                             LexicalErrorType::OtherError("only integer literals are allowed in Subscript expressions in help end escape command".to_string().into_boxed_str()),
-                            range.start(),
+                            *range,
                         ));
                     };
                     unparse_expr(value, buffer)?;
@@ -415,7 +403,7 @@ IpyHelpEndEscapeCommandStatement: ast::Stmt = {
                 _ => {
                     return Err(LexicalError::new(
                         LexicalErrorType::OtherError("only Name, Subscript and Attribute expressions are allowed in help end escape command".to_string().into_boxed_str()),
-                        expr.start(),
+                        expr.range(),
                     ));
                 }
             }
@@ -426,7 +414,7 @@ IpyHelpEndEscapeCommandStatement: ast::Stmt = {
             return Err(ParseError::User {
                 error: LexicalError::new(
                     LexicalErrorType::OtherError("IPython escape commands are only allowed in `Mode::Ipython`".to_string().into_boxed_str()),
-                    location,
+                    (location..end_location).into(),
                 ),
             });
         }
@@ -438,7 +426,7 @@ IpyHelpEndEscapeCommandStatement: ast::Stmt = {
                 return Err(ParseError::User {
                     error: LexicalError::new(
                         LexicalErrorType::OtherError("maximum of 2 `?` tokens are allowed in help end escape command".to_string().into_boxed_str()),
-                        location,
+                        (location..end_location).into(),
                     ),
                 });
             }
@@ -581,7 +569,7 @@ AsPattern: ast::Pattern = {
         if name.as_str() == "_" {
             Err(LexicalError::new(
                 LexicalErrorType::OtherError("cannot use '_' as a target".to_string().into_boxed_str()),
-                location,
+                (location..end_location).into(),
             ))?
         } else {
             Ok(ast::Pattern::MatchAs(
@@ -648,13 +636,13 @@ StarPattern: ast::Pattern = {
     }.into(),
 }
 
-NumberAtom: crate::parser::ParenthesizedExpr = {
+NumberAtom: ParenthesizedExpr = {
     <location:@L> <value:Number> <end_location:@R> => ast::Expr::NumberLiteral(
         ast::ExprNumberLiteral { value, range: (location..end_location).into() }
     ).into(),
 }
 
-NumberExpr: crate::parser::ParenthesizedExpr = {
+NumberExpr: ParenthesizedExpr = {
     NumberAtom,
     <location:@L> "-" <operand:NumberAtom> <end_location:@R> => ast::Expr::UnaryOp(
         ast::ExprUnaryOp {
@@ -665,7 +653,7 @@ NumberExpr: crate::parser::ParenthesizedExpr = {
     ).into(),
 }
 
-AddOpExpr: crate::parser::ParenthesizedExpr = {
+AddOpExpr: ParenthesizedExpr = {
     <location:@L> <left:NumberExpr> <op:AddOp> <right:NumberAtom> <end_location:@R> => ast::ExprBinOp {
         left: Box::new(left.into()),
         op,
@@ -1249,7 +1237,7 @@ ParameterListStarArgs<ParameterType, StarParameterType, DoubleStarParameterType>
         if kwonlyargs.is_empty() {
             return Err(LexicalError::new(
                 LexicalErrorType::OtherError("named arguments must follow bare *".to_string().into_boxed_str()),
-                location,
+                (location..location).into(),
             ))?;
         }
 
@@ -1314,7 +1302,7 @@ Decorator: ast::Decorator = {
     },
 };
 
-YieldExpr: crate::parser::ParenthesizedExpr = {
+YieldExpr: ParenthesizedExpr = {
     <location:@L> "yield" <value:TestList?> <end_location:@R> => ast::ExprYield {
         value: value.map(ast::Expr::from).map(Box::new),
         range: (location..end_location).into(),
@@ -1325,7 +1313,7 @@ YieldExpr: crate::parser::ParenthesizedExpr = {
     }.into(),
 };
 
-Test<Goal>: crate::parser::ParenthesizedExpr = {
+Test<Goal>: ParenthesizedExpr = {
     <location:@L> <body:OrTest<"all">> "if" <test:OrTest<"all">> "else" <orelse:Test<"all">> <end_location:@R> => ast::ExprIf {
         test: Box::new(test.into()),
         body: Box::new(body.into()),
@@ -1336,12 +1324,12 @@ Test<Goal>: crate::parser::ParenthesizedExpr = {
     LambdaDef,
 };
 
-NamedExpressionTest: crate::parser::ParenthesizedExpr = {
+NamedExpressionTest: ParenthesizedExpr = {
     NamedExpression,
     Test<"all">,
 }
 
-NamedExpressionName: crate::parser::ParenthesizedExpr = {
+NamedExpressionName: ParenthesizedExpr = {
     <location:@L> <id:Identifier> <end_location:@R> => ast::ExprName {
         id: id.into(),
         ctx: ast::ExprContext::Store,
@@ -1349,7 +1337,7 @@ NamedExpressionName: crate::parser::ParenthesizedExpr = {
     }.into(),
 }
 
-NamedExpression: crate::parser::ParenthesizedExpr = {
+NamedExpression: ParenthesizedExpr = {
     <location:@L> <target:NamedExpressionName> ":=" <value:Test<"all">> <end_location:@R> => {
         ast::ExprNamed {
             target: Box::new(target.into()),
@@ -1359,12 +1347,12 @@ NamedExpression: crate::parser::ParenthesizedExpr = {
     },
 };
 
-LambdaDef: crate::parser::ParenthesizedExpr = {
+LambdaDef: ParenthesizedExpr = {
     <location:@L> "lambda" <location_args:@L> <parameters:ParameterList<UntypedParameter, StarUntypedParameter, DoubleStarUntypedParameter>?> <end_location_args:@R> ":" <fstring_middle:fstring_middle?> <body:Test<"all">> <end_location:@R> =>? {
         if fstring_middle.is_some() {
             return Err(LexicalError::new(
                 LexicalErrorType::FStringError(FStringErrorType::LambdaWithoutParentheses),
-                location,
+                (location..end_location).into(),
             ))?;
         }
         parameters.as_ref().map(validate_arguments).transpose()?;
@@ -1377,7 +1365,7 @@ LambdaDef: crate::parser::ParenthesizedExpr = {
     }
 }
 
-OrTest<Goal>: crate::parser::ParenthesizedExpr = {
+OrTest<Goal>: ParenthesizedExpr = {
     <location:@L> <values:(<AndTest<"all">> "or")+> <last: AndTest<"all">> <end_location:@R> => {
         let values = values.into_iter().chain(std::iter::once(last)).map(ast::Expr::from).collect();
         ast::ExprBoolOp { op: ast::BoolOp::Or, values, range: (location..end_location).into() }.into()
@@ -1385,7 +1373,7 @@ OrTest<Goal>: crate::parser::ParenthesizedExpr = {
     AndTest<Goal>,
 };
 
-AndTest<Goal>: crate::parser::ParenthesizedExpr = {
+AndTest<Goal>: ParenthesizedExpr = {
     <location:@L> <values:(<NotTest<"all">> "and")+> <last:NotTest<"all">> <end_location:@R> => {
         let values = values.into_iter().chain(std::iter::once(last)).map(ast::Expr::from).collect();
         ast::ExprBoolOp { op: ast::BoolOp::And, values, range: (location..end_location).into() }.into()
@@ -1393,7 +1381,7 @@ AndTest<Goal>: crate::parser::ParenthesizedExpr = {
     NotTest<Goal>,
 };
 
-NotTest<Goal>: crate::parser::ParenthesizedExpr = {
+NotTest<Goal>: ParenthesizedExpr = {
     <location:@L> "not" <operand:NotTest<"all">> <end_location:@R> => ast::ExprUnaryOp {
         operand: Box::new(operand.into()),
         op: ast::UnaryOp::Not,
@@ -1402,7 +1390,7 @@ NotTest<Goal>: crate::parser::ParenthesizedExpr = {
     Comparison<Goal>,
 };
 
-Comparison<Goal>: crate::parser::ParenthesizedExpr = {
+Comparison<Goal>: ParenthesizedExpr = {
     <location:@L> <left:Expression<"all">> <comparisons:(CompOp Expression<"all">)+> <end_location:@R> => {
         let mut ops = Vec::with_capacity(comparisons.len());
         let mut comparators = Vec::with_capacity(comparisons.len());
@@ -1433,7 +1421,7 @@ CompOp: ast::CmpOp = {
     "is" "not" => ast::CmpOp::IsNot,
 };
 
-Expression<Goal>: crate::parser::ParenthesizedExpr = {
+Expression<Goal>: ParenthesizedExpr = {
     <location:@L> <left:Expression<"all">> "|" <right:XorExpression<"all">> <end_location:@R> => ast::ExprBinOp {
         left: Box::new(left.into()),
         op: ast::Operator::BitOr,
@@ -1443,7 +1431,7 @@ Expression<Goal>: crate::parser::ParenthesizedExpr = {
     XorExpression<Goal>,
 };
 
-XorExpression<Goal>: crate::parser::ParenthesizedExpr = {
+XorExpression<Goal>: ParenthesizedExpr = {
     <location:@L> <left:XorExpression<"all">> "^" <right:AndExpression<"all">> <end_location:@R> => ast::ExprBinOp {
         left: Box::new(left.into()),
         op: ast::Operator::BitXor,
@@ -1453,7 +1441,7 @@ XorExpression<Goal>: crate::parser::ParenthesizedExpr = {
     AndExpression<Goal>,
 };
 
-AndExpression<Goal>: crate::parser::ParenthesizedExpr = {
+AndExpression<Goal>: ParenthesizedExpr = {
     <location:@L> <left:AndExpression<"all">> "&" <right:ShiftExpression<"all">> <end_location:@R> => ast::ExprBinOp {
         left: Box::new(left.into()),
         op: ast::Operator::BitAnd,
@@ -1463,7 +1451,7 @@ AndExpression<Goal>: crate::parser::ParenthesizedExpr = {
     ShiftExpression<Goal>,
 };
 
-ShiftExpression<Goal>: crate::parser::ParenthesizedExpr = {
+ShiftExpression<Goal>: ParenthesizedExpr = {
     <location:@L> <left:ShiftExpression<"all">> <op:ShiftOp> <right:ArithmeticExpression<"all">> <end_location:@R> => ast::ExprBinOp {
         left: Box::new(left.into()),
         op,
@@ -1478,7 +1466,7 @@ ShiftOp: ast::Operator = {
     ">>" => ast::Operator::RShift,
 };
 
-ArithmeticExpression<Goal>: crate::parser::ParenthesizedExpr = {
+ArithmeticExpression<Goal>: ParenthesizedExpr = {
     <location:@L> <left:ArithmeticExpression<"all">> <op:AddOp> <right:Term<"all">> <end_location:@R> => ast::ExprBinOp {
         left: Box::new(left.into()),
         op,
@@ -1493,7 +1481,7 @@ AddOp: ast::Operator = {
     "-" => ast::Operator::Sub,
 };
 
-Term<Goal>: crate::parser::ParenthesizedExpr = {
+Term<Goal>: ParenthesizedExpr = {
     <location:@L> <left:Term<"all">> <op:MulOp> <right:Factor<"all">> <end_location:@R> => ast::ExprBinOp {
         left: Box::new(left.into()),
         op,
@@ -1511,7 +1499,7 @@ MulOp: ast::Operator = {
     "@" => ast::Operator::MatMult,
 };
 
-Factor<Goal>: crate::parser::ParenthesizedExpr = {
+Factor<Goal>: ParenthesizedExpr = {
     <location:@L> <op:UnaryOp> <operand:Factor<"all">> <end_location:@R> => ast::ExprUnaryOp {
         operand: Box::new(operand.into()),
         op,
@@ -1526,7 +1514,7 @@ UnaryOp: ast::UnaryOp = {
     "~" => ast::UnaryOp::Invert,
 };
 
-Power<Goal>: crate::parser::ParenthesizedExpr = {
+Power<Goal>: ParenthesizedExpr = {
     <location:@L> <left:AtomExpr<"all">> "**" <right:Factor<"all">> <end_location:@R> => ast::ExprBinOp {
         left: Box::new(left.into()),
         op: ast::Operator::Pow,
@@ -1536,14 +1524,14 @@ Power<Goal>: crate::parser::ParenthesizedExpr = {
     AtomExpr<Goal>,
 };
 
-AtomExpr<Goal>: crate::parser::ParenthesizedExpr = {
+AtomExpr<Goal>: ParenthesizedExpr = {
     <location:@L> "await" <value:AtomExpr2<"all">> <end_location:@R> => {
         ast::ExprAwait { value: Box::new(value.into()), range: (location..end_location).into() }.into()
     },
     AtomExpr2<Goal>,
 }
 
-AtomExpr2<Goal>: crate::parser::ParenthesizedExpr = {
+AtomExpr2<Goal>: ParenthesizedExpr = {
     Atom<Goal>,
     <location:@L> <func:AtomExpr2<"all">> <arguments:Arguments> <end_location:@R> => ast::ExprCall {
         func: Box::new(func.into()),
@@ -1564,14 +1552,14 @@ AtomExpr2<Goal>: crate::parser::ParenthesizedExpr = {
     }.into(),
 };
 
-SubscriptList: crate::parser::ParenthesizedExpr = {
+SubscriptList: ParenthesizedExpr = {
     Subscript,
     <location:@L> <s1:Subscript> "," <end_location:@R> => {
         ast::ExprTuple {
             elts: vec![s1.into()],
             ctx: ast::ExprContext::Load,
             range: (location..end_location).into(),
-            parenthesized: false
+            parenthesized: false,
         }.into()
     },
     <location:@L> <elts:TwoOrMoreSep<Subscript, ",">> ","? <end_location:@R> => {
@@ -1580,12 +1568,12 @@ SubscriptList: crate::parser::ParenthesizedExpr = {
             elts,
             ctx: ast::ExprContext::Load,
             range: (location..end_location).into(),
-            parenthesized: false
+            parenthesized: false,
         }.into()
     }
 };
 
-Subscript: crate::parser::ParenthesizedExpr = {
+Subscript: ParenthesizedExpr = {
     TestOrStarNamedExpr,
     <location:@L> <lower:Test<"all">?> ":" <upper:Test<"all">?> <step:SliceOp?> <end_location:@R> => {
         let lower = lower.map(ast::Expr::from).map(Box::new);
@@ -1597,7 +1585,7 @@ Subscript: crate::parser::ParenthesizedExpr = {
     }
 };
 
-SliceOp: Option<crate::parser::ParenthesizedExpr> = {
+SliceOp: Option<ParenthesizedExpr> = {
     <location:@L> ":" <e:Test<"all">?> => e,
 }
 
@@ -1634,7 +1622,7 @@ FStringMiddlePattern: ast::FStringElement = {
     FStringReplacementField,
     <location:@L> <fstring_middle:fstring_middle> <end_location:@R> =>? {
         let (source, kind) = fstring_middle;
-        Ok(parse_fstring_literal_element(source, kind, (location..end_location).into())?)
+        Ok(ast::FStringElement::Literal(parse_fstring_literal_element(source, kind, (location..end_location).into())?))
     }
 };
 
@@ -1643,7 +1631,7 @@ FStringReplacementField: ast::FStringElement = {
         if value.expr.is_lambda_expr() && !value.is_parenthesized() {
             return Err(LexicalError::new(
                 LexicalErrorType::FStringError(FStringErrorType::LambdaWithoutParentheses),
-                value.start(),
+                value.range(),
             ))?;
         }
         let debug_text = debug.map(|_| {
@@ -1694,14 +1682,14 @@ FStringConversion: (TextSize, ast::ConversionFlag) = {
             "a" => ast::ConversionFlag::Ascii,
             _ => Err(LexicalError::new(
                 LexicalErrorType::FStringError(FStringErrorType::InvalidConversionFlag),
-                name_location,
+                (location..name_location).into(),
             ))?
         };
         Ok((location, conversion))
     }
 };
 
-Atom<Goal>: crate::parser::ParenthesizedExpr = {
+Atom<Goal>: ParenthesizedExpr = {
     <expr:String> => expr.into(),
     <location:@L> <value:Number> <end_location:@R> => ast::ExprNumberLiteral {
         value,
@@ -1721,7 +1709,7 @@ Atom<Goal>: crate::parser::ParenthesizedExpr = {
     },
     <location:@L> "(" <elts:OneOrMore<Test<"all">>> <trailing_comma:","?> ")" <end_location:@R> if Goal != "no-withitems" => {
         if elts.len() == 1 && trailing_comma.is_none() {
-            crate::parser::ParenthesizedExpr {
+            ParenthesizedExpr {
                 expr: elts.into_iter().next().unwrap().into(),
                 range: (location..end_location).into(),
             }
@@ -1740,10 +1728,10 @@ Atom<Goal>: crate::parser::ParenthesizedExpr = {
             if mid.expr.is_starred_expr() {
                 return Err(LexicalError::new(
                     LexicalErrorType::OtherError("cannot use starred expression here".to_string().into_boxed_str()),
-                    mid.start(),
+                    mid.range(),
                 ))?;
             }
-            Ok(crate::parser::ParenthesizedExpr {
+            Ok(ParenthesizedExpr {
                 expr: mid.into(),
                 range: (location..end_location).into(),
             })
@@ -1761,9 +1749,9 @@ Atom<Goal>: crate::parser::ParenthesizedExpr = {
         elts: Vec::new(),
         ctx: ast::ExprContext::Load,
         range: (location..end_location).into(),
-        parenthesized: true
+        parenthesized: true,
     }.into(),
-    <location:@L> "(" <e:YieldExpr> ")" <end_location:@R> => crate::parser::ParenthesizedExpr {
+    <location:@L> "(" <e:YieldExpr> ")" <end_location:@R> => ParenthesizedExpr {
         expr: e.into(),
         range: (location..end_location).into(),
     },
@@ -1776,7 +1764,7 @@ Atom<Goal>: crate::parser::ParenthesizedExpr = {
     "(" <location:@L> "**" <e:Expression<"all">> ")" <end_location:@R> =>? {
         Err(LexicalError::new(
             LexicalErrorType::OtherError("cannot use double starred expression here".to_string().into_boxed_str()),
-            location,
+            (location..end_location).into(),
         ).into())
     },
     <location:@L> "{" <e:DictLiteralValues?> "}" <end_location:@R> => {
@@ -1813,37 +1801,37 @@ Atom<Goal>: crate::parser::ParenthesizedExpr = {
     <location:@L> "..." <end_location:@R> => ast::ExprEllipsisLiteral { range: (location..end_location).into() }.into(),
 };
 
-ListLiteralValues: Vec<crate::parser::ParenthesizedExpr> = {
+ListLiteralValues: Vec<ParenthesizedExpr> = {
     <e:OneOrMore<TestOrStarNamedExpr>> ","? => e,
 };
 
-DictLiteralValues: Vec<(Option<Box<crate::parser::ParenthesizedExpr>>, crate::parser::ParenthesizedExpr)> = {
+DictLiteralValues: Vec<(Option<Box<ParenthesizedExpr>>, ParenthesizedExpr)> = {
     <elements:OneOrMore<DictElement>> ","? => elements,
 };
 
-DictEntry: (crate::parser::ParenthesizedExpr, crate::parser::ParenthesizedExpr) = {
+DictEntry: (ParenthesizedExpr, ParenthesizedExpr) = {
     <e1: Test<"all">> ":" <e2: Test<"all">> => (e1, e2),
 };
 
-DictElement: (Option<Box<crate::parser::ParenthesizedExpr>>, crate::parser::ParenthesizedExpr) = {
+DictElement: (Option<Box<ParenthesizedExpr>>, ParenthesizedExpr) = {
     <e:DictEntry> => (Some(Box::new(e.0)), e.1),
     "**" <e:Expression<"all">> => (None, e),
 };
 
-SetLiteralValues: Vec<crate::parser::ParenthesizedExpr> = {
+SetLiteralValues: Vec<ParenthesizedExpr> = {
     <e1:OneOrMore<TestOrStarNamedExpr>> ","? => e1
 };
 
-ExpressionOrStarExpression: crate::parser::ParenthesizedExpr = {
+ExpressionOrStarExpression: ParenthesizedExpr = {
     Expression<"all">,
     StarExpr
 };
 
-ExpressionList: crate::parser::ParenthesizedExpr = {
+ExpressionList: ParenthesizedExpr = {
     GenericList<ExpressionOrStarExpression>
 };
 
-ExpressionList2: Vec<crate::parser::ParenthesizedExpr> = {
+ExpressionList2: Vec<ParenthesizedExpr> = {
     <elements:OneOrMore<ExpressionOrStarExpression>> ","? => elements,
 };
 
@@ -1852,14 +1840,14 @@ ExpressionList2: Vec<crate::parser::ParenthesizedExpr> = {
 // - a single expression
 // - a single expression followed by a trailing comma
 #[inline]
-TestList: crate::parser::ParenthesizedExpr = {
+TestList: ParenthesizedExpr = {
     GenericList<TestOrStarExpr>
 };
 
-GenericList<Element>: crate::parser::ParenthesizedExpr = {
+GenericList<Element>: ParenthesizedExpr = {
     <location:@L> <elts:OneOrMore<Element>> <trailing_comma:","?> <end_location:@R> => {
         if elts.len() == 1 && trailing_comma.is_none() {
-            crate::parser::ParenthesizedExpr {
+            ParenthesizedExpr {
               expr: elts.into_iter().next().unwrap().into(),
               range: (location..end_location).into(),
             }
@@ -1876,7 +1864,7 @@ GenericList<Element>: crate::parser::ParenthesizedExpr = {
 }
 
 // Test
-StarExpr: crate::parser::ParenthesizedExpr = {
+StarExpr: ParenthesizedExpr = {
     <location:@L> "*" <value:Expression<"all">> <end_location:@R> => ast::ExprStarred {
         value: Box::new(value.into()),
         ctx: ast::ExprContext::Load,
@@ -1901,8 +1889,8 @@ SingleForComprehension: ast::Comprehension = {
     }
 };
 
-ExpressionNoCond: crate::parser::ParenthesizedExpr = OrTest<"all">;
-ComprehensionIf: crate::parser::ParenthesizedExpr = "if" <c:ExpressionNoCond> => c;
+ExpressionNoCond: ParenthesizedExpr = OrTest<"all">;
+ComprehensionIf: ParenthesizedExpr = "if" <c:ExpressionNoCond> => c;
 
 Arguments: ast::Arguments = {
     <location:@L> "(" <e: Comma<FunctionArgument>> ")" <end_location:@R> =>? {
@@ -2048,8 +2036,6 @@ extern {
         ">" => token::Tok::Greater,
         ">=" => token::Tok::GreaterEqual,
         "->" => token::Tok::Rarrow,
-        "++" => token::Tok::Increment,
-        "--" => token::Tok::Decrement,
         "and" => token::Tok::And,
         "as" => token::Tok::As,
         "assert" => token::Tok::Assert,

crates/ruff_python_parser/src/lalrpop/snapshots/ruff_python_parser__lalrpop__context__tests__ann_assign_name.snap

@@ -1,5 +1,5 @@
 ---
-source: crates/ruff_python_parser/src/context.rs
+source: crates/ruff_python_parser/src/lalrpop/context.rs
 expression: parse_ast
 ---
 [

crates/ruff_python_parser/src/lalrpop/snapshots/ruff_python_parser__lalrpop__context__tests__assign_attribute.snap

@@ -1,5 +1,5 @@
 ---
-source: crates/ruff_python_parser/src/context.rs
+source: crates/ruff_python_parser/src/lalrpop/context.rs
 expression: parse_ast
 ---
 [

crates/ruff_python_parser/src/lalrpop/snapshots/ruff_python_parser__lalrpop__context__tests__assign_for.snap

@@ -1,5 +1,5 @@
 ---
-source: crates/ruff_python_parser/src/context.rs
+source: crates/ruff_python_parser/src/lalrpop/context.rs
 expression: parse_ast
 ---
 [

crates/ruff_python_parser/src/lalrpop/snapshots/ruff_python_parser__lalrpop__context__tests__assign_list.snap

@@ -1,5 +1,5 @@
 ---
-source: crates/ruff_python_parser/src/context.rs
+source: crates/ruff_python_parser/src/lalrpop/context.rs
 expression: parse_ast
 ---
 [

crates/ruff_python_parser/src/lalrpop/snapshots/ruff_python_parser__lalrpop__context__tests__assign_list_comp.snap

@@ -1,5 +1,5 @@
 ---
-source: crates/ruff_python_parser/src/context.rs
+source: crates/ruff_python_parser/src/lalrpop/context.rs
 expression: parse_ast
 ---
 [

crates/ruff_python_parser/src/lalrpop/snapshots/ruff_python_parser__lalrpop__context__tests__assign_name.snap

@@ -1,5 +1,5 @@
 ---
-source: crates/ruff_python_parser/src/context.rs
+source: crates/ruff_python_parser/src/lalrpop/context.rs
 expression: parse_ast
 ---
 [

crates/ruff_python_parser/src/lalrpop/snapshots/ruff_python_parser__lalrpop__context__tests__assign_named_expr.snap

@@ -1,5 +1,5 @@
 ---
-source: crates/ruff_python_parser/src/context.rs
+source: crates/ruff_python_parser/src/lalrpop/context.rs
 expression: parse_ast
 ---
 [

crates/ruff_python_parser/src/lalrpop/snapshots/ruff_python_parser__lalrpop__context__tests__assign_set_comp.snap

@@ -1,5 +1,5 @@
 ---
-source: crates/ruff_python_parser/src/context.rs
+source: crates/ruff_python_parser/src/lalrpop/context.rs
 expression: parse_ast
 ---
 [

crates/ruff_python_parser/src/lalrpop/snapshots/ruff_python_parser__lalrpop__context__tests__assign_starred.snap

@@ -1,5 +1,5 @@
 ---
-source: crates/ruff_python_parser/src/context.rs
+source: crates/ruff_python_parser/src/lalrpop/context.rs
 expression: parse_ast
 ---
 [