Replace LALRPOP parser with hand-written parser

Co-authored-by: Micha Reiser <micha@reiser.io>

crates/ruff_python_parser/resources/.editorconfig

@@ -0,0 +1,6 @@
+# Check http://editorconfig.org for more information
+# This is the main config file for this project:
+root = true
+
+[*.py]
+insert_final_newline = false

crates/ruff_python_parser/resources/invalid/expressions/call_invalid_argument_ordering.py

@@ -0,0 +1,6 @@
+f(b=20, c)
+
+f(**b, *c)
+
+# Duplicate keyword argument
+f(a=20, a=30)

crates/ruff_python_parser/resources/invalid/expressions/emoji_identifiers.py

@@ -0,0 +1,7 @@
+a = (🐶
+    # comment 🐶
+)
+
+a = (🐶 +
+    # comment
+🐶)

crates/ruff_python_parser/resources/invalid/expressions/emoji_statement.py

@@ -0,0 +1 @@
+👍

crates/ruff_python_parser/resources/invalid/expressions/lambda_default_parameters.py

@@ -0,0 +1,2 @@
+# TODO(micha): The offset of the generated error message is off by one.
+lambda a, b=20, c: 1

crates/ruff_python_parser/resources/invalid/expressions/lambda_duplicate_parameters.py

@@ -0,0 +1,9 @@
+lambda a, a: 1
+
+lambda a, *, a: 1
+
+lambda a, a=20: 1
+
+lambda a, *a: 1
+
+lambda a, *, **a: 1

crates/ruff_python_parser/resources/invalid/statements/assignment_keyword_target.py

@@ -0,0 +1,7 @@
+a = pass = c
+
+a + b
+
+a = b = pass = c
+
+a + b

crates/ruff_python_parser/resources/invalid/statements/assignment_missing_target.py

@@ -0,0 +1,7 @@
+a = = c
+
+a + b
+
+
+
+

crates/ruff_python_parser/resources/invalid/statements/assignment_missing_value.py

@@ -0,0 +1,3 @@
+a =
+
+a + b

crates/ruff_python_parser/resources/invalid/statements/function_default_parameters.py

@@ -0,0 +1,2 @@
+# TODO(micha): The range of the generated error message is off by one.
+def f(a, b=20, c): pass

crates/ruff_python_parser/resources/invalid/statements/function_duplicate_parameters.py

@@ -0,0 +1,12 @@
+def f(a, a): pass
+
+def f2(a, *, a): pass
+
+def f3(a, a=20): pass
+
+def f4(a, *a): pass
+
+def f5(a, *, **a): pass
+
+# TODO(micha): This is inconsistent. All other examples only highlight the argument name.
+def f6(a, a: str): pass

crates/ruff_python_parser/resources/invalid/statements/function_type_parameters.py

@@ -0,0 +1,24 @@
+# FIXME: The type param related error message and the parser recovery are looking pretty good **except**
+# that the lexer never recovers from the unclosed `[`, resulting in it lexing `NonLogicalNewline` tokens instead of `Newline` tokens.
+# That's because the parser has no way of feeding the error recovery back to the lexer,
+# so they don't agree on the state of the world which can lead to all kind of errors further down in the file.
+# This is not just a problem with parentheses but also with the transformation made by the
+# `SoftKeywordTransformer` because the `Parser` and `Transfomer` may not agree if they're
+# currently in a position where the `type` keyword is allowed or not.
+# That roughly means that any kind of recovery can lead to unrelated syntax errors
+# on following lines.
+
+def unclosed[A, *B(test: name):
+    pass
+
+    a + b
+
+def keyword[A, await](): ...
+
+def not_a_type_param[A, |, B](): ...
+
+def multiple_commas[A,,B](): ...
+
+def multiple_trailing_commas[A,,](): ...
+
+def multiple_commas_and_recovery[A,,100](): ...

crates/ruff_python_parser/resources/invalid/statements/if_elif.py

@@ -0,0 +1,7 @@
+if True:
+    pass
+elif False:
+    pass
+elf:
+    pass
+

crates/ruff_python_parser/resources/invalid/statements/if_extra_closing_parentheses.py

@@ -0,0 +1,3 @@
+# FIXME(micha): This creates two syntax errors instead of just one (and overlapping ones)
+if True)):
+    pass

crates/ruff_python_parser/resources/invalid/statements/if_extra_indent.py

@@ -0,0 +1,8 @@
+# Improving the recovery would require changing the lexer to emit an extra dedent token after `a + b`.
+if True:
+    pass
+        a + b
+
+    pass
+
+a = 10

crates/ruff_python_parser/resources/invalid/statements/if_missing_body.py

@@ -0,0 +1,6 @@
+if True:
+
+
+a + b
+
+if False: # This if statement has neither an indent nor a newline.

crates/ruff_python_parser/resources/invalid/statements/if_missing_colon.py

@@ -0,0 +1,4 @@
+if True
+    pass
+
+a = 10

crates/ruff_python_parser/resources/invalid/statements/import_from.py

@@ -0,0 +1,11 @@
+from abc import a, b,
+
+a + b
+
+from abc import ,,
+
+from abc import
+
+from abc import (a, b, c
+
+a + b

crates/ruff_python_parser/resources/invalid/statements/invalid_assignment_targets.py

@@ -0,0 +1,53 @@
+# Regression test: https://github.com/astral-sh/ruff/issues/6895
+# First we test, broadly, that various kinds of assignments are now
+# rejected by the parser. e.g., `5 = 3`, `5 += 3`, `(5): int = 3`.
+
+5 = 3
+
+5 += 3
+
+(5): int = 3
+
+# Now we exhaustively test all possible cases where assignment can fail.
+x or y = 42
+(x := 5) = 42
+x + y = 42
+-x = 42
+(lambda _: 1) = 42
+a if b else c = 42
+{"a": 5} = 42
+{a} = 42
+[x for x in xs] = 42
+{x for x in xs} = 42
+{x: x * 2 for x in xs} = 42
+(x for x in xs) = 42
+await x = 42
+(yield x) = 42
+(yield from xs) = 42
+a < b < c = 42
+foo() = 42
+
+# N.B. It looks like the parser can't generate a top-level
+# FormattedValue, where as the official Python AST permits
+# representing a single f-string containing just a variable as a
+# FormattedValue directly.
+#
+# Bottom line is that because of this, this test is (at present)
+# duplicative with the `fstring` test. That is, in theory these tests
+# could fail independently, but in practice their failure or success
+# is coupled.
+#
+# See: https://docs.python.org/3/library/ast.html#ast.FormattedValue
+f"{quux}" = 42
+f"{foo} and {bar}" = 42
+
+"foo" = 42
+b"foo" = 42
+123 = 42
+True = 42
+None = 42
+... = 42
+*foo() = 42
+[x, foo(), y] = [42, 42, 42]
+[[a, b], [[42]], d] = [[1, 2], [[3]], 4]
+(x, foo(), y) = (42, 42, 42)

crates/ruff_python_parser/resources/invalid/statements/named_expression_statement.py

@@ -0,0 +1,3 @@
+# This test previously passed before the assignment operator checking
+# above, but we include it here for good measure.
+(5 := 3)

crates/ruff_python_parser/resources/valid/expressions/dictionary_comprehension.py

@@ -0,0 +1 @@
+x = {y for y in (1, 2, 3)}

crates/ruff_python_parser/resources/valid/expressions/lambda.py

@@ -0,0 +1,11 @@
+lambda: 1
+
+lambda a, b, c: 1
+
+lambda a, b=20, c=30: 1
+
+lambda *, a, b, c: 1
+
+lambda *, a, b=20, c=30: 1
+
+lambda a, b, c, *, d, e: 0

crates/ruff_python_parser/resources/valid/expressions/list_comprehension.py

@@ -0,0 +1 @@
+x = [y for y in (1, 2, 3)]

crates/ruff_python_parser/resources/valid/expressions/named_expression.py

@@ -0,0 +1,4 @@
+if x := 1:
+    pass
+
+(x := 5)

crates/ruff_python_parser/resources/valid/statement/annotated_assignment.py

@@ -0,0 +1 @@
+x: int = 1

crates/ruff_python_parser/resources/valid/statement/assignment.py

@@ -0,0 +1,40 @@
+x = (1, 2, 3)
+
+(x, y) = (1, 2, 3)
+
+[x, y] = (1, 2, 3)
+
+x.y = (1, 2, 3)
+
+x[y] = (1, 2, 3)
+
+(x, *y) = (1, 2, 3)
+
+
+# This last group of tests checks that assignments we expect to be parsed
+# (including some interesting ones) continue to be parsed successfully.
+
+*foo = 42
+
+[x, y, z] = [1, 2, 3]
+
+(x, y, z) = (1, 2, 3)
+x[0] = 42
+
+# This is actually a type error, not a syntax error. So check that it
+# doesn't fail parsing.
+
+5[0] = 42
+x[1:2] = [42]
+
+# This is actually a type error, not a syntax error. So check that it
+# doesn't fail parsing.
+5[1:2] = [42]
+
+foo.bar = 42
+
+# This is actually an attribute error, not a syntax error. So check that
+# it doesn't fail parsing.
+"foo".y = 42
+
+foo = 42

crates/ruff_python_parser/resources/valid/statement/augmented_assignment.py

@@ -0,0 +1,3 @@
+x += 1
+x.y += (1, 2, 3)
+x[y] += (1, 2, 3)

crates/ruff_python_parser/resources/valid/statement/delete.py

@@ -0,0 +1,3 @@
+del x
+del x.y
+del x[y]

crates/ruff_python_parser/resources/valid/statement/for.py

@@ -0,0 +1,2 @@
+for x in (1, 2, 3):
+    pass

crates/ruff_python_parser/resources/valid/statement/function.py

@@ -0,0 +1,38 @@
+def no_parameters():
+    pass
+
+
+def positional_parameters(a, b, c):
+    pass
+
+
+def positional_parameters_with_default_values(a, b=20, c=30):
+    pass
+
+
+def keyword_only_parameters(*, a, b, c):
+    pass
+
+
+def keyword_only_parameters_with_defaults(*, a, b=20, c=30):
+    pass
+
+
+def positional_and_keyword_parameters(a, b, c, *, d, e, f):
+    pass
+
+
+def positional_and_keyword_parameters_with_defaults(a, b, c, *, d, e=20, f=30):
+    pass
+
+
+def positional_and_keyword_parameters_with_defaults_and_varargs(
+    a, b, c, *args, d, e=20, f=30
+):
+    pass
+
+
+def positional_and_keyword_parameters_with_defaults_and_varargs_and_kwargs(
+    a, b, c, *args, d, e=20, f=30, **kwargs
+):
+    pass

crates/ruff_python_parser/resources/valid/statement/with.py

@@ -0,0 +1,2 @@
+with 1 as x:
+    pass