Assert the parser is at augmented assign token (#10269)

## Summary

This PR updates fixes one of the `FIXME` comment to assert that the
parser is at one of the possible augmented assignment token when parsing
an augmented assignment statement.

## Test Plan

1. Add valid test cases for all the possible augmented assignment tokens
2. Add invalid test cases similar to assignment statement
This commit is contained in:
Dhruv Manilawala
2024-03-07 18:37:09 +05:30
committed by GitHub
parent b5cc384bb1
commit 035ac75fae
8 changed files with 2032 additions and 151 deletions

View File

@@ -27,17 +27,6 @@ await x = 42
a < b < c = 42
foo() = 42
# N.B. It looks like the parser can't generate a top-level
# FormattedValue, where as the official Python AST permits
# representing a single f-string containing just a variable as a
# FormattedValue directly.
#
# Bottom line is that because of this, this test is (at present)
# duplicative with the `fstring` test. That is, in theory these tests
# could fail independently, but in practice their failure or success
# is coupled.
#
# See: https://docs.python.org/3/library/ast.html#ast.FormattedValue
f"{quux}" = 42
f"{foo} and {bar}" = 42

View File

@@ -0,0 +1,34 @@
# This is similar to `./invalid_assignment_targets.py`, but for augmented
# assignment targets.
x or y += 42
(x := 5) += 42
x + y += 42
-x += 42
(lambda _: 1) += 42
a if b else c += 42
{"a": 5} += 42
{a} += 42
[x for x in xs] += 42
{x for x in xs} += 42
{x: x * 2 for x in xs} += 42
(x for x in xs) += 42
await x += 42
(yield x) += 42
(yield from xs) += 42
a < b < c += 42
foo() += 42
f"{quux}" += 42
f"{foo} and {bar}" += 42
"foo" += 42
b"foo" += 42
123 += 42
True += 42
None += 42
... += 42
*foo() += 42
[x, foo(), y] += [42, 42, 42]
[[a, b], [[42]], d] += [[1, 2], [[3]], 4]
(x, foo(), y) += (42, 42, 42)

View File

@@ -1,3 +1,18 @@
x += 1
x.y += (1, 2, 3)
x[y] += (1, 2, 3)
# All possible augmented assignment tokens
x += 1
x -= 1
x *= 1
x /= 1
x //= 1
x %= 1
x **= 1
x &= 1
x |= 1
x ^= 1
x <<= 1
x >>= 1
x @= 1

View File

@@ -347,6 +347,21 @@ impl<'src> Parser<'src> {
self.next_token()
}
/// Bumps the current token assuming it is found in the given token set.
///
/// # Panics
///
/// If the current token is not found in the given token set.
///
/// # Returns
///
/// The current token.
fn bump_ts(&mut self, ts: TokenSet) -> (Tok, TextRange) {
assert!(ts.contains(self.current_kind()));
self.next_token()
}
fn expect(&mut self, expected: TokenKind) -> bool {
if self.eat(expected) {
return true;

View File

@@ -52,6 +52,23 @@ const SIMPLE_STMT_SET2: TokenSet = SIMPLE_STMT_SET.union(EXPR_SET);
const STMTS_SET: TokenSet = SIMPLE_STMT_SET2.union(COMPOUND_STMT_SET);
/// Tokens that represent operators that can be used in augmented assignments.
const AUGMENTED_ASSIGN_SET: TokenSet = TokenSet::new([
TokenKind::PlusEqual,
TokenKind::MinusEqual,
TokenKind::StarEqual,
TokenKind::DoubleStarEqual,
TokenKind::SlashEqual,
TokenKind::DoubleSlashEqual,
TokenKind::PercentEqual,
TokenKind::AtEqual,
TokenKind::AmperEqual,
TokenKind::VbarEqual,
TokenKind::CircumflexEqual,
TokenKind::LeftShiftEqual,
TokenKind::RightShiftEqual,
]);
impl<'src> Parser<'src> {
fn at_compound_stmt(&self) -> bool {
self.at_ts(COMPOUND_STMT_SET)
@@ -603,6 +620,12 @@ impl<'src> Parser<'src> {
}
}
/// Parses an augmented assignment statement.
///
/// # Panics
///
/// If the parser isn't positioned at an augmented assignment token.
///
/// See: <https://docs.python.org/3/reference/simple_stmts.html#grammar-token-python-grammar-augmented_assignment_stmt>
fn parse_augmented_assignment_statement(
&mut self,
@@ -611,8 +634,7 @@ impl<'src> Parser<'src> {
start: TextSize,
) -> ast::StmtAugAssign {
// Consume the operator
// FIXME(micha): assert that it is an augmented assign token
self.next_token();
self.bump_ts(AUGMENTED_ASSIGN_SET);
if !helpers::is_valid_aug_assignment_target(&target.expr) {
self.add_error(ParseErrorType::AugAssignmentError, target.range());

View File

@@ -7,7 +7,7 @@ input_file: crates/ruff_python_parser/resources/invalid/statements/invalid_assig
```
Module(
ModModule {
range: 0..1289,
range: 0..788,
body: [
Assign(
StmtAssign {
@@ -767,23 +767,23 @@ Module(
),
Assign(
StmtAssign {
range: 1077..1091,
range: 576..590,
targets: [
FString(
ExprFString {
range: 1077..1086,
range: 576..585,
value: FStringValue {
inner: Single(
FString(
FString {
range: 1077..1086,
range: 576..585,
elements: [
Expression(
FStringExpressionElement {
range: 1079..1085,
range: 578..584,
expression: Name(
ExprName {
range: 1080..1084,
range: 579..583,
id: "quux",
ctx: Load,
},
@@ -803,7 +803,7 @@ Module(
],
value: NumberLiteral(
ExprNumberLiteral {
range: 1089..1091,
range: 588..590,
value: Int(
42,
),
@@ -813,23 +813,23 @@ Module(
),
Assign(
StmtAssign {
range: 1092..1115,
range: 591..614,
targets: [
FString(
ExprFString {
range: 1092..1110,
range: 591..609,
value: FStringValue {
inner: Single(
FString(
FString {
range: 1092..1110,
range: 591..609,
elements: [
Expression(
FStringExpressionElement {
range: 1094..1099,
range: 593..598,
expression: Name(
ExprName {
range: 1095..1098,
range: 594..597,
id: "foo",
ctx: Load,
},
@@ -841,16 +841,16 @@ Module(
),
Literal(
FStringLiteralElement {
range: 1099..1104,
range: 598..603,
value: " and ",
},
),
Expression(
FStringExpressionElement {
range: 1104..1109,
range: 603..608,
expression: Name(
ExprName {
range: 1105..1108,
range: 604..607,
id: "bar",
ctx: Load,
},
@@ -870,7 +870,7 @@ Module(
],
value: NumberLiteral(
ExprNumberLiteral {
range: 1113..1115,
range: 612..614,
value: Int(
42,
),
@@ -880,15 +880,15 @@ Module(
),
Assign(
StmtAssign {
range: 1117..1127,
range: 616..626,
targets: [
StringLiteral(
ExprStringLiteral {
range: 1117..1122,
range: 616..621,
value: StringLiteralValue {
inner: Single(
StringLiteral {
range: 1117..1122,
range: 616..621,
value: "foo",
unicode: false,
},
@@ -899,7 +899,7 @@ Module(
],
value: NumberLiteral(
ExprNumberLiteral {
range: 1125..1127,
range: 624..626,
value: Int(
42,
),
@@ -909,15 +909,15 @@ Module(
),
Assign(
StmtAssign {
range: 1128..1139,
range: 627..638,
targets: [
BytesLiteral(
ExprBytesLiteral {
range: 1128..1134,
range: 627..633,
value: BytesLiteralValue {
inner: Single(
BytesLiteral {
range: 1128..1134,
range: 627..633,
value: [
102,
111,
@@ -931,7 +931,7 @@ Module(
],
value: NumberLiteral(
ExprNumberLiteral {
range: 1137..1139,
range: 636..638,
value: Int(
42,
),
@@ -941,11 +941,11 @@ Module(
),
Assign(
StmtAssign {
range: 1140..1148,
range: 639..647,
targets: [
NumberLiteral(
ExprNumberLiteral {
range: 1140..1143,
range: 639..642,
value: Int(
123,
),
@@ -954,7 +954,7 @@ Module(
],
value: NumberLiteral(
ExprNumberLiteral {
range: 1146..1148,
range: 645..647,
value: Int(
42,
),
@@ -964,18 +964,18 @@ Module(
),
Assign(
StmtAssign {
range: 1149..1158,
range: 648..657,
targets: [
BooleanLiteral(
ExprBooleanLiteral {
range: 1149..1153,
range: 648..652,
value: true,
},
),
],
value: NumberLiteral(
ExprNumberLiteral {
range: 1156..1158,
range: 655..657,
value: Int(
42,
),
@@ -985,17 +985,17 @@ Module(
),
Assign(
StmtAssign {
range: 1159..1168,
range: 658..667,
targets: [
NoneLiteral(
ExprNoneLiteral {
range: 1159..1163,
range: 658..662,
},
),
],
value: NumberLiteral(
ExprNumberLiteral {
range: 1166..1168,
range: 665..667,
value: Int(
42,
),
@@ -1005,17 +1005,17 @@ Module(
),
Assign(
StmtAssign {
range: 1169..1177,
range: 668..676,
targets: [
EllipsisLiteral(
ExprEllipsisLiteral {
range: 1169..1172,
range: 668..671,
},
),
],
value: NumberLiteral(
ExprNumberLiteral {
range: 1175..1177,
range: 674..676,
value: Int(
42,
),
@@ -1025,23 +1025,23 @@ Module(
),
Assign(
StmtAssign {
range: 1178..1189,
range: 677..688,
targets: [
Starred(
ExprStarred {
range: 1178..1184,
range: 677..683,
value: Call(
ExprCall {
range: 1179..1184,
range: 678..683,
func: Name(
ExprName {
range: 1179..1182,
range: 678..681,
id: "foo",
ctx: Load,
},
),
arguments: Arguments {
range: 1182..1184,
range: 681..683,
args: [],
keywords: [],
},
@@ -1053,7 +1053,7 @@ Module(
],
value: NumberLiteral(
ExprNumberLiteral {
range: 1187..1189,
range: 686..688,
value: Int(
42,
),
@@ -1063,31 +1063,31 @@ Module(
),
Assign(
StmtAssign {
range: 1190..1218,
range: 689..717,
targets: [
List(
ExprList {
range: 1190..1203,
range: 689..702,
elts: [
Name(
ExprName {
range: 1191..1192,
range: 690..691,
id: "x",
ctx: Store,
},
),
Call(
ExprCall {
range: 1194..1199,
range: 693..698,
func: Name(
ExprName {
range: 1194..1197,
range: 693..696,
id: "foo",
ctx: Load,
},
),
arguments: Arguments {
range: 1197..1199,
range: 696..698,
args: [],
keywords: [],
},
@@ -1095,7 +1095,7 @@ Module(
),
Name(
ExprName {
range: 1201..1202,
range: 700..701,
id: "y",
ctx: Store,
},
@@ -1107,11 +1107,11 @@ Module(
],
value: List(
ExprList {
range: 1206..1218,
range: 705..717,
elts: [
NumberLiteral(
ExprNumberLiteral {
range: 1207..1209,
range: 706..708,
value: Int(
42,
),
@@ -1119,7 +1119,7 @@ Module(
),
NumberLiteral(
ExprNumberLiteral {
range: 1211..1213,
range: 710..712,
value: Int(
42,
),
@@ -1127,7 +1127,7 @@ Module(
),
NumberLiteral(
ExprNumberLiteral {
range: 1215..1217,
range: 714..716,
value: Int(
42,
),
@@ -1141,26 +1141,26 @@ Module(
),
Assign(
StmtAssign {
range: 1219..1259,
range: 718..758,
targets: [
List(
ExprList {
range: 1219..1238,
range: 718..737,
elts: [
List(
ExprList {
range: 1220..1226,
range: 719..725,
elts: [
Name(
ExprName {
range: 1221..1222,
range: 720..721,
id: "a",
ctx: Store,
},
),
Name(
ExprName {
range: 1224..1225,
range: 723..724,
id: "b",
ctx: Store,
},
@@ -1171,15 +1171,15 @@ Module(
),
List(
ExprList {
range: 1228..1234,
range: 727..733,
elts: [
List(
ExprList {
range: 1229..1233,
range: 728..732,
elts: [
NumberLiteral(
ExprNumberLiteral {
range: 1230..1232,
range: 729..731,
value: Int(
42,
),
@@ -1195,7 +1195,7 @@ Module(
),
Name(
ExprName {
range: 1236..1237,
range: 735..736,
id: "d",
ctx: Store,
},
@@ -1207,15 +1207,15 @@ Module(
],
value: List(
ExprList {
range: 1241..1259,
range: 740..758,
elts: [
List(
ExprList {
range: 1242..1248,
range: 741..747,
elts: [
NumberLiteral(
ExprNumberLiteral {
range: 1243..1244,
range: 742..743,
value: Int(
1,
),
@@ -1223,7 +1223,7 @@ Module(
),
NumberLiteral(
ExprNumberLiteral {
range: 1246..1247,
range: 745..746,
value: Int(
2,
),
@@ -1235,15 +1235,15 @@ Module(
),
List(
ExprList {
range: 1250..1255,
range: 749..754,
elts: [
List(
ExprList {
range: 1251..1254,
range: 750..753,
elts: [
NumberLiteral(
ExprNumberLiteral {
range: 1252..1253,
range: 751..752,
value: Int(
3,
),
@@ -1259,7 +1259,7 @@ Module(
),
NumberLiteral(
ExprNumberLiteral {
range: 1257..1258,
range: 756..757,
value: Int(
4,
),
@@ -1273,31 +1273,31 @@ Module(
),
Assign(
StmtAssign {
range: 1260..1288,
range: 759..787,
targets: [
Tuple(
ExprTuple {
range: 1260..1273,
range: 759..772,
elts: [
Name(
ExprName {
range: 1261..1262,
range: 760..761,
id: "x",
ctx: Store,
},
),
Call(
ExprCall {
range: 1264..1269,
range: 763..768,
func: Name(
ExprName {
range: 1264..1267,
range: 763..766,
id: "foo",
ctx: Load,
},
),
arguments: Arguments {
range: 1267..1269,
range: 766..768,
args: [],
keywords: [],
},
@@ -1305,7 +1305,7 @@ Module(
),
Name(
ExprName {
range: 1271..1272,
range: 770..771,
id: "y",
ctx: Store,
},
@@ -1318,11 +1318,11 @@ Module(
],
value: Tuple(
ExprTuple {
range: 1276..1288,
range: 775..787,
elts: [
NumberLiteral(
ExprNumberLiteral {
range: 1277..1279,
range: 776..778,
value: Int(
42,
),
@@ -1330,7 +1330,7 @@ Module(
),
NumberLiteral(
ExprNumberLiteral {
range: 1281..1283,
range: 780..782,
value: Int(
42,
),
@@ -1338,7 +1338,7 @@ Module(
),
NumberLiteral(
ExprNumberLiteral {
range: 1285..1287,
range: 784..786,
value: Int(
42,
),
@@ -1551,120 +1551,119 @@ Module(
28 | foo() = 42
| ^^^^^ Syntax Error: invalid assignment target
29 |
30 | # N.B. It looks like the parser can't generate a top-level
30 | f"{quux}" = 42
|
|
39 | #
40 | # See: https://docs.python.org/3/library/ast.html#ast.FormattedValue
41 | f"{quux}" = 42
28 | foo() = 42
29 |
30 | f"{quux}" = 42
| ^^^^^^^^^ Syntax Error: invalid assignment target
42 | f"{foo} and {bar}" = 42
31 | f"{foo} and {bar}" = 42
|
|
40 | # See: https://docs.python.org/3/library/ast.html#ast.FormattedValue
41 | f"{quux}" = 42
42 | f"{foo} and {bar}" = 42
30 | f"{quux}" = 42
31 | f"{foo} and {bar}" = 42
| ^^^^^^^^^^^^^^^^^^ Syntax Error: invalid assignment target
43 |
44 | "foo" = 42
32 |
33 | "foo" = 42
|
|
42 | f"{foo} and {bar}" = 42
43 |
44 | "foo" = 42
31 | f"{foo} and {bar}" = 42
32 |
33 | "foo" = 42
| ^^^^^ Syntax Error: invalid assignment target
45 | b"foo" = 42
46 | 123 = 42
34 | b"foo" = 42
35 | 123 = 42
|
|
44 | "foo" = 42
45 | b"foo" = 42
33 | "foo" = 42
34 | b"foo" = 42
| ^^^^^^ Syntax Error: invalid assignment target
46 | 123 = 42
47 | True = 42
35 | 123 = 42
36 | True = 42
|
|
44 | "foo" = 42
45 | b"foo" = 42
46 | 123 = 42
33 | "foo" = 42
34 | b"foo" = 42
35 | 123 = 42
| ^^^ Syntax Error: invalid assignment target
47 | True = 42
48 | None = 42
36 | True = 42
37 | None = 42
|
|
45 | b"foo" = 42
46 | 123 = 42
47 | True = 42
34 | b"foo" = 42
35 | 123 = 42
36 | True = 42
| ^^^^ Syntax Error: invalid assignment target
48 | None = 42
49 | ... = 42
37 | None = 42
38 | ... = 42
|
|
46 | 123 = 42
47 | True = 42
48 | None = 42
35 | 123 = 42
36 | True = 42
37 | None = 42
| ^^^^ Syntax Error: invalid assignment target
49 | ... = 42
50 | *foo() = 42
38 | ... = 42
39 | *foo() = 42
|
|
47 | True = 42
48 | None = 42
49 | ... = 42
36 | True = 42
37 | None = 42
38 | ... = 42
| ^^^ Syntax Error: invalid assignment target
50 | *foo() = 42
51 | [x, foo(), y] = [42, 42, 42]
39 | *foo() = 42
40 | [x, foo(), y] = [42, 42, 42]
|
|
48 | None = 42
49 | ... = 42
50 | *foo() = 42
37 | None = 42
38 | ... = 42
39 | *foo() = 42
| ^^^^^^ Syntax Error: invalid assignment target
51 | [x, foo(), y] = [42, 42, 42]
52 | [[a, b], [[42]], d] = [[1, 2], [[3]], 4]
40 | [x, foo(), y] = [42, 42, 42]
41 | [[a, b], [[42]], d] = [[1, 2], [[3]], 4]
|
|
49 | ... = 42
50 | *foo() = 42
51 | [x, foo(), y] = [42, 42, 42]
38 | ... = 42
39 | *foo() = 42
40 | [x, foo(), y] = [42, 42, 42]
| ^^^^^^^^^^^^^ Syntax Error: invalid assignment target
52 | [[a, b], [[42]], d] = [[1, 2], [[3]], 4]
53 | (x, foo(), y) = (42, 42, 42)
41 | [[a, b], [[42]], d] = [[1, 2], [[3]], 4]
42 | (x, foo(), y) = (42, 42, 42)
|
|
50 | *foo() = 42
51 | [x, foo(), y] = [42, 42, 42]
52 | [[a, b], [[42]], d] = [[1, 2], [[3]], 4]
39 | *foo() = 42
40 | [x, foo(), y] = [42, 42, 42]
41 | [[a, b], [[42]], d] = [[1, 2], [[3]], 4]
| ^^^^^^^^^^^^^^^^^^^ Syntax Error: invalid assignment target
53 | (x, foo(), y) = (42, 42, 42)
42 | (x, foo(), y) = (42, 42, 42)
|
|
51 | [x, foo(), y] = [42, 42, 42]
52 | [[a, b], [[42]], d] = [[1, 2], [[3]], 4]
53 | (x, foo(), y) = (42, 42, 42)
40 | [x, foo(), y] = [42, 42, 42]
41 | [[a, b], [[42]], d] = [[1, 2], [[3]], 4]
42 | (x, foo(), y) = (42, 42, 42)
| ^^^^^^^^^^^^^ Syntax Error: invalid assignment target
|

View File

@@ -7,7 +7,7 @@ input_file: crates/ruff_python_parser/resources/valid/statement/augmented_assign
```
Module(
ModModule {
range: 0..42,
range: 0..181,
body: [
AugAssign(
StmtAugAssign {
@@ -145,6 +145,279 @@ Module(
),
},
),
AugAssign(
StmtAugAssign {
range: 86..92,
target: Name(
ExprName {
range: 86..87,
id: "x",
ctx: Store,
},
),
op: Add,
value: NumberLiteral(
ExprNumberLiteral {
range: 91..92,
value: Int(
1,
),
},
),
},
),
AugAssign(
StmtAugAssign {
range: 93..99,
target: Name(
ExprName {
range: 93..94,
id: "x",
ctx: Store,
},
),
op: Sub,
value: NumberLiteral(
ExprNumberLiteral {
range: 98..99,
value: Int(
1,
),
},
),
},
),
AugAssign(
StmtAugAssign {
range: 100..106,
target: Name(
ExprName {
range: 100..101,
id: "x",
ctx: Store,
},
),
op: Mult,
value: NumberLiteral(
ExprNumberLiteral {
range: 105..106,
value: Int(
1,
),
},
),
},
),
AugAssign(
StmtAugAssign {
range: 107..113,
target: Name(
ExprName {
range: 107..108,
id: "x",
ctx: Store,
},
),
op: Div,
value: NumberLiteral(
ExprNumberLiteral {
range: 112..113,
value: Int(
1,
),
},
),
},
),
AugAssign(
StmtAugAssign {
range: 114..121,
target: Name(
ExprName {
range: 114..115,
id: "x",
ctx: Store,
},
),
op: FloorDiv,
value: NumberLiteral(
ExprNumberLiteral {
range: 120..121,
value: Int(
1,
),
},
),
},
),
AugAssign(
StmtAugAssign {
range: 122..128,
target: Name(
ExprName {
range: 122..123,
id: "x",
ctx: Store,
},
),
op: Mod,
value: NumberLiteral(
ExprNumberLiteral {
range: 127..128,
value: Int(
1,
),
},
),
},
),
AugAssign(
StmtAugAssign {
range: 129..136,
target: Name(
ExprName {
range: 129..130,
id: "x",
ctx: Store,
},
),
op: Pow,
value: NumberLiteral(
ExprNumberLiteral {
range: 135..136,
value: Int(
1,
),
},
),
},
),
AugAssign(
StmtAugAssign {
range: 137..143,
target: Name(
ExprName {
range: 137..138,
id: "x",
ctx: Store,
},
),
op: BitAnd,
value: NumberLiteral(
ExprNumberLiteral {
range: 142..143,
value: Int(
1,
),
},
),
},
),
AugAssign(
StmtAugAssign {
range: 144..150,
target: Name(
ExprName {
range: 144..145,
id: "x",
ctx: Store,
},
),
op: BitOr,
value: NumberLiteral(
ExprNumberLiteral {
range: 149..150,
value: Int(
1,
),
},
),
},
),
AugAssign(
StmtAugAssign {
range: 151..157,
target: Name(
ExprName {
range: 151..152,
id: "x",
ctx: Store,
},
),
op: BitXor,
value: NumberLiteral(
ExprNumberLiteral {
range: 156..157,
value: Int(
1,
),
},
),
},
),
AugAssign(
StmtAugAssign {
range: 158..165,
target: Name(
ExprName {
range: 158..159,
id: "x",
ctx: Store,
},
),
op: LShift,
value: NumberLiteral(
ExprNumberLiteral {
range: 164..165,
value: Int(
1,
),
},
),
},
),
AugAssign(
StmtAugAssign {
range: 166..173,
target: Name(
ExprName {
range: 166..167,
id: "x",
ctx: Store,
},
),
op: RShift,
value: NumberLiteral(
ExprNumberLiteral {
range: 172..173,
value: Int(
1,
),
},
),
},
),
AugAssign(
StmtAugAssign {
range: 174..180,
target: Name(
ExprName {
range: 174..175,
id: "x",
ctx: Store,
},
),
op: MatMult,
value: NumberLiteral(
ExprNumberLiteral {
range: 179..180,
value: Int(
1,
),
},
),
},
),
],
},
)