Cyclic0007/ruff
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Move range from Attributed to Nodes (#22)

Browse Source 
* Move `range` from `Attributed` to `Node`s

* No Attributed + custom for Range PoC

* Generate all located variants, generate enum implementations

* Implement `Copy` on simple enums

* Move `Suite` to `ranged` and `located`

* Update tests
---------

Co-authored-by: Jeong YunWon <jeong@youknowone.org>
This commit is contained in:
Micha Reiser
2023-05-15 08:08:12 +02:00
committed by GitHub
parent a983f4383f
commit 192379cede
126 changed files with 29410 additions and 30670 deletions
Download Patch File Download Diff File Expand all files Collapse all files

82
parser/src/string.rs
View File

@@ -3,8 +3,9 @@
// The lexer doesn't do any special handling of f-strings, it just treats them as
// regular strings. Since the parser has no definition of f-string formats (Pending PEP 701)
// we have to do the parsing here, manually.
use crate::text_size::TextRange;
use crate::{
ast::{self, Constant, Expr, ExprKind, Int},
ast::{self, Constant, Expr, Int},
lexer::{LexicalError, LexicalErrorType},
parser::{parse_expression_starts_at, LalrpopError, ParseError, ParseErrorType},
token::{StringKind, Tok},
@@ -67,8 +68,12 @@ impl<'a> StringParser<'a> {
}
#[inline]
fn expr(&self, node: ExprKind) -> Expr {
Expr::new(self.start..self.end, node)
fn expr(&self, node: Expr) -> Expr {
node
}
fn range(&self) -> TextRange {
TextRange::new(self.start, self.end)
}
fn parse_unicode_literal(&mut self, literal_number: usize) -> Result<char, LexicalError> {
@@ -237,6 +242,7 @@ impl<'a> StringParser<'a> {
self.expr(
ast::ExprJoinedStr {
values: parsed_spec,
range: self.range(),
}
.into(),
),
@@ -316,6 +322,7 @@ impl<'a> StringParser<'a> {
),
conversion: Int::new(conversion as _),
format_spec: spec,
range: self.range(),
}
.into(),
)]
@@ -325,6 +332,7 @@ impl<'a> StringParser<'a> {
ast::ExprConstant {
value: Constant::Str(expression.to_owned() + "="),
kind: None,
range: self.range(),
}
.into(),
),
@@ -332,6 +340,7 @@ impl<'a> StringParser<'a> {
ast::ExprConstant {
value: trailing_seq.into(),
kind: None,
range: self.range(),
}
.into(),
),
@@ -353,6 +362,7 @@ impl<'a> StringParser<'a> {
}) as _,
),
format_spec: spec,
range: self.range(),
}
.into(),
),
@@ -400,6 +410,7 @@ impl<'a> StringParser<'a> {
ast::ExprConstant {
value: constant_piece.drain(..).collect::<String>().into(),
kind: None,
range: self.range(),
}
.into(),
),
@@ -424,6 +435,7 @@ impl<'a> StringParser<'a> {
ast::ExprConstant {
value: constant_piece.drain(..).collect::<String>().into(),
kind: None,
range: self.range(),
}
.into(),
),
@@ -465,6 +477,7 @@ impl<'a> StringParser<'a> {
ast::ExprConstant {
value: content.drain(..).collect::<String>().into(),
kind: None,
range: self.range(),
}
.into(),
),
@@ -503,6 +516,7 @@ impl<'a> StringParser<'a> {
ast::ExprConstant {
value: content.into(),
kind: None,
range: self.range(),
}
.into(),
),
@@ -537,6 +551,7 @@ impl<'a> StringParser<'a> {
ast::ExprConstant {
value: Constant::Bytes(content.chars().map(|c| c as u8).collect()),
kind: None,
range: self.range(),
}
.into(),
))
@@ -556,6 +571,7 @@ impl<'a> StringParser<'a> {
ast::ExprConstant {
value: Constant::Str(content),
kind: self.kind.is_unicode().then(|| "u".to_string()),
range: self.range(),
}
.into(),
))
@@ -615,8 +631,8 @@ pub(crate) fn parse_strings(
let mut content: Vec<u8> = vec![];
for (start, (source, kind, triple_quoted), end) in values {
for value in parse_string(&source, kind, triple_quoted, start, end)? {
match value.into_node() {
ExprKind::Constant(ast::ExprConstant {
match value {
Expr::Constant(ast::ExprConstant {
value: Constant::Bytes(value),
..
}) => content.extend(value),
@@ -624,21 +640,20 @@ pub(crate) fn parse_strings(
}
}
}
return Ok(Expr::new(
initial_start..last_end,
ast::ExprConstant {
value: Constant::Bytes(content),
kind: None,
},
));
return Ok(ast::ExprConstant {
value: Constant::Bytes(content),
kind: None,
range: TextRange::new(initial_start, last_end),
}
.into());
}
if !has_fstring {
let mut content: Vec<String> = vec![];
for (start, (source, kind, triple_quoted), end) in values {
for value in parse_string(&source, kind, triple_quoted, start, end)? {
match value.into_node() {
ExprKind::Constant(ast::ExprConstant {
match value {
Expr::Constant(ast::ExprConstant {
value: Constant::Str(value),
..
}) => content.push(value),
@@ -646,13 +661,12 @@ pub(crate) fn parse_strings(
}
}
}
return Ok(Expr::new(
initial_start..last_end,
ast::ExprConstant {
value: Constant::Str(content.join("")),
kind: initial_kind,
},
));
return Ok(ast::ExprConstant {
value: Constant::Str(content.join("")),
kind: initial_kind,
range: TextRange::new(initial_start, last_end),
}
.into());
}
// De-duplicate adjacent constants.
@@ -660,25 +674,23 @@ pub(crate) fn parse_strings(
let mut current: Vec<String> = vec![];
let take_current = |current: &mut Vec<String>| -> Expr {
Expr::new(
initial_start..last_end,
ast::ExprConstant {
value: Constant::Str(current.drain(..).join("")),
kind: initial_kind.clone(),
},
)
Expr::Constant(ast::ExprConstant {
value: Constant::Str(current.drain(..).join("")),
kind: initial_kind.clone(),
range: TextRange::new(initial_start, last_end),
})
};
for (start, (source, kind, triple_quoted), end) in values {
for value in parse_string(&source, kind, triple_quoted, start, end)? {
match value.node {
ExprKind::FormattedValue { .. } => {
match value {
Expr::FormattedValue { .. } => {
if !current.is_empty() {
deduped.push(take_current(&mut current));
}
deduped.push(value)
}
ExprKind::Constant(ast::ExprConstant {
Expr::Constant(ast::ExprConstant {
value: Constant::Str(value),
..
}) => current.push(value),
@@ -690,10 +702,10 @@ pub(crate) fn parse_strings(
deduped.push(take_current(&mut current));
}
Ok(Expr::new(
initial_start..last_end,
ast::ExprJoinedStr { values: deduped },
))
Ok(Expr::JoinedStr(ast::ExprJoinedStr {
values: deduped,
range: TextRange::new(initial_start, last_end),
}))
}
// TODO: consolidate these with ParseError