Cyclic0007/ruff
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Replace LALRPOP parser with hand-written parser

Browse Source 
Co-authored-by: Micha Reiser <micha@reiser.io>
This commit is contained in:
Victor Hugo Gomes
2024-02-19 09:40:00 +05:30
committed by Dhruv Manilawala
parent 461cdad53a
commit 78ee6441a7
253 changed files with 37848 additions and 5629 deletions
Download Patch File Download Diff File Expand all files Collapse all files

144
crates/ruff_python_parser/src/string.rs
View File

@@ -6,12 +6,14 @@ use ruff_python_ast::{self as ast, Expr};
use ruff_text_size::{Ranged, TextRange, TextSize};
use crate::lexer::{LexicalError, LexicalErrorType};
use crate::token::{StringKind, Tok};
use crate::token::StringKind;
#[derive(Debug)]
pub(crate) enum StringType {
Str(ast::StringLiteral),
Bytes(ast::BytesLiteral),
FString(ast::FString),
Invalid(ast::StringLiteral),
}
impl Ranged for StringType {
@@ -20,6 +22,7 @@ impl Ranged for StringType {
Self::Str(node) => node.range(),
Self::Bytes(node) => node.range(),
Self::FString(node) => node.range(),
Self::Invalid(node) => node.range(),
}
}
}
@@ -30,6 +33,7 @@ impl From<StringType> for Expr {
StringType::Str(node) => Expr::from(node),
StringType::Bytes(node) => Expr::from(node),
StringType::FString(node) => Expr::from(node),
StringType::Invalid(node) => Expr::from(node),
}
}
}
@@ -70,6 +74,11 @@ impl StringParser {
self.offset + TextSize::try_from(self.cursor).unwrap()
}
#[inline]
fn range(&self, start_location: TextSize) -> TextRange {
TextRange::new(dbg!(start_location), dbg!(self.offset))
}
/// Returns the next byte in the string, if there is one.
///
/// # Panics
@@ -98,19 +107,31 @@ impl StringParser {
fn parse_unicode_literal(&mut self, literal_number: usize) -> Result<char, LexicalError> {
let mut p: u32 = 0u32;
let unicode_error = LexicalError::new(LexicalErrorType::UnicodeError, self.get_pos());
for i in 1..=literal_number {
match self.next_char() {
Some(c) => match c.to_digit(16) {
Some(d) => p += d << ((literal_number - i) * 4),
None => return Err(unicode_error),
None => {
return Err(LexicalError::new(
LexicalErrorType::UnicodeError,
TextRange::empty(self.get_pos()),
))
}
},
None => return Err(unicode_error),
None => {
return Err(LexicalError::new(
LexicalErrorType::UnicodeError,
TextRange::empty(self.get_pos()),
))
}
}
}
match p {
0xD800..=0xDFFF => Ok(std::char::REPLACEMENT_CHARACTER),
_ => std::char::from_u32(p).ok_or(unicode_error),
_ => std::char::from_u32(p).ok_or(LexicalError::new(
LexicalErrorType::UnicodeError,
TextRange::empty(self.get_pos()),
)),
}
}
@@ -137,14 +158,17 @@ impl StringParser {
let start_pos = self.get_pos();
let Some('{') = self.next_char() else {
return Err(LexicalError::new(LexicalErrorType::StringError, start_pos));
return Err(LexicalError::new(
LexicalErrorType::MissingUnicodeLbrace,
self.range(start_pos),
));
};
let start_pos = self.get_pos();
let Some(close_idx) = self.source[self.cursor..].find('}') else {
return Err(LexicalError::new(
LexicalErrorType::StringError,
self.get_pos(),
LexicalErrorType::MissingUnicodeRbrace,
self.range(self.get_pos()),
));
};
@@ -152,15 +176,16 @@ impl StringParser {
let name = &name_and_ending[..name_and_ending.len() - 1];
unicode_names2::character(name)
.ok_or_else(|| LexicalError::new(LexicalErrorType::UnicodeError, start_pos))
.ok_or_else(|| LexicalError::new(LexicalErrorType::UnicodeError, self.range(start_pos)))
}
/// Parse an escaped character, returning the new character.
fn parse_escaped_char(&mut self) -> Result<Option<EscapedChar>, LexicalError> {
let Some(first_char) = self.next_char() else {
// TODO: check when this error case happens
return Err(LexicalError::new(
LexicalErrorType::StringError,
self.get_pos(),
self.range(self.get_pos()),
));
};
@@ -197,7 +222,7 @@ impl StringParser {
.to_string()
.into_boxed_str(),
),
self.get_pos(),
self.range(self.get_pos()),
));
}
@@ -268,6 +293,16 @@ impl StringParser {
}
}
ch => {
if !ch.is_ascii() {
return Err(LexicalError::new(
LexicalErrorType::OtherError(
"bytes can only contain ASCII literal characters"
.to_string()
.into_boxed_str(),
),
self.range(self.get_pos()),
));
}
value.push(char::from(*ch));
}
}
@@ -298,7 +333,7 @@ impl StringParser {
.to_string()
.into_boxed_str(),
),
self.offset + TextSize::try_from(index).unwrap(),
self.range(TextSize::try_from(index).unwrap()),
));
}
@@ -461,7 +496,7 @@ pub(crate) fn concatenated_strings(
match string {
StringType::FString(_) => has_fstring = true,
StringType::Bytes(_) => byte_literal_count += 1,
StringType::Str(_) => {}
StringType::Str(_) | StringType::Invalid(_) => {}
}
}
let has_bytes = byte_literal_count > 0;
@@ -473,7 +508,7 @@ pub(crate) fn concatenated_strings(
.to_string()
.into_boxed_str(),
),
range.start(),
range,
));
}
@@ -495,7 +530,7 @@ pub(crate) fn concatenated_strings(
let mut values = Vec::with_capacity(strings.len());
for string in strings {
match string {
StringType::Str(value) => values.push(value),
StringType::Str(value) | StringType::Invalid(value) => values.push(value),
_ => unreachable!("Unexpected non-string literal."),
}
}
@@ -510,6 +545,7 @@ pub(crate) fn concatenated_strings(
match string {
StringType::FString(fstring) => parts.push(ast::FStringPart::FString(fstring)),
StringType::Str(string) => parts.push(ast::FStringPart::Literal(string)),
StringType::Invalid(_) => {}
StringType::Bytes(_) => unreachable!("Unexpected bytes literal."),
}
}
@@ -521,75 +557,10 @@ pub(crate) fn concatenated_strings(
.into())
}
// TODO: consolidate these with ParseError
/// An error that occurred during parsing of an f-string.
#[derive(Debug, Clone, PartialEq)]
struct FStringError {
/// The type of error that occurred.
pub(crate) error: FStringErrorType,
/// The location of the error.
pub(crate) location: TextSize,
}
impl From<FStringError> for LexicalError {
fn from(err: FStringError) -> Self {
LexicalError::new(LexicalErrorType::FStringError(err.error), err.location)
}
}
/// Represents the different types of errors that can occur during parsing of an f-string.
#[derive(Copy, Debug, Clone, PartialEq)]
pub enum FStringErrorType {
/// Expected a right brace after an opened left brace.
UnclosedLbrace,
/// An invalid conversion flag was encountered.
InvalidConversionFlag,
/// A single right brace was encountered.
SingleRbrace,
/// Unterminated string.
UnterminatedString,
/// Unterminated triple-quoted string.
UnterminatedTripleQuotedString,
// TODO(dhruvmanila): The parser can't catch all cases of this error, but
// wherever it can, we'll display the correct error message.
/// A lambda expression without parentheses was encountered.
LambdaWithoutParentheses,
}
impl std::fmt::Display for FStringErrorType {
fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
use FStringErrorType::{
InvalidConversionFlag, LambdaWithoutParentheses, SingleRbrace, UnclosedLbrace,
UnterminatedString, UnterminatedTripleQuotedString,
};
match self {
UnclosedLbrace => write!(f, "expecting '}}'"),
InvalidConversionFlag => write!(f, "invalid conversion character"),
SingleRbrace => write!(f, "single '}}' is not allowed"),
UnterminatedString => write!(f, "unterminated string"),
UnterminatedTripleQuotedString => write!(f, "unterminated triple-quoted string"),
LambdaWithoutParentheses => {
write!(f, "lambda expressions are not allowed without parentheses")
}
}
}
}
impl From<FStringError> for crate::parser::LalrpopError<TextSize, Tok, LexicalError> {
fn from(err: FStringError) -> Self {
lalrpop_util::ParseError::User {
error: LexicalError::new(LexicalErrorType::FStringError(err.error), err.location),
}
}
}
#[cfg(test)]
mod tests {
use crate::lexer::LexicalErrorType;
use crate::parser::parse_suite;
use crate::{ParseErrorType, Suite};
use super::*;
use crate::{parse_suite, FStringErrorType, ParseErrorType, Suite};
const WINDOWS_EOL: &str = "\r\n";
const MAC_EOL: &str = "\r";
@@ -675,6 +646,7 @@ mod tests {
parse_suite(source)
.map_err(|e| match e.error {
ParseErrorType::Lexical(LexicalErrorType::FStringError(e)) => e,
ParseErrorType::FStringError(e) => e,
e => unreachable!("Expected FStringError: {:?}", e),
})
.expect_err("Expected error")
@@ -689,10 +661,12 @@ mod tests {
parse_fstring_error("f'{lambda x:{x}}'"),
LambdaWithoutParentheses
);
assert_eq!(
parse_fstring_error("f'{lambda x: {x}}'"),
LambdaWithoutParentheses
);
// NOTE: The parser produces the `LambdaWithoutParentheses` for this case, but
// since the parser only return the first error to maintain compatibility with
// the rest of the codebase, this test case fails. The `LambdaWithoutParentheses`
// error appears after the unexpected `FStringMiddle` token, which is between the
// `:` and the `{`.
// assert_eq!(parse_fstring_error("f'{lambda x: {x}}'"), LambdaWithoutParentheses);
assert!(parse_suite(r#"f"{class}""#,).is_err());
}