diff --git a/crates/ruff_linter/src/checkers/tokens.rs b/crates/ruff_linter/src/checkers/tokens.rs index c67c21ba4d..fc17a24675 100644 --- a/crates/ruff_linter/src/checkers/tokens.rs +++ b/crates/ruff_linter/src/checkers/tokens.rs @@ -61,7 +61,7 @@ pub(crate) fn check_tokens( } } Tok::FStringMiddle { .. } => Context::String, - Tok::Comment(_) => Context::Comment, + Tok::Comment => Context::Comment, _ => continue, }; ruff::rules::ambiguous_unicode_character( diff --git a/crates/ruff_linter/src/doc_lines.rs b/crates/ruff_linter/src/doc_lines.rs index 7b69fb8669..a6abec9685 100644 --- a/crates/ruff_linter/src/doc_lines.rs +++ b/crates/ruff_linter/src/doc_lines.rs @@ -39,7 +39,7 @@ impl Iterator for DocLines<'_> { let (tok, range) = self.inner.next()?; match tok { - Tok::Comment(..) => { + Tok::Comment => { if at_start_of_line { break Some(range.start()); } diff --git a/crates/ruff_linter/src/importer/insertion.rs b/crates/ruff_linter/src/importer/insertion.rs index 15a46c6f02..cae8d0aca9 100644 --- a/crates/ruff_linter/src/importer/insertion.rs +++ b/crates/ruff_linter/src/importer/insertion.rs @@ -174,7 +174,7 @@ impl<'a> Insertion<'a> { // Once we've seen the colon, we're looking for a newline; otherwise, there's no // block body (e.g. `if True: pass`). Awaiting::Newline => match tok { - Tok::Comment(..) => {} + Tok::Comment => {} Tok::Newline => { state = Awaiting::Indent; } @@ -185,7 +185,7 @@ impl<'a> Insertion<'a> { }, // Once we've seen the newline, we're looking for the indentation of the block body. Awaiting::Indent => match tok { - Tok::Comment(..) => {} + Tok::Comment => {} Tok::NonLogicalNewline => {} Tok::Indent => { // This is like: diff --git a/crates/ruff_linter/src/lex/docstring_detection.rs b/crates/ruff_linter/src/lex/docstring_detection.rs index 33f8a4538a..066838f22e 100644 --- a/crates/ruff_linter/src/lex/docstring_detection.rs +++ b/crates/ruff_linter/src/lex/docstring_detection.rs @@ -33,11 +33,9 @@ pub(crate) struct StateMachine { impl StateMachine { pub(crate) fn consume(&mut self, tok: &Tok) -> bool { match tok { - Tok::NonLogicalNewline - | Tok::Newline - | Tok::Indent - | Tok::Dedent - | Tok::Comment(..) => false, + Tok::NonLogicalNewline | Tok::Newline | Tok::Indent | Tok::Dedent | Tok::Comment => { + false + } Tok::String { .. } => { if matches!( diff --git a/crates/ruff_linter/src/rules/flake8_commas/rules/trailing_commas.rs b/crates/ruff_linter/src/rules/flake8_commas/rules/trailing_commas.rs index f0dc92b600..410434f993 100644 --- a/crates/ruff_linter/src/rules/flake8_commas/rules/trailing_commas.rs +++ b/crates/ruff_linter/src/rules/flake8_commas/rules/trailing_commas.rs @@ -242,7 +242,7 @@ pub(crate) fn trailing_commas( .flatten() .filter_map(|spanned @ (tok, tok_range)| match tok { // Completely ignore comments -- they just interfere with the logic. - Tok::Comment(_) => None, + Tok::Comment => None, // F-strings are handled as `String` token type with the complete range // of the outermost f-string. This means that the expression inside the // f-string is not checked for trailing commas. diff --git a/crates/ruff_linter/src/rules/flake8_quotes/rules/check_string_quotes.rs b/crates/ruff_linter/src/rules/flake8_quotes/rules/check_string_quotes.rs index 45a845fc64..ecd29f4199 100644 --- a/crates/ruff_linter/src/rules/flake8_quotes/rules/check_string_quotes.rs +++ b/crates/ruff_linter/src/rules/flake8_quotes/rules/check_string_quotes.rs @@ -461,7 +461,7 @@ pub(crate) fn check_string_quotes( // range to the sequence. sequence.push(fstring_range_builder.finish()); } - Tok::Comment(..) | Tok::NonLogicalNewline => continue, + Tok::Comment | Tok::NonLogicalNewline => continue, _ => { // Otherwise, consume the sequence. if !sequence.is_empty() { diff --git a/crates/ruff_linter/src/rules/isort/helpers.rs b/crates/ruff_linter/src/rules/isort/helpers.rs index 6f519f8923..39d10df047 100644 --- a/crates/ruff_linter/src/rules/isort/helpers.rs +++ b/crates/ruff_linter/src/rules/isort/helpers.rs @@ -26,7 +26,7 @@ pub(super) fn trailing_comma( if count == 1 { if matches!( tok, - Tok::NonLogicalNewline | Tok::Indent | Tok::Dedent | Tok::Comment(_) + Tok::NonLogicalNewline | Tok::Indent | Tok::Dedent | Tok::Comment ) { continue; } else if matches!(tok, Tok::Comma) { diff --git a/crates/ruff_linter/src/rules/pycodestyle/rules/compound_statements.rs b/crates/ruff_linter/src/rules/pycodestyle/rules/compound_statements.rs index 2b7434dd23..99952a8e2e 100644 --- a/crates/ruff_linter/src/rules/pycodestyle/rules/compound_statements.rs +++ b/crates/ruff_linter/src/rules/pycodestyle/rules/compound_statements.rs @@ -239,7 +239,7 @@ pub(crate) fn compound_statements( semi = Some((range.start(), range.end())); allow_ellipsis = false; } - Tok::Comment(..) | Tok::Indent | Tok::Dedent | Tok::NonLogicalNewline => {} + Tok::Comment | Tok::Indent | Tok::Dedent | Tok::NonLogicalNewline => {} _ => { if let Some((start, end)) = semi { diagnostics.push(Diagnostic::new( @@ -347,7 +347,7 @@ fn has_non_trivia_tokens_till<'a>( } if !matches!( tok, - Tok::Newline | Tok::Comment(_) | Tok::EndOfFile | Tok::NonLogicalNewline + Tok::Newline | Tok::Comment | Tok::EndOfFile | Tok::NonLogicalNewline ) { return true; } diff --git a/crates/ruff_linter/src/rules/pyupgrade/rules/extraneous_parentheses.rs b/crates/ruff_linter/src/rules/pyupgrade/rules/extraneous_parentheses.rs index 621412d20c..0450436740 100644 --- a/crates/ruff_linter/src/rules/pyupgrade/rules/extraneous_parentheses.rs +++ b/crates/ruff_linter/src/rules/pyupgrade/rules/extraneous_parentheses.rs @@ -48,7 +48,7 @@ fn match_extraneous_parentheses(tokens: &[LexResult], mut i: usize) -> Option<(u return None; }; match tok { - Tok::Comment(..) | Tok::NonLogicalNewline => { + Tok::Comment | Tok::NonLogicalNewline => { i += 1; } Tok::Lpar => { @@ -88,12 +88,7 @@ fn match_extraneous_parentheses(tokens: &[LexResult], mut i: usize) -> Option<(u let end = i; // Verify that we're not in an empty tuple. - if (start + 1..i).all(|i| { - matches!( - tokens[i], - Ok((Tok::Comment(..) | Tok::NonLogicalNewline, _)) - ) - }) { + if (start + 1..i).all(|i| matches!(tokens[i], Ok((Tok::Comment | Tok::NonLogicalNewline, _)))) { return None; } @@ -107,7 +102,7 @@ fn match_extraneous_parentheses(tokens: &[LexResult], mut i: usize) -> Option<(u return None; }; match tok { - Tok::Comment(..) | Tok::NonLogicalNewline => { + Tok::Comment | Tok::NonLogicalNewline => { i += 1; } _ => { diff --git a/crates/ruff_linter/src/rules/pyupgrade/rules/f_strings.rs b/crates/ruff_linter/src/rules/pyupgrade/rules/f_strings.rs index 2955417791..4c1acb5706 100644 --- a/crates/ruff_linter/src/rules/pyupgrade/rules/f_strings.rs +++ b/crates/ruff_linter/src/rules/pyupgrade/rules/f_strings.rs @@ -426,7 +426,7 @@ pub(crate) fn f_strings( if !lexer::lex_starts_at(rest, Mode::Expression, prev_end) .flatten() .all(|(token, _)| match token { - Tok::Comment(_) | Tok::Newline | Tok::NonLogicalNewline | Tok::Indent | Tok::Dedent => { + Tok::Comment | Tok::Newline | Tok::NonLogicalNewline | Tok::Indent | Tok::Dedent => { true } Tok::String { value, .. } => value.is_empty(), diff --git a/crates/ruff_python_parser/src/lexer.rs b/crates/ruff_python_parser/src/lexer.rs index 154a9a26bf..5f99165869 100644 --- a/crates/ruff_python_parser/src/lexer.rs +++ b/crates/ruff_python_parser/src/lexer.rs @@ -411,7 +411,7 @@ impl<'source> Lexer<'source> { let offset = memchr::memchr2(b'\n', b'\r', bytes).unwrap_or(bytes.len()); self.cursor.skip_bytes(offset); - Tok::Comment(self.token_text().to_string()) + Tok::Comment } /// Lex a single IPython escape command. diff --git a/crates/ruff_python_parser/src/lib.rs b/crates/ruff_python_parser/src/lib.rs index 7566ee7e80..d7d04b4a08 100644 --- a/crates/ruff_python_parser/src/lib.rs +++ b/crates/ruff_python_parser/src/lib.rs @@ -176,7 +176,7 @@ pub fn locate_cmp_ops(expr: &Expr, source: &str) -> Vec { .flatten() .skip(1) .map(|(tok, range)| (tok, range - TextSize::from(1))) - .filter(|(tok, _)| !matches!(tok, Tok::NonLogicalNewline | Tok::Comment(_))) + .filter(|(tok, _)| !matches!(tok, Tok::NonLogicalNewline | Tok::Comment)) .peekable(); let mut ops: Vec = vec![]; diff --git a/crates/ruff_python_parser/src/python.lalrpop b/crates/ruff_python_parser/src/python.lalrpop index aa87fcd72d..fef5ff6a00 100644 --- a/crates/ruff_python_parser/src/python.lalrpop +++ b/crates/ruff_python_parser/src/python.lalrpop @@ -2076,6 +2076,6 @@ extern { }, "\n" => token::Tok::Newline, ";" => token::Tok::Semi, - // "#" => token::Tok::Comment(_), + // "#" => token::Tok::Comment, } } diff --git a/crates/ruff_python_parser/src/python.rs b/crates/ruff_python_parser/src/python.rs index 5771d7099f..ae431fdaf3 100644 --- a/crates/ruff_python_parser/src/python.rs +++ b/crates/ruff_python_parser/src/python.rs @@ -1,5 +1,5 @@ // auto-generated: "lalrpop 0.20.0" -// sha3: 031689e389556292d9dbd8a1b1ff8ca29bac76d83f1b345630481d620b89e1c2 +// sha3: 64183bfe2809b4e883c796c3b5125541e4be6bf9022efb1374d4bf2b842236a7 use ruff_text_size::{Ranged, TextLen, TextRange, TextSize}; use ruff_python_ast::{self as ast, Int, IpyEscapeKind}; use crate::{ diff --git a/crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__comment_until_mac_eol.snap b/crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__comment_until_mac_eol.snap index 5a0e7933e9..1fa49c0e79 100644 --- a/crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__comment_until_mac_eol.snap +++ b/crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__comment_until_mac_eol.snap @@ -10,9 +10,7 @@ expression: comment_until_eol(MAC_EOL) 0..3, ), ( - Comment( - "# Foo", - ), + Comment, 5..10, ), ( diff --git a/crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__comment_until_unix_eol.snap b/crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__comment_until_unix_eol.snap index 3fdbd4c10f..961d88ec61 100644 --- a/crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__comment_until_unix_eol.snap +++ b/crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__comment_until_unix_eol.snap @@ -10,9 +10,7 @@ expression: comment_until_eol(UNIX_EOL) 0..3, ), ( - Comment( - "# Foo", - ), + Comment, 5..10, ), ( diff --git a/crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__comment_until_windows_eol.snap b/crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__comment_until_windows_eol.snap index fcf5cfcb80..c448c8bcd8 100644 --- a/crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__comment_until_windows_eol.snap +++ b/crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__comment_until_windows_eol.snap @@ -10,9 +10,7 @@ expression: comment_until_eol(WINDOWS_EOL) 0..3, ), ( - Comment( - "# Foo", - ), + Comment, 5..10, ), ( diff --git a/crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__fstring_comments.snap b/crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__fstring_comments.snap index b0427ad58f..b731941da9 100644 --- a/crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__fstring_comments.snap +++ b/crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__fstring_comments.snap @@ -19,9 +19,7 @@ expression: lex_source(source) 21..22, ), ( - Comment( - "# comment {", - ), + Comment, 23..34, ), ( diff --git a/crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__line_comment_empty.snap b/crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__line_comment_empty.snap index 34d9125a63..34ab11a22a 100644 --- a/crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__line_comment_empty.snap +++ b/crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__line_comment_empty.snap @@ -10,9 +10,7 @@ expression: lex_source(&source) 0..5, ), ( - Comment( - "#", - ), + Comment, 7..8, ), ( diff --git a/crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__line_comment_long.snap b/crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__line_comment_long.snap index 0731cf4711..ed12789714 100644 --- a/crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__line_comment_long.snap +++ b/crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__line_comment_long.snap @@ -10,9 +10,7 @@ expression: lex_source(&source) 0..5, ), ( - Comment( - "# foo", - ), + Comment, 7..12, ), ( diff --git a/crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__line_comment_single_whitespace.snap b/crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__line_comment_single_whitespace.snap index f248b93ef1..1449082ee2 100644 --- a/crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__line_comment_single_whitespace.snap +++ b/crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__line_comment_single_whitespace.snap @@ -10,9 +10,7 @@ expression: lex_source(&source) 0..5, ), ( - Comment( - "# ", - ), + Comment, 7..9, ), ( diff --git a/crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__line_comment_whitespace.snap b/crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__line_comment_whitespace.snap index 4593910098..c36540960b 100644 --- a/crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__line_comment_whitespace.snap +++ b/crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__line_comment_whitespace.snap @@ -10,9 +10,7 @@ expression: lex_source(&source) 0..5, ), ( - Comment( - "# ", - ), + Comment, 7..10, ), ( diff --git a/crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__logical_newline_line_comment.snap b/crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__logical_newline_line_comment.snap index 944ad882a0..bbf5f47973 100644 --- a/crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__logical_newline_line_comment.snap +++ b/crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__logical_newline_line_comment.snap @@ -4,9 +4,7 @@ expression: lex_source(source) --- [ ( - Comment( - "#Hello", - ), + Comment, 0..6, ), ( @@ -14,9 +12,7 @@ expression: lex_source(source) 6..7, ), ( - Comment( - "#World", - ), + Comment, 7..13, ), ( diff --git a/crates/ruff_python_parser/src/token.rs b/crates/ruff_python_parser/src/token.rs index ac441395ff..2b668c35ce 100644 --- a/crates/ruff_python_parser/src/token.rs +++ b/crates/ruff_python_parser/src/token.rs @@ -66,7 +66,7 @@ pub enum Tok { kind: IpyEscapeKind, }, /// Token value for a comment. These are filtered out of the token stream prior to parsing. - Comment(String), + Comment, /// Token value for a newline. Newline, /// Token value for a newline that is not a logical line break. These are filtered out of @@ -268,7 +268,7 @@ impl fmt::Display for Tok { Rsqb => f.write_str("']'"), Colon => f.write_str("':'"), Comma => f.write_str("','"), - Comment(value) => f.write_str(value), + Comment => f.write_str("Comment"), Semi => f.write_str("';'"), Plus => f.write_str("'+'"), Minus => f.write_str("'-'"), @@ -806,7 +806,7 @@ impl TokenKind { Tok::FStringMiddle { .. } => TokenKind::FStringMiddle, Tok::FStringEnd => TokenKind::FStringEnd, Tok::IpyEscapeCommand { .. } => TokenKind::EscapeCommand, - Tok::Comment(_) => TokenKind::Comment, + Tok::Comment => TokenKind::Comment, Tok::Newline => TokenKind::Newline, Tok::NonLogicalNewline => TokenKind::NonLogicalNewline, Tok::Indent => TokenKind::Indent, diff --git a/crates/ruff_python_trivia/src/tokenizer.rs b/crates/ruff_python_trivia/src/tokenizer.rs index 7865a80187..4a5c323d68 100644 --- a/crates/ruff_python_trivia/src/tokenizer.rs +++ b/crates/ruff_python_trivia/src/tokenizer.rs @@ -1019,7 +1019,7 @@ mod tests { let comment_ranges: Vec<_> = lex(self.source, Mode::Module) .filter_map(|result| { let (token, range) = result.expect("Input to be a valid python program."); - if matches!(token, Tok::Comment(_)) { + if matches!(token, Tok::Comment) { Some(range) } else { None