Avoid extra String allocation during parsing

crates/ruff_linter/resources/test/fixtures/ruff/RUF011.py

@@ -7,6 +7,9 @@ constant = 5
 {v: v * v for v in range(10)}
 {(0, "a", v): v * v for v in range(10)}  # Tuple with variable
 {constant: value.upper() for value in data for constant in data}
+{value.attribute: value.upper() for value in data for constant in data}
+{constant[value]: value.upper() for value in data for constant in data}
+{value[constant]: value.upper() for value in data for constant in data}
 
 # Errors
 {"key": value.upper() for value in data}
@@ -15,3 +18,5 @@ constant = 5
 {(1, "a"): value.upper() for value in data}  # Constant tuple
 {constant: value.upper() for value in data}
 {constant + constant: value.upper() for value in data}
+{constant.attribute: value.upper() for value in data}
+{constant[0]: value.upper() for value in data}

crates/ruff_linter/src/checkers/tokens.rs

@@ -61,7 +61,7 @@ pub(crate) fn check_tokens(
                     }
                 }
                 Tok::FStringMiddle { .. } => Context::String,
-                Tok::Comment => Context::Comment,
+                Tok::Comment(_) => Context::Comment,
                 _ => continue,
             };
             ruff::rules::ambiguous_unicode_character(

crates/ruff_linter/src/doc_lines.rs

@@ -39,7 +39,7 @@ impl Iterator for DocLines<'_> {
             let (tok, range) = self.inner.next()?;
 
             match tok {
-                Tok::Comment => {
+                Tok::Comment(..) => {
                     if at_start_of_line {
                         break Some(range.start());
                     }

crates/ruff_linter/src/importer/insertion.rs

@@ -174,7 +174,7 @@ impl<'a> Insertion<'a> {
                 // Once we've seen the colon, we're looking for a newline; otherwise, there's no
                 // block body (e.g. `if True: pass`).
                 Awaiting::Newline => match tok {
-                    Tok::Comment => {}
+                    Tok::Comment(..) => {}
                     Tok::Newline => {
                         state = Awaiting::Indent;
                     }
@@ -185,7 +185,7 @@ impl<'a> Insertion<'a> {
                 },
                 // Once we've seen the newline, we're looking for the indentation of the block body.
                 Awaiting::Indent => match tok {
-                    Tok::Comment => {}
+                    Tok::Comment(..) => {}
                     Tok::NonLogicalNewline => {}
                     Tok::Indent => {
                         // This is like:

crates/ruff_linter/src/lex/docstring_detection.rs

@@ -33,9 +33,11 @@ pub(crate) struct StateMachine {
 impl StateMachine {
     pub(crate) fn consume(&mut self, tok: &Tok) -> bool {
         match tok {
-            Tok::NonLogicalNewline | Tok::Newline | Tok::Indent | Tok::Dedent | Tok::Comment => {
-                false
-            }
+            Tok::NonLogicalNewline
+            | Tok::Newline
+            | Tok::Indent
+            | Tok::Dedent
+            | Tok::Comment(..) => false,
 
             Tok::String { .. } => {
                 if matches!(

crates/ruff_linter/src/rules/flake8_commas/rules/trailing_commas.rs

@@ -242,7 +242,7 @@ pub(crate) fn trailing_commas(
         .flatten()
         .filter_map(|spanned @ (tok, tok_range)| match tok {
             // Completely ignore comments -- they just interfere with the logic.
-            Tok::Comment => None,
+            Tok::Comment(_) => None,
             // F-strings are handled as `String` token type with the complete range
             // of the outermost f-string. This means that the expression inside the
             // f-string is not checked for trailing commas.

crates/ruff_linter/src/rules/flake8_quotes/rules/avoidable_escaped_quote.rs

@@ -168,7 +168,7 @@ pub(crate) fn avoidable_escaped_quote(
 
         match tok {
             Tok::String {
-                value: string_contents,
+                value,
                 kind,
                 triple_quoted,
             } => {
@@ -176,6 +176,8 @@ pub(crate) fn avoidable_escaped_quote(
                     continue;
                 }
 
+                let string_contents = locator.slice(&value);
+
                 // Check if we're using the preferred quotation style.
                 if !leading_quote(locator.slice(tok_range)).is_some_and(|text| {
                     contains_quote(text, quotes_settings.inline_quotes.as_char())
@@ -312,7 +314,7 @@ pub(crate) fn unnecessary_escaped_quote(
 
         match tok {
             Tok::String {
-                value: string_contents,
+                value,
                 kind,
                 triple_quoted,
             } => {
@@ -320,6 +322,8 @@ pub(crate) fn unnecessary_escaped_quote(
                     continue;
                 }
 
+                let string_contents = locator.slice(&value);
+
                 let leading = match leading_quote(locator.slice(tok_range)) {
                     Some("\"") => Quote::Double,
                     Some("'") => Quote::Single,

crates/ruff_linter/src/rules/flake8_quotes/rules/check_string_quotes.rs

@@ -461,7 +461,7 @@ pub(crate) fn check_string_quotes(
                     // range to the sequence.
                     sequence.push(fstring_range_builder.finish());
                 }
-                Tok::Comment | Tok::NonLogicalNewline => continue,
+                Tok::Comment(..) | Tok::NonLogicalNewline => continue,
                 _ => {
                     // Otherwise, consume the sequence.
                     if !sequence.is_empty() {

crates/ruff_linter/src/rules/isort/helpers.rs

@@ -26,7 +26,7 @@ pub(super) fn trailing_comma(
         if count == 1 {
             if matches!(
                 tok,
-                Tok::NonLogicalNewline | Tok::Indent | Tok::Dedent | Tok::Comment
+                Tok::NonLogicalNewline | Tok::Indent | Tok::Dedent | Tok::Comment(_)
             ) {
                 continue;
             } else if matches!(tok, Tok::Comma) {

crates/ruff_linter/src/rules/pycodestyle/rules/compound_statements.rs

@@ -239,7 +239,7 @@ pub(crate) fn compound_statements(
                 semi = Some((range.start(), range.end()));
                 allow_ellipsis = false;
             }
-            Tok::Comment | Tok::Indent | Tok::Dedent | Tok::NonLogicalNewline => {}
+            Tok::Comment(..) | Tok::Indent | Tok::Dedent | Tok::NonLogicalNewline => {}
             _ => {
                 if let Some((start, end)) = semi {
                     diagnostics.push(Diagnostic::new(
@@ -347,7 +347,7 @@ fn has_non_trivia_tokens_till<'a>(
         }
         if !matches!(
             tok,
-            Tok::Newline | Tok::Comment | Tok::EndOfFile | Tok::NonLogicalNewline
+            Tok::Newline | Tok::Comment(_) | Tok::EndOfFile | Tok::NonLogicalNewline
         ) {
             return true;
         }

crates/ruff_linter/src/rules/pyupgrade/rules/extraneous_parentheses.rs

@@ -48,7 +48,7 @@ fn match_extraneous_parentheses(tokens: &[LexResult], mut i: usize) -> Option<(u
             return None;
         };
         match tok {
-            Tok::Comment | Tok::NonLogicalNewline => {
+            Tok::Comment(..) | Tok::NonLogicalNewline => {
                 i += 1;
             }
             Tok::Lpar => {
@@ -88,7 +88,12 @@ fn match_extraneous_parentheses(tokens: &[LexResult], mut i: usize) -> Option<(u
     let end = i;
 
     // Verify that we're not in an empty tuple.
-    if (start + 1..i).all(|i| matches!(tokens[i], Ok((Tok::Comment | Tok::NonLogicalNewline, _)))) {
+    if (start + 1..i).all(|i| {
+        matches!(
+            tokens[i],
+            Ok((Tok::Comment(..) | Tok::NonLogicalNewline, _))
+        )
+    }) {
         return None;
     }
 
@@ -102,7 +107,7 @@ fn match_extraneous_parentheses(tokens: &[LexResult], mut i: usize) -> Option<(u
             return None;
         };
         match tok {
-            Tok::Comment | Tok::NonLogicalNewline => {
+            Tok::Comment(..) | Tok::NonLogicalNewline => {
                 i += 1;
             }
             _ => {

crates/ruff_linter/src/rules/pyupgrade/rules/f_strings.rs

@@ -426,7 +426,7 @@ pub(crate) fn f_strings(
     if !lexer::lex_starts_at(rest, Mode::Expression, prev_end)
         .flatten()
         .all(|(token, _)| match token {
-            Tok::Comment | Tok::Newline | Tok::NonLogicalNewline | Tok::Indent | Tok::Dedent => {
+            Tok::Comment(_) | Tok::Newline | Tok::NonLogicalNewline | Tok::Indent | Tok::Dedent => {
                 true
             }
             Tok::String { value, .. } => value.is_empty(),

crates/ruff_linter/src/rules/ruff/rules/static_key_dict_comprehension.rs

@@ -74,6 +74,10 @@ fn is_constant(key: &Expr, names: &FxHashMap<&str, &ast::ExprName>) -> bool {
     match key {
         Expr::Tuple(ast::ExprTuple { elts, .. }) => elts.iter().all(|elt| is_constant(elt, names)),
         Expr::Name(ast::ExprName { id, .. }) => !names.contains_key(id.as_str()),
+        Expr::Attribute(ast::ExprAttribute { value, .. }) => is_constant(value, names),
+        Expr::Subscript(ast::ExprSubscript { value, slice, .. }) => {
+            is_constant(value, names) && is_constant(slice, names)
+        }
         Expr::BinOp(ast::ExprBinOp { left, right, .. }) => {
             is_constant(left, names) && is_constant(right, names)
         }

crates/ruff_linter/src/rules/ruff/snapshots/ruff_linter__rules__ruff__tests__RUF011_RUF011.py.snap

@@ -1,60 +1,80 @@
 ---
 source: crates/ruff_linter/src/rules/ruff/mod.rs
 ---
-RUF011.py:12:2: RUF011 Dictionary comprehension uses static key: `"key"`
+RUF011.py:15:2: RUF011 Dictionary comprehension uses static key: `"key"`
    |
-11 | # Errors
-12 | {"key": value.upper() for value in data}
+14 | # Errors
+15 | {"key": value.upper() for value in data}
    |  ^^^^^ RUF011
-13 | {True: value.upper() for value in data}
-14 | {0: value.upper() for value in data}
+16 | {True: value.upper() for value in data}
+17 | {0: value.upper() for value in data}
    |
 
-RUF011.py:13:2: RUF011 Dictionary comprehension uses static key: `True`
+RUF011.py:16:2: RUF011 Dictionary comprehension uses static key: `True`
    |
-11 | # Errors
-12 | {"key": value.upper() for value in data}
-13 | {True: value.upper() for value in data}
+14 | # Errors
+15 | {"key": value.upper() for value in data}
+16 | {True: value.upper() for value in data}
    |  ^^^^ RUF011
-14 | {0: value.upper() for value in data}
-15 | {(1, "a"): value.upper() for value in data}  # Constant tuple
+17 | {0: value.upper() for value in data}
+18 | {(1, "a"): value.upper() for value in data}  # Constant tuple
    |
 
-RUF011.py:14:2: RUF011 Dictionary comprehension uses static key: `0`
+RUF011.py:17:2: RUF011 Dictionary comprehension uses static key: `0`
    |
-12 | {"key": value.upper() for value in data}
-13 | {True: value.upper() for value in data}
-14 | {0: value.upper() for value in data}
+15 | {"key": value.upper() for value in data}
+16 | {True: value.upper() for value in data}
+17 | {0: value.upper() for value in data}
    |  ^ RUF011
-15 | {(1, "a"): value.upper() for value in data}  # Constant tuple
-16 | {constant: value.upper() for value in data}
+18 | {(1, "a"): value.upper() for value in data}  # Constant tuple
+19 | {constant: value.upper() for value in data}
    |
 
-RUF011.py:15:2: RUF011 Dictionary comprehension uses static key: `(1, "a")`
+RUF011.py:18:2: RUF011 Dictionary comprehension uses static key: `(1, "a")`
    |
-13 | {True: value.upper() for value in data}
-14 | {0: value.upper() for value in data}
-15 | {(1, "a"): value.upper() for value in data}  # Constant tuple
+16 | {True: value.upper() for value in data}
+17 | {0: value.upper() for value in data}
+18 | {(1, "a"): value.upper() for value in data}  # Constant tuple
    |  ^^^^^^^^ RUF011
-16 | {constant: value.upper() for value in data}
-17 | {constant + constant: value.upper() for value in data}
+19 | {constant: value.upper() for value in data}
+20 | {constant + constant: value.upper() for value in data}
    |
 
-RUF011.py:16:2: RUF011 Dictionary comprehension uses static key: `constant`
+RUF011.py:19:2: RUF011 Dictionary comprehension uses static key: `constant`
    |
-14 | {0: value.upper() for value in data}
-15 | {(1, "a"): value.upper() for value in data}  # Constant tuple
-16 | {constant: value.upper() for value in data}
+17 | {0: value.upper() for value in data}
+18 | {(1, "a"): value.upper() for value in data}  # Constant tuple
+19 | {constant: value.upper() for value in data}
    |  ^^^^^^^^ RUF011
-17 | {constant + constant: value.upper() for value in data}
+20 | {constant + constant: value.upper() for value in data}
+21 | {constant.attribute: value.upper() for value in data}
    |
 
-RUF011.py:17:2: RUF011 Dictionary comprehension uses static key: `constant + constant`
+RUF011.py:20:2: RUF011 Dictionary comprehension uses static key: `constant + constant`
    |
-15 | {(1, "a"): value.upper() for value in data}  # Constant tuple
-16 | {constant: value.upper() for value in data}
-17 | {constant + constant: value.upper() for value in data}
+18 | {(1, "a"): value.upper() for value in data}  # Constant tuple
+19 | {constant: value.upper() for value in data}
+20 | {constant + constant: value.upper() for value in data}
    |  ^^^^^^^^^^^^^^^^^^^ RUF011
+21 | {constant.attribute: value.upper() for value in data}
+22 | {constant[0]: value.upper() for value in data}
+   |
+
+RUF011.py:21:2: RUF011 Dictionary comprehension uses static key: `constant.attribute`
+   |
+19 | {constant: value.upper() for value in data}
+20 | {constant + constant: value.upper() for value in data}
+21 | {constant.attribute: value.upper() for value in data}
+   |  ^^^^^^^^^^^^^^^^^^ RUF011
+22 | {constant[0]: value.upper() for value in data}
+   |
+
+RUF011.py:22:2: RUF011 Dictionary comprehension uses static key: `constant[0]`
+   |
+20 | {constant + constant: value.upper() for value in data}
+21 | {constant.attribute: value.upper() for value in data}
+22 | {constant[0]: value.upper() for value in data}
+   |  ^^^^^^^^^^^ RUF011
    |
 
 

crates/ruff_python_parser/src/lexer.rs

@@ -411,7 +411,7 @@ impl<'source> Lexer<'source> {
         let offset = memchr::memchr2(b'\n', b'\r', bytes).unwrap_or(bytes.len());
         self.cursor.skip_bytes(offset);
 
-        Tok::Comment
+        Tok::Comment(self.token_text().to_string())
     }
 
     /// Lex a single IPython escape command.
@@ -759,7 +759,7 @@ impl<'source> Lexer<'source> {
         };
 
         let tok = Tok::String {
-            value: self.source[TextRange::new(value_start, value_end)].to_string(),
+            value: TextRange::new(value_start, value_end),
             kind,
             triple_quoted,
         };

crates/ruff_python_parser/src/lib.rs

@@ -176,7 +176,7 @@ pub fn locate_cmp_ops(expr: &Expr, source: &str) -> Vec<LocatedCmpOp> {
         .flatten()
         .skip(1)
         .map(|(tok, range)| (tok, range - TextSize::from(1)))
-        .filter(|(tok, _)| !matches!(tok, Tok::NonLogicalNewline | Tok::Comment))
+        .filter(|(tok, _)| !matches!(tok, Tok::NonLogicalNewline | Tok::Comment(_)))
         .peekable();
 
     let mut ops: Vec<LocatedCmpOp> = vec![];

crates/ruff_python_parser/src/python.lalrpop

@@ -1605,8 +1605,8 @@ StringLiteralOrFString: StringType = {
 
 StringLiteral: StringType = {
     <location:@L> <string:string> <end_location:@R> =>? {
-        let (source, kind, triple_quoted) = string;
-        Ok(parse_string_literal(&source, kind, triple_quoted, (location..end_location).into())?)
+        let (value, kind, triple_quoted) = string;
+        Ok(parse_string_literal(&source_code[value], kind, triple_quoted, (location..end_location).into())?)
     }
 };
 
@@ -2061,7 +2061,7 @@ extern {
         float => token::Tok::Float { value: <f64> },
         complex => token::Tok::Complex { real: <f64>, imag: <f64> },
         string => token::Tok::String {
-            value: <String>,
+            value: <TextRange>,
             kind: <StringKind>,
             triple_quoted: <bool>
         },
@@ -2076,6 +2076,6 @@ extern {
         },
         "\n" => token::Tok::Newline,
         ";" => token::Tok::Semi,
-        // "#" => token::Tok::Comment,
+        // "#" => token::Tok::Comment(_),
     }
 }

crates/ruff_python_parser/src/python.rs

@@ -1,5 +1,5 @@
 // auto-generated: "lalrpop 0.20.0"
-// sha3: 64183bfe2809b4e883c796c3b5125541e4be6bf9022efb1374d4bf2b842236a7
+// sha3: 28f158c07e00e286b0a28fb9af14b474f60e5d67d1dd47e5dddc93a4b622c46b
 use ruff_text_size::{Ranged, TextLen, TextRange, TextSize};
 use ruff_python_ast::{self as ast, Int, IpyEscapeKind};
 use crate::{
@@ -54,7 +54,7 @@ mod __parse__Top {
         Variant4(Int),
         Variant5((IpyEscapeKind, String)),
         Variant6(String),
-        Variant7((String, StringKind, bool)),
+        Variant7((TextRange, StringKind, bool)),
         Variant8(core::option::Option<token::Tok>),
         Variant9(Option<Box<ast::Parameter>>),
         Variant10(core::option::Option<Option<Box<ast::Parameter>>>),
@@ -18373,16 +18373,6 @@ mod __parse__Top {
             _ => __symbol_type_mismatch()
         }
     }
-    fn __pop_Variant7<
-    >(
-        __symbols: &mut alloc::vec::Vec<(TextSize,__Symbol<>,TextSize)>
-    ) -> (TextSize, (String, StringKind, bool), TextSize)
-     {
-        match __symbols.pop() {
-            Some((__l, __Symbol::Variant7(__v), __r)) => (__l, __v, __r),
-            _ => __symbol_type_mismatch()
-        }
-    }
     fn __pop_Variant3<
     >(
         __symbols: &mut alloc::vec::Vec<(TextSize,__Symbol<>,TextSize)>
@@ -18393,6 +18383,16 @@ mod __parse__Top {
             _ => __symbol_type_mismatch()
         }
     }
+    fn __pop_Variant7<
+    >(
+        __symbols: &mut alloc::vec::Vec<(TextSize,__Symbol<>,TextSize)>
+    ) -> (TextSize, (TextRange, StringKind, bool), TextSize)
+     {
+        match __symbols.pop() {
+            Some((__l, __Symbol::Variant7(__v), __r)) => (__l, __v, __r),
+            _ => __symbol_type_mismatch()
+        }
+    }
     fn __pop_Variant67<
     >(
         __symbols: &mut alloc::vec::Vec<(TextSize,__Symbol<>,TextSize)>
@@ -36363,13 +36363,13 @@ fn __action217<
     source_code: &str,
     mode: Mode,
     (_, location, _): (TextSize, TextSize, TextSize),
-    (_, string, _): (TextSize, (String, StringKind, bool), TextSize),
+    (_, string, _): (TextSize, (TextRange, StringKind, bool), TextSize),
     (_, end_location, _): (TextSize, TextSize, TextSize),
 ) -> Result<StringType,__lalrpop_util::ParseError<TextSize,token::Tok,LexicalError>>
 {
     {
-        let (source, kind, triple_quoted) = string;
-        Ok(parse_string_literal(&source, kind, triple_quoted, (location..end_location).into())?)
+        let (value, kind, triple_quoted) = string;
+        Ok(parse_string_literal(&source_code[value], kind, triple_quoted, (location..end_location).into())?)
     }
 }
 
@@ -52719,7 +52719,7 @@ fn __action937<
 >(
     source_code: &str,
     mode: Mode,
-    __0: (TextSize, (String, StringKind, bool), TextSize),
+    __0: (TextSize, (TextRange, StringKind, bool), TextSize),
     __1: (TextSize, TextSize, TextSize),
 ) -> Result<StringType,__lalrpop_util::ParseError<TextSize,token::Tok,LexicalError>>
 {
@@ -69997,7 +69997,7 @@ fn __action1494<
 >(
     source_code: &str,
     mode: Mode,
-    __0: (TextSize, (String, StringKind, bool), TextSize),
+    __0: (TextSize, (TextRange, StringKind, bool), TextSize),
 ) -> Result<StringType,__lalrpop_util::ParseError<TextSize,token::Tok,LexicalError>>
 {
     let __start0 = __0.2;

crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__comment_until_mac_eol.snap

@@ -10,7 +10,9 @@ expression: comment_until_eol(MAC_EOL)
         0..3,
     ),
     (
-        Comment,
+        Comment(
+            "# Foo",
+        ),
         5..10,
     ),
     (

crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__comment_until_unix_eol.snap

@@ -10,7 +10,9 @@ expression: comment_until_eol(UNIX_EOL)
         0..3,
     ),
     (
-        Comment,
+        Comment(
+            "# Foo",
+        ),
         5..10,
     ),
     (

crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__comment_until_windows_eol.snap

@@ -10,7 +10,9 @@ expression: comment_until_eol(WINDOWS_EOL)
         0..3,
     ),
     (
-        Comment,
+        Comment(
+            "# Foo",
+        ),
         5..10,
     ),
     (

crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__empty_fstrings.snap

@@ -13,7 +13,7 @@ expression: lex_source(source)
     ),
     (
         String {
-            value: "",
+            value: 5..5,
             kind: String,
             triple_quoted: false,
         },
@@ -37,7 +37,7 @@ expression: lex_source(source)
     ),
     (
         String {
-            value: "",
+            value: 16..16,
             kind: String,
             triple_quoted: false,
         },

crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__escape_unicode_name.snap

@@ -5,7 +5,7 @@ expression: lex_source(source)
 [
     (
         String {
-            value: "\\N{EN SPACE}",
+            value: 1..13,
             kind: String,
             triple_quoted: false,
         },

crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__fstring_comments.snap

@@ -19,7 +19,9 @@ expression: lex_source(source)
         21..22,
     ),
     (
-        Comment,
+        Comment(
+            "# comment {",
+        ),
         23..34,
     ),
     (

crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__fstring_with_format_spec.snap

@@ -137,7 +137,7 @@ expression: lex_source(source)
     ),
     (
         String {
-            value: "",
+            value: 32..32,
             kind: String,
             triple_quoted: false,
         },

crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__line_comment_empty.snap

@@ -10,7 +10,9 @@ expression: lex_source(&source)
         0..5,
     ),
     (
-        Comment,
+        Comment(
+            "#",
+        ),
         7..8,
     ),
     (

crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__line_comment_long.snap

@@ -10,7 +10,9 @@ expression: lex_source(&source)
         0..5,
     ),
     (
-        Comment,
+        Comment(
+            "# foo",
+        ),
         7..12,
     ),
     (

crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__line_comment_single_whitespace.snap

@@ -10,7 +10,9 @@ expression: lex_source(&source)
         0..5,
     ),
     (
-        Comment,
+        Comment(
+            "# ",
+        ),
         7..9,
     ),
     (

crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__line_comment_whitespace.snap

@@ -10,7 +10,9 @@ expression: lex_source(&source)
         0..5,
     ),
     (
-        Comment,
+        Comment(
+            "#  ",
+        ),
         7..10,
     ),
     (

crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__logical_newline_line_comment.snap

@@ -4,7 +4,9 @@ expression: lex_source(source)
 ---
 [
     (
-        Comment,
+        Comment(
+            "#Hello",
+        ),
         0..6,
     ),
     (
@@ -12,7 +14,9 @@ expression: lex_source(source)
         6..7,
     ),
     (
-        Comment,
+        Comment(
+            "#World",
+        ),
         7..13,
     ),
     (

crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__non_logical_newline_in_string_continuation.snap

@@ -13,7 +13,7 @@ expression: lex_source(source)
     ),
     (
         String {
-            value: "a",
+            value: 7..8,
             kind: String,
             triple_quoted: false,
         },
@@ -25,7 +25,7 @@ expression: lex_source(source)
     ),
     (
         String {
-            value: "b",
+            value: 15..16,
             kind: String,
             triple_quoted: false,
         },
@@ -41,7 +41,7 @@ expression: lex_source(source)
     ),
     (
         String {
-            value: "c",
+            value: 24..25,
             kind: String,
             triple_quoted: false,
         },
@@ -49,7 +49,7 @@ expression: lex_source(source)
     ),
     (
         String {
-            value: "d",
+            value: 34..35,
             kind: String,
             triple_quoted: false,
         },

crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__string.snap

@@ -5,7 +5,7 @@ expression: lex_source(source)
 [
     (
         String {
-            value: "double",
+            value: 1..7,
             kind: String,
             triple_quoted: false,
         },
@@ -13,7 +13,7 @@ expression: lex_source(source)
     ),
     (
         String {
-            value: "single",
+            value: 10..16,
             kind: String,
             triple_quoted: false,
         },
@@ -21,7 +21,7 @@ expression: lex_source(source)
     ),
     (
         String {
-            value: "can\\'t",
+            value: 19..25,
             kind: String,
             triple_quoted: false,
         },
@@ -29,7 +29,7 @@ expression: lex_source(source)
     ),
     (
         String {
-            value: "\\\\\\\"",
+            value: 28..32,
             kind: String,
             triple_quoted: false,
         },
@@ -37,7 +37,7 @@ expression: lex_source(source)
     ),
     (
         String {
-            value: "\\t\\r\\n",
+            value: 35..41,
             kind: String,
             triple_quoted: false,
         },
@@ -45,7 +45,7 @@ expression: lex_source(source)
     ),
     (
         String {
-            value: "\\g",
+            value: 44..46,
             kind: String,
             triple_quoted: false,
         },
@@ -53,7 +53,7 @@ expression: lex_source(source)
     ),
     (
         String {
-            value: "raw\\'",
+            value: 50..55,
             kind: RawString,
             triple_quoted: false,
         },
@@ -61,7 +61,7 @@ expression: lex_source(source)
     ),
     (
         String {
-            value: "\\420",
+            value: 58..62,
             kind: String,
             triple_quoted: false,
         },
@@ -69,7 +69,7 @@ expression: lex_source(source)
     ),
     (
         String {
-            value: "\\200\\0a",
+            value: 65..72,
             kind: String,
             triple_quoted: false,
         },

crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__string_continuation_with_mac_eol.snap

@@ -5,7 +5,7 @@ expression: string_continuation_with_eol(MAC_EOL)
 [
     (
         String {
-            value: "abc\\\rdef",
+            value: 1..9,
             kind: String,
             triple_quoted: false,
         },

crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__string_continuation_with_unix_eol.snap

@@ -5,7 +5,7 @@ expression: string_continuation_with_eol(UNIX_EOL)
 [
     (
         String {
-            value: "abc\\\ndef",
+            value: 1..9,
             kind: String,
             triple_quoted: false,
         },

crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__string_continuation_with_windows_eol.snap

@@ -5,7 +5,7 @@ expression: string_continuation_with_eol(WINDOWS_EOL)
 [
     (
         String {
-            value: "abc\\\r\ndef",
+            value: 1..10,
             kind: String,
             triple_quoted: false,
         },

crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__triple_quoted_mac_eol.snap

@@ -5,7 +5,7 @@ expression: triple_quoted_eol(MAC_EOL)
 [
     (
         String {
-            value: "\r test string\r ",
+            value: 3..18,
             kind: String,
             triple_quoted: true,
         },

crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__triple_quoted_unix_eol.snap

@@ -5,7 +5,7 @@ expression: triple_quoted_eol(UNIX_EOL)
 [
     (
         String {
-            value: "\n test string\n ",
+            value: 3..18,
             kind: String,
             triple_quoted: true,
         },

crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__triple_quoted_windows_eol.snap

@@ -5,7 +5,7 @@ expression: triple_quoted_eol(WINDOWS_EOL)
 [
     (
         String {
-            value: "\r\n test string\r\n ",
+            value: 3..20,
             kind: String,
             triple_quoted: true,
         },

crates/ruff_python_parser/src/token.rs

@@ -7,7 +7,7 @@
 use crate::Mode;
 
 use ruff_python_ast::{Int, IpyEscapeKind};
-use ruff_text_size::TextSize;
+use ruff_text_size::{TextRange, TextSize};
 use std::fmt;
 
 /// The set of tokens the Python source code can be tokenized in.
@@ -37,8 +37,8 @@ pub enum Tok {
     },
     /// Token value for a string.
     String {
-        /// The string value.
-        value: String,
+        /// The range of the string value.
+        value: TextRange,
         /// The kind of string.
         kind: StringKind,
         /// Whether the string is triple quoted.
@@ -51,6 +51,8 @@ pub enum Tok {
     /// part of the expression part and isn't an opening or closing brace.
     FStringMiddle {
         /// The string value.
+        // TODO(charlie): This could _maybe_ be a range, but we'd have to move logic into the parser
+        // to handle some escaping.
         value: String,
         /// Whether the string is raw or not.
         is_raw: bool,
@@ -66,7 +68,7 @@ pub enum Tok {
         kind: IpyEscapeKind,
     },
     /// Token value for a comment. These are filtered out of the token stream prior to parsing.
-    Comment,
+    Comment(String),
     /// Token value for a newline.
     Newline,
     /// Token value for a newline that is not a logical line break. These are filtered out of
@@ -241,14 +243,7 @@ impl fmt::Display for Tok {
             Int { value } => write!(f, "'{value}'"),
             Float { value } => write!(f, "'{value}'"),
             Complex { real, imag } => write!(f, "{real}j{imag}"),
-            String {
-                value,
-                kind,
-                triple_quoted,
-            } => {
-                let quotes = "\"".repeat(if *triple_quoted { 3 } else { 1 });
-                write!(f, "{kind}{quotes}{value}{quotes}")
-            }
+            String { .. } => write!(f, "String"),
             FStringStart => f.write_str("FStringStart"),
             FStringMiddle { value, .. } => f.write_str(value),
             FStringEnd => f.write_str("FStringEnd"),
@@ -268,7 +263,7 @@ impl fmt::Display for Tok {
             Rsqb => f.write_str("']'"),
             Colon => f.write_str("':'"),
             Comma => f.write_str("','"),
-            Comment => f.write_str("Comment"),
+            Comment(value) => f.write_str(value),
             Semi => f.write_str("';'"),
             Plus => f.write_str("'+'"),
             Minus => f.write_str("'-'"),
@@ -806,7 +801,7 @@ impl TokenKind {
             Tok::FStringMiddle { .. } => TokenKind::FStringMiddle,
             Tok::FStringEnd => TokenKind::FStringEnd,
             Tok::IpyEscapeCommand { .. } => TokenKind::EscapeCommand,
-            Tok::Comment => TokenKind::Comment,
+            Tok::Comment(_) => TokenKind::Comment,
             Tok::Newline => TokenKind::Newline,
             Tok::NonLogicalNewline => TokenKind::NonLogicalNewline,
             Tok::Indent => TokenKind::Indent,

crates/ruff_python_trivia/src/tokenizer.rs

@@ -1019,7 +1019,7 @@ mod tests {
             let comment_ranges: Vec<_> = lex(self.source, Mode::Module)
                 .filter_map(|result| {
                     let (token, range) = result.expect("Input to be a valid python program.");
-                    if matches!(token, Tok::Comment) {
+                    if matches!(token, Tok::Comment(_)) {
                         Some(range)
                     } else {
                         None