Avoid extra String allocation during parsing

crates/ruff_linter/resources/test/fixtures/ruff/RUF011.py

@@ -7,6 +7,9 @@ constant = 5
 {v: v * v for v in range(10)}
 {(0, "a", v): v * v for v in range(10)}  # Tuple with variable
 {constant: value.upper() for value in data for constant in data}
+{value.attribute: value.upper() for value in data for constant in data}
+{constant[value]: value.upper() for value in data for constant in data}
+{value[constant]: value.upper() for value in data for constant in data}
 
 # Errors
 {"key": value.upper() for value in data}
@@ -15,3 +18,5 @@ constant = 5
 {(1, "a"): value.upper() for value in data}  # Constant tuple
 {constant: value.upper() for value in data}
 {constant + constant: value.upper() for value in data}
+{constant.attribute: value.upper() for value in data}
+{constant[0]: value.upper() for value in data}

crates/ruff_linter/src/rules/flake8_quotes/rules/avoidable_escaped_quote.rs

@@ -168,7 +168,7 @@ pub(crate) fn avoidable_escaped_quote(
 
         match tok {
             Tok::String {
-                value: string_contents,
+                value,
                 kind,
                 triple_quoted,
             } => {
@@ -176,6 +176,8 @@ pub(crate) fn avoidable_escaped_quote(
                     continue;
                 }
 
+                let string_contents = locator.slice(&value);
+
                 // Check if we're using the preferred quotation style.
                 if !leading_quote(locator.slice(tok_range)).is_some_and(|text| {
                     contains_quote(text, quotes_settings.inline_quotes.as_char())
@@ -312,7 +314,7 @@ pub(crate) fn unnecessary_escaped_quote(
 
         match tok {
             Tok::String {
-                value: string_contents,
+                value,
                 kind,
                 triple_quoted,
             } => {
@@ -320,6 +322,8 @@ pub(crate) fn unnecessary_escaped_quote(
                     continue;
                 }
 
+                let string_contents = locator.slice(&value);
+
                 let leading = match leading_quote(locator.slice(tok_range)) {
                     Some("\"") => Quote::Double,
                     Some("'") => Quote::Single,

crates/ruff_linter/src/rules/ruff/rules/static_key_dict_comprehension.rs

@@ -74,6 +74,10 @@ fn is_constant(key: &Expr, names: &FxHashMap<&str, &ast::ExprName>) -> bool {
     match key {
         Expr::Tuple(ast::ExprTuple { elts, .. }) => elts.iter().all(|elt| is_constant(elt, names)),
         Expr::Name(ast::ExprName { id, .. }) => !names.contains_key(id.as_str()),
+        Expr::Attribute(ast::ExprAttribute { value, .. }) => is_constant(value, names),
+        Expr::Subscript(ast::ExprSubscript { value, slice, .. }) => {
+            is_constant(value, names) && is_constant(slice, names)
+        }
         Expr::BinOp(ast::ExprBinOp { left, right, .. }) => {
             is_constant(left, names) && is_constant(right, names)
         }

crates/ruff_linter/src/rules/ruff/snapshots/ruff_linter__rules__ruff__tests__RUF011_RUF011.py.snap

@@ -1,60 +1,80 @@
 ---
 source: crates/ruff_linter/src/rules/ruff/mod.rs
 ---
-RUF011.py:12:2: RUF011 Dictionary comprehension uses static key: `"key"`
+RUF011.py:15:2: RUF011 Dictionary comprehension uses static key: `"key"`
    |
-11 | # Errors
-12 | {"key": value.upper() for value in data}
+14 | # Errors
+15 | {"key": value.upper() for value in data}
    |  ^^^^^ RUF011
-13 | {True: value.upper() for value in data}
-14 | {0: value.upper() for value in data}
+16 | {True: value.upper() for value in data}
+17 | {0: value.upper() for value in data}
    |
 
-RUF011.py:13:2: RUF011 Dictionary comprehension uses static key: `True`
+RUF011.py:16:2: RUF011 Dictionary comprehension uses static key: `True`
    |
-11 | # Errors
-12 | {"key": value.upper() for value in data}
-13 | {True: value.upper() for value in data}
+14 | # Errors
+15 | {"key": value.upper() for value in data}
+16 | {True: value.upper() for value in data}
    |  ^^^^ RUF011
-14 | {0: value.upper() for value in data}
-15 | {(1, "a"): value.upper() for value in data}  # Constant tuple
+17 | {0: value.upper() for value in data}
+18 | {(1, "a"): value.upper() for value in data}  # Constant tuple
    |
 
-RUF011.py:14:2: RUF011 Dictionary comprehension uses static key: `0`
+RUF011.py:17:2: RUF011 Dictionary comprehension uses static key: `0`
    |
-12 | {"key": value.upper() for value in data}
-13 | {True: value.upper() for value in data}
-14 | {0: value.upper() for value in data}
+15 | {"key": value.upper() for value in data}
+16 | {True: value.upper() for value in data}
+17 | {0: value.upper() for value in data}
    |  ^ RUF011
-15 | {(1, "a"): value.upper() for value in data}  # Constant tuple
-16 | {constant: value.upper() for value in data}
+18 | {(1, "a"): value.upper() for value in data}  # Constant tuple
+19 | {constant: value.upper() for value in data}
    |
 
-RUF011.py:15:2: RUF011 Dictionary comprehension uses static key: `(1, "a")`
+RUF011.py:18:2: RUF011 Dictionary comprehension uses static key: `(1, "a")`
    |
-13 | {True: value.upper() for value in data}
-14 | {0: value.upper() for value in data}
-15 | {(1, "a"): value.upper() for value in data}  # Constant tuple
+16 | {True: value.upper() for value in data}
+17 | {0: value.upper() for value in data}
+18 | {(1, "a"): value.upper() for value in data}  # Constant tuple
    |  ^^^^^^^^ RUF011
-16 | {constant: value.upper() for value in data}
-17 | {constant + constant: value.upper() for value in data}
+19 | {constant: value.upper() for value in data}
+20 | {constant + constant: value.upper() for value in data}
    |
 
-RUF011.py:16:2: RUF011 Dictionary comprehension uses static key: `constant`
+RUF011.py:19:2: RUF011 Dictionary comprehension uses static key: `constant`
    |
-14 | {0: value.upper() for value in data}
-15 | {(1, "a"): value.upper() for value in data}  # Constant tuple
-16 | {constant: value.upper() for value in data}
+17 | {0: value.upper() for value in data}
+18 | {(1, "a"): value.upper() for value in data}  # Constant tuple
+19 | {constant: value.upper() for value in data}
    |  ^^^^^^^^ RUF011
-17 | {constant + constant: value.upper() for value in data}
+20 | {constant + constant: value.upper() for value in data}
+21 | {constant.attribute: value.upper() for value in data}
    |
 
-RUF011.py:17:2: RUF011 Dictionary comprehension uses static key: `constant + constant`
+RUF011.py:20:2: RUF011 Dictionary comprehension uses static key: `constant + constant`
    |
-15 | {(1, "a"): value.upper() for value in data}  # Constant tuple
-16 | {constant: value.upper() for value in data}
-17 | {constant + constant: value.upper() for value in data}
+18 | {(1, "a"): value.upper() for value in data}  # Constant tuple
+19 | {constant: value.upper() for value in data}
+20 | {constant + constant: value.upper() for value in data}
    |  ^^^^^^^^^^^^^^^^^^^ RUF011
+21 | {constant.attribute: value.upper() for value in data}
+22 | {constant[0]: value.upper() for value in data}
+   |
+
+RUF011.py:21:2: RUF011 Dictionary comprehension uses static key: `constant.attribute`
+   |
+19 | {constant: value.upper() for value in data}
+20 | {constant + constant: value.upper() for value in data}
+21 | {constant.attribute: value.upper() for value in data}
+   |  ^^^^^^^^^^^^^^^^^^ RUF011
+22 | {constant[0]: value.upper() for value in data}
+   |
+
+RUF011.py:22:2: RUF011 Dictionary comprehension uses static key: `constant[0]`
+   |
+20 | {constant + constant: value.upper() for value in data}
+21 | {constant.attribute: value.upper() for value in data}
+22 | {constant[0]: value.upper() for value in data}
+   |  ^^^^^^^^^^^ RUF011
    |
 
 

crates/ruff_python_parser/src/lexer.rs

@@ -759,7 +759,7 @@ impl<'source> Lexer<'source> {
         };
 
         let tok = Tok::String {
-            value: self.source[TextRange::new(value_start, value_end)].to_string(),
+            value: TextRange::new(value_start, value_end),
             kind,
             triple_quoted,
         };

crates/ruff_python_parser/src/python.lalrpop

@@ -1605,8 +1605,8 @@ StringLiteralOrFString: StringType = {
 
 StringLiteral: StringType = {
     <location:@L> <string:string> <end_location:@R> =>? {
-        let (source, kind, triple_quoted) = string;
-        Ok(parse_string_literal(&source, kind, triple_quoted, (location..end_location).into())?)
+        let (value, kind, triple_quoted) = string;
+        Ok(parse_string_literal(&source_code[value], kind, triple_quoted, (location..end_location).into())?)
     }
 };
 
@@ -2061,7 +2061,7 @@ extern {
         float => token::Tok::Float { value: <f64> },
         complex => token::Tok::Complex { real: <f64>, imag: <f64> },
         string => token::Tok::String {
-            value: <String>,
+            value: <TextRange>,
             kind: <StringKind>,
             triple_quoted: <bool>
         },

crates/ruff_python_parser/src/python.rs

@@ -1,5 +1,5 @@
 // auto-generated: "lalrpop 0.20.0"
-// sha3: 031689e389556292d9dbd8a1b1ff8ca29bac76d83f1b345630481d620b89e1c2
+// sha3: 28f158c07e00e286b0a28fb9af14b474f60e5d67d1dd47e5dddc93a4b622c46b
 use ruff_text_size::{Ranged, TextLen, TextRange, TextSize};
 use ruff_python_ast::{self as ast, Int, IpyEscapeKind};
 use crate::{
@@ -54,7 +54,7 @@ mod __parse__Top {
         Variant4(Int),
         Variant5((IpyEscapeKind, String)),
         Variant6(String),
-        Variant7((String, StringKind, bool)),
+        Variant7((TextRange, StringKind, bool)),
         Variant8(core::option::Option<token::Tok>),
         Variant9(Option<Box<ast::Parameter>>),
         Variant10(core::option::Option<Option<Box<ast::Parameter>>>),
@@ -18373,16 +18373,6 @@ mod __parse__Top {
             _ => __symbol_type_mismatch()
         }
     }
-    fn __pop_Variant7<
-    >(
-        __symbols: &mut alloc::vec::Vec<(TextSize,__Symbol<>,TextSize)>
-    ) -> (TextSize, (String, StringKind, bool), TextSize)
-     {
-        match __symbols.pop() {
-            Some((__l, __Symbol::Variant7(__v), __r)) => (__l, __v, __r),
-            _ => __symbol_type_mismatch()
-        }
-    }
     fn __pop_Variant3<
     >(
         __symbols: &mut alloc::vec::Vec<(TextSize,__Symbol<>,TextSize)>
@@ -18393,6 +18383,16 @@ mod __parse__Top {
             _ => __symbol_type_mismatch()
         }
     }
+    fn __pop_Variant7<
+    >(
+        __symbols: &mut alloc::vec::Vec<(TextSize,__Symbol<>,TextSize)>
+    ) -> (TextSize, (TextRange, StringKind, bool), TextSize)
+     {
+        match __symbols.pop() {
+            Some((__l, __Symbol::Variant7(__v), __r)) => (__l, __v, __r),
+            _ => __symbol_type_mismatch()
+        }
+    }
     fn __pop_Variant67<
     >(
         __symbols: &mut alloc::vec::Vec<(TextSize,__Symbol<>,TextSize)>
@@ -36363,13 +36363,13 @@ fn __action217<
     source_code: &str,
     mode: Mode,
     (_, location, _): (TextSize, TextSize, TextSize),
-    (_, string, _): (TextSize, (String, StringKind, bool), TextSize),
+    (_, string, _): (TextSize, (TextRange, StringKind, bool), TextSize),
     (_, end_location, _): (TextSize, TextSize, TextSize),
 ) -> Result<StringType,__lalrpop_util::ParseError<TextSize,token::Tok,LexicalError>>
 {
     {
-        let (source, kind, triple_quoted) = string;
-        Ok(parse_string_literal(&source, kind, triple_quoted, (location..end_location).into())?)
+        let (value, kind, triple_quoted) = string;
+        Ok(parse_string_literal(&source_code[value], kind, triple_quoted, (location..end_location).into())?)
     }
 }
 
@@ -52719,7 +52719,7 @@ fn __action937<
 >(
     source_code: &str,
     mode: Mode,
-    __0: (TextSize, (String, StringKind, bool), TextSize),
+    __0: (TextSize, (TextRange, StringKind, bool), TextSize),
     __1: (TextSize, TextSize, TextSize),
 ) -> Result<StringType,__lalrpop_util::ParseError<TextSize,token::Tok,LexicalError>>
 {
@@ -69997,7 +69997,7 @@ fn __action1494<
 >(
     source_code: &str,
     mode: Mode,
-    __0: (TextSize, (String, StringKind, bool), TextSize),
+    __0: (TextSize, (TextRange, StringKind, bool), TextSize),
 ) -> Result<StringType,__lalrpop_util::ParseError<TextSize,token::Tok,LexicalError>>
 {
     let __start0 = __0.2;

crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__empty_fstrings.snap

@@ -13,7 +13,7 @@ expression: lex_source(source)
     ),
     (
         String {
-            value: "",
+            value: 5..5,
             kind: String,
             triple_quoted: false,
         },
@@ -37,7 +37,7 @@ expression: lex_source(source)
     ),
     (
         String {
-            value: "",
+            value: 16..16,
             kind: String,
             triple_quoted: false,
         },

crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__escape_unicode_name.snap

@@ -5,7 +5,7 @@ expression: lex_source(source)
 [
     (
         String {
-            value: "\\N{EN SPACE}",
+            value: 1..13,
             kind: String,
             triple_quoted: false,
         },

crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__fstring_with_format_spec.snap

@@ -137,7 +137,7 @@ expression: lex_source(source)
     ),
     (
         String {
-            value: "",
+            value: 32..32,
             kind: String,
             triple_quoted: false,
         },

crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__non_logical_newline_in_string_continuation.snap

@@ -13,7 +13,7 @@ expression: lex_source(source)
     ),
     (
         String {
-            value: "a",
+            value: 7..8,
             kind: String,
             triple_quoted: false,
         },
@@ -25,7 +25,7 @@ expression: lex_source(source)
     ),
     (
         String {
-            value: "b",
+            value: 15..16,
             kind: String,
             triple_quoted: false,
         },
@@ -41,7 +41,7 @@ expression: lex_source(source)
     ),
     (
         String {
-            value: "c",
+            value: 24..25,
             kind: String,
             triple_quoted: false,
         },
@@ -49,7 +49,7 @@ expression: lex_source(source)
     ),
     (
         String {
-            value: "d",
+            value: 34..35,
             kind: String,
             triple_quoted: false,
         },

crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__string.snap

@@ -5,7 +5,7 @@ expression: lex_source(source)
 [
     (
         String {
-            value: "double",
+            value: 1..7,
             kind: String,
             triple_quoted: false,
         },
@@ -13,7 +13,7 @@ expression: lex_source(source)
     ),
     (
         String {
-            value: "single",
+            value: 10..16,
             kind: String,
             triple_quoted: false,
         },
@@ -21,7 +21,7 @@ expression: lex_source(source)
     ),
     (
         String {
-            value: "can\\'t",
+            value: 19..25,
             kind: String,
             triple_quoted: false,
         },
@@ -29,7 +29,7 @@ expression: lex_source(source)
     ),
     (
         String {
-            value: "\\\\\\\"",
+            value: 28..32,
             kind: String,
             triple_quoted: false,
         },
@@ -37,7 +37,7 @@ expression: lex_source(source)
     ),
     (
         String {
-            value: "\\t\\r\\n",
+            value: 35..41,
             kind: String,
             triple_quoted: false,
         },
@@ -45,7 +45,7 @@ expression: lex_source(source)
     ),
     (
         String {
-            value: "\\g",
+            value: 44..46,
             kind: String,
             triple_quoted: false,
         },
@@ -53,7 +53,7 @@ expression: lex_source(source)
     ),
     (
         String {
-            value: "raw\\'",
+            value: 50..55,
             kind: RawString,
             triple_quoted: false,
         },
@@ -61,7 +61,7 @@ expression: lex_source(source)
     ),
     (
         String {
-            value: "\\420",
+            value: 58..62,
             kind: String,
             triple_quoted: false,
         },
@@ -69,7 +69,7 @@ expression: lex_source(source)
     ),
     (
         String {
-            value: "\\200\\0a",
+            value: 65..72,
             kind: String,
             triple_quoted: false,
         },

crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__string_continuation_with_mac_eol.snap

@@ -5,7 +5,7 @@ expression: string_continuation_with_eol(MAC_EOL)
 [
     (
         String {
-            value: "abc\\\rdef",
+            value: 1..9,
             kind: String,
             triple_quoted: false,
         },

crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__string_continuation_with_unix_eol.snap

@@ -5,7 +5,7 @@ expression: string_continuation_with_eol(UNIX_EOL)
 [
     (
         String {
-            value: "abc\\\ndef",
+            value: 1..9,
             kind: String,
             triple_quoted: false,
         },

crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__string_continuation_with_windows_eol.snap

@@ -5,7 +5,7 @@ expression: string_continuation_with_eol(WINDOWS_EOL)
 [
     (
         String {
-            value: "abc\\\r\ndef",
+            value: 1..10,
             kind: String,
             triple_quoted: false,
         },

crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__triple_quoted_mac_eol.snap

@@ -5,7 +5,7 @@ expression: triple_quoted_eol(MAC_EOL)
 [
     (
         String {
-            value: "\r test string\r ",
+            value: 3..18,
             kind: String,
             triple_quoted: true,
         },

crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__triple_quoted_unix_eol.snap

@@ -5,7 +5,7 @@ expression: triple_quoted_eol(UNIX_EOL)
 [
     (
         String {
-            value: "\n test string\n ",
+            value: 3..18,
             kind: String,
             triple_quoted: true,
         },

crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__triple_quoted_windows_eol.snap

@@ -5,7 +5,7 @@ expression: triple_quoted_eol(WINDOWS_EOL)
 [
     (
         String {
-            value: "\r\n test string\r\n ",
+            value: 3..20,
             kind: String,
             triple_quoted: true,
         },

crates/ruff_python_parser/src/token.rs

@@ -7,7 +7,7 @@
 use crate::Mode;
 
 use ruff_python_ast::{Int, IpyEscapeKind};
-use ruff_text_size::TextSize;
+use ruff_text_size::{TextRange, TextSize};
 use std::fmt;
 
 /// The set of tokens the Python source code can be tokenized in.
@@ -37,8 +37,8 @@ pub enum Tok {
     },
     /// Token value for a string.
     String {
-        /// The string value.
-        value: String,
+        /// The range of the string value.
+        value: TextRange,
         /// The kind of string.
         kind: StringKind,
         /// Whether the string is triple quoted.
@@ -51,6 +51,8 @@ pub enum Tok {
     /// part of the expression part and isn't an opening or closing brace.
     FStringMiddle {
         /// The string value.
+        // TODO(charlie): This could _maybe_ be a range, but we'd have to move logic into the parser
+        // to handle some escaping.
         value: String,
         /// Whether the string is raw or not.
         is_raw: bool,
@@ -241,14 +243,7 @@ impl fmt::Display for Tok {
             Int { value } => write!(f, "'{value}'"),
             Float { value } => write!(f, "'{value}'"),
             Complex { real, imag } => write!(f, "{real}j{imag}"),
-            String {
-                value,
-                kind,
-                triple_quoted,
-            } => {
-                let quotes = "\"".repeat(if *triple_quoted { 3 } else { 1 });
-                write!(f, "{kind}{quotes}{value}{quotes}")
-            }
+            String { .. } => write!(f, "String"),
             FStringStart => f.write_str("FStringStart"),
             FStringMiddle { value, .. } => f.write_str(value),
             FStringEnd => f.write_str("FStringEnd"),