Implement template strings (#17851)

This PR implements template strings (t-strings) in the parser and
formatter for Ruff.

Minimal changes necessary to compile were made in other parts of the code (e.g. ty, the linter, etc.). These will be covered properly in follow-up PRs.

crates/ruff_python_formatter/src/other/f_string.rs

@@ -1,12 +1,9 @@
-use ruff_formatter::write;
-use ruff_python_ast::{AnyStringFlags, FString, StringFlags};
-use ruff_source_file::LineRanges;
-use ruff_text_size::Ranged;
-
+use super::interpolated_string_element::FormatInterpolatedStringElement;
+use crate::other::interpolated_string::{InterpolatedStringContext, InterpolatedStringLayout};
 use crate::prelude::*;
 use crate::string::{StringNormalizer, StringQuotes};
-
-use super::f_string_element::FormatFStringElement;
+use ruff_formatter::write;
+use ruff_python_ast::{FString, StringFlags};
 
 /// Formats an f-string which is part of a larger f-string expression.
 ///
@@ -21,9 +18,12 @@ impl FormatNodeRule<FString> for FormatFString {
 
         let string_kind = normalizer.choose_quotes(item.into()).flags();
 
-        let context = FStringContext::new(
+        let context = InterpolatedStringContext::new(
             string_kind,
-            FStringLayout::from_f_string(item, f.context().source()),
+            InterpolatedStringLayout::from_interpolated_string_elements(
+                &item.elements,
+                f.context().source(),
+            ),
         );
 
         // Starting prefix and quote
@@ -31,78 +31,10 @@ impl FormatNodeRule<FString> for FormatFString {
         write!(f, [string_kind.prefix(), quotes])?;
 
         for element in &item.elements {
-            FormatFStringElement::new(element, context).fmt(f)?;
+            FormatInterpolatedStringElement::new(element, context).fmt(f)?;
         }
 
         // Ending quote
         quotes.fmt(f)
     }
 }
-
-#[derive(Clone, Copy, Debug)]
-pub(crate) struct FStringContext {
-    /// The string flags of the enclosing f-string part.
-    enclosing_flags: AnyStringFlags,
-    layout: FStringLayout,
-}
-
-impl FStringContext {
-    pub(crate) const fn new(flags: AnyStringFlags, layout: FStringLayout) -> Self {
-        Self {
-            enclosing_flags: flags,
-            layout,
-        }
-    }
-
-    pub(crate) fn flags(self) -> AnyStringFlags {
-        self.enclosing_flags
-    }
-
-    pub(crate) const fn layout(self) -> FStringLayout {
-        self.layout
-    }
-}
-
-#[derive(Copy, Clone, Debug)]
-pub(crate) enum FStringLayout {
-    /// Original f-string is flat.
-    /// Don't break expressions to keep the string flat.
-    Flat,
-    /// Original f-string has multiline expressions in the replacement fields.
-    /// Allow breaking expressions across multiple lines.
-    Multiline,
-}
-
-impl FStringLayout {
-    pub(crate) fn from_f_string(f_string: &FString, source: &str) -> Self {
-        // Heuristic: Allow breaking the f-string expressions across multiple lines
-        // only if there already is at least one multiline expression. This puts the
-        // control in the hands of the user to decide if they want to break the
-        // f-string expressions across multiple lines or not. This is similar to
-        // how Prettier does it for template literals in JavaScript.
-        //
-        // If it's single quoted f-string and it contains a multiline expression, then we
-        // assume that the target version of Python supports it (3.12+). If there are comments
-        // used in any of the expression of the f-string, then it's always going to be multiline
-        // and we assume that the target version of Python supports it (3.12+).
-        //
-        // Reference: https://prettier.io/docs/en/next/rationale.html#template-literals
-        if f_string
-            .elements
-            .expressions()
-            .any(|expr| source.contains_line_break(expr.range()))
-        {
-            Self::Multiline
-        } else {
-            Self::Flat
-        }
-    }
-
-    pub(crate) const fn is_flat(self) -> bool {
-        matches!(self, FStringLayout::Flat)
-    }
-
-    pub(crate) const fn is_multiline(self) -> bool {
-        matches!(self, FStringLayout::Multiline)
-    }
-}

crates/ruff_python_formatter/src/other/interpolated_string.rs

@@ -0,0 +1,73 @@
+use ruff_python_ast::{AnyStringFlags, InterpolatedStringElements};
+use ruff_source_file::LineRanges;
+use ruff_text_size::Ranged;
+
+#[derive(Clone, Copy, Debug)]
+pub(crate) struct InterpolatedStringContext {
+    /// The string flags of the enclosing f/t-string part.
+    enclosing_flags: AnyStringFlags,
+    layout: InterpolatedStringLayout,
+}
+
+impl InterpolatedStringContext {
+    pub(crate) const fn new(flags: AnyStringFlags, layout: InterpolatedStringLayout) -> Self {
+        Self {
+            enclosing_flags: flags,
+            layout,
+        }
+    }
+
+    pub(crate) fn flags(self) -> AnyStringFlags {
+        self.enclosing_flags
+    }
+
+    pub(crate) const fn layout(self) -> InterpolatedStringLayout {
+        self.layout
+    }
+}
+
+#[derive(Copy, Clone, Debug)]
+pub(crate) enum InterpolatedStringLayout {
+    /// Original f/t-string is flat.
+    /// Don't break expressions to keep the string flat.
+    Flat,
+    /// Original f/t-string has multiline expressions in the replacement fields.
+    /// Allow breaking expressions across multiple lines.
+    Multiline,
+}
+
+impl InterpolatedStringLayout {
+    // Heuristic: Allow breaking the f/t-string expressions across multiple lines
+    // only if there already is at least one multiline expression. This puts the
+    // control in the hands of the user to decide if they want to break the
+    // f/t-string expressions across multiple lines or not. This is similar to
+    // how Prettier does it for template literals in JavaScript.
+    //
+    // If it's single quoted f-string and it contains a multiline expression, then we
+    // assume that the target version of Python supports it (3.12+). If there are comments
+    // used in any of the expression of the f-string, then it's always going to be multiline
+    // and we assume that the target version of Python supports it (3.12+).
+    //
+    // Reference: https://prettier.io/docs/en/next/rationale.html#template-literals
+    pub(crate) fn from_interpolated_string_elements(
+        elements: &InterpolatedStringElements,
+        source: &str,
+    ) -> Self {
+        if elements
+            .interpolations()
+            .any(|expr| source.contains_line_break(expr.range()))
+        {
+            Self::Multiline
+        } else {
+            Self::Flat
+        }
+    }
+
+    pub(crate) const fn is_flat(self) -> bool {
+        matches!(self, InterpolatedStringLayout::Flat)
+    }
+
+    pub(crate) const fn is_multiline(self) -> bool {
+        matches!(self, InterpolatedStringLayout::Multiline)
+    }
+}

crates/ruff_python_formatter/src/other/interpolated_string_element.rs

@@ -2,42 +2,47 @@ use std::borrow::Cow;
 
 use ruff_formatter::{Buffer, RemoveSoftLinesBuffer, format_args, write};
 use ruff_python_ast::{
-    AnyStringFlags, ConversionFlag, Expr, FStringElement, FStringExpressionElement,
-    FStringLiteralElement, StringFlags,
+    AnyStringFlags, ConversionFlag, Expr, InterpolatedElement, InterpolatedStringElement,
+    InterpolatedStringLiteralElement, StringFlags,
 };
 use ruff_text_size::{Ranged, TextSlice};
 
 use crate::comments::{dangling_open_parenthesis_comments, trailing_comments};
-use crate::context::{FStringState, NodeLevel, WithFStringState, WithNodeLevel};
+use crate::context::{
+    InterpolatedStringState, NodeLevel, WithInterpolatedStringState, WithNodeLevel,
+};
 use crate::expression::left_most;
 use crate::prelude::*;
 use crate::string::normalize_string;
 use crate::verbatim::verbatim_text;
 
-use super::f_string::FStringContext;
+use super::interpolated_string::InterpolatedStringContext;
 
 /// Formats an f-string element which is either a literal or a formatted expression.
 ///
 /// This delegates the actual formatting to the appropriate formatter.
-pub(crate) struct FormatFStringElement<'a> {
-    element: &'a FStringElement,
-    context: FStringContext,
+pub(crate) struct FormatInterpolatedStringElement<'a> {
+    element: &'a InterpolatedStringElement,
+    context: InterpolatedStringContext,
 }
 
-impl<'a> FormatFStringElement<'a> {
-    pub(crate) fn new(element: &'a FStringElement, context: FStringContext) -> Self {
+impl<'a> FormatInterpolatedStringElement<'a> {
+    pub(crate) fn new(
+        element: &'a InterpolatedStringElement,
+        context: InterpolatedStringContext,
+    ) -> Self {
         Self { element, context }
     }
 }
 
-impl Format<PyFormatContext<'_>> for FormatFStringElement<'_> {
+impl Format<PyFormatContext<'_>> for FormatInterpolatedStringElement<'_> {
     fn fmt(&self, f: &mut PyFormatter) -> FormatResult<()> {
         match self.element {
-            FStringElement::Literal(string_literal) => {
+            InterpolatedStringElement::Literal(string_literal) => {
                 FormatFStringLiteralElement::new(string_literal, self.context.flags()).fmt(f)
             }
-            FStringElement::Expression(expression) => {
-                FormatFStringExpressionElement::new(expression, self.context).fmt(f)
+            InterpolatedStringElement::Interpolation(expression) => {
+                FormatInterpolatedElement::new(expression, self.context).fmt(f)
             }
         }
     }
@@ -45,13 +50,16 @@ impl Format<PyFormatContext<'_>> for FormatFStringElement<'_> {
 
 /// Formats an f-string literal element.
 pub(crate) struct FormatFStringLiteralElement<'a> {
-    element: &'a FStringLiteralElement,
+    element: &'a InterpolatedStringLiteralElement,
     /// Flags of the enclosing F-string part
     fstring_flags: AnyStringFlags,
 }
 
 impl<'a> FormatFStringLiteralElement<'a> {
-    pub(crate) fn new(element: &'a FStringLiteralElement, fstring_flags: AnyStringFlags) -> Self {
+    pub(crate) fn new(
+        element: &'a InterpolatedStringLiteralElement,
+        fstring_flags: AnyStringFlags,
+    ) -> Self {
         Self {
             element,
             fstring_flags,
@@ -72,16 +80,16 @@ impl Format<PyFormatContext<'_>> for FormatFStringLiteralElement<'_> {
 
 /// Context representing an f-string expression element.
 #[derive(Clone, Copy, Debug)]
-pub(crate) struct FStringExpressionElementContext {
+pub(crate) struct InterpolatedElementContext {
     /// The context of the parent f-string containing this expression element.
-    parent_context: FStringContext,
+    parent_context: InterpolatedStringContext,
     /// Indicates whether this expression element has format specifier or not.
     has_format_spec: bool,
 }
 
-impl FStringExpressionElementContext {
-    /// Returns the [`FStringContext`] containing this expression element.
-    pub(crate) fn f_string(self) -> FStringContext {
+impl InterpolatedElementContext {
+    /// Returns the [`InterpolatedStringContext`] containing this expression element.
+    pub(crate) fn interpolated_string(self) -> InterpolatedStringContext {
         self.parent_context
     }
 
@@ -113,16 +121,19 @@ impl FStringExpressionElementContext {
 }
 
 /// Formats an f-string expression element.
-pub(crate) struct FormatFStringExpressionElement<'a> {
-    element: &'a FStringExpressionElement,
-    context: FStringExpressionElementContext,
+pub(crate) struct FormatInterpolatedElement<'a> {
+    element: &'a InterpolatedElement,
+    context: InterpolatedElementContext,
 }
 
-impl<'a> FormatFStringExpressionElement<'a> {
-    pub(crate) fn new(element: &'a FStringExpressionElement, context: FStringContext) -> Self {
+impl<'a> FormatInterpolatedElement<'a> {
+    pub(crate) fn new(
+        element: &'a InterpolatedElement,
+        context: InterpolatedStringContext,
+    ) -> Self {
         Self {
             element,
-            context: FStringExpressionElementContext {
+            context: InterpolatedElementContext {
                 parent_context: context,
                 has_format_spec: element.format_spec.is_some(),
             },
@@ -130,9 +141,9 @@ impl<'a> FormatFStringExpressionElement<'a> {
     }
 }
 
-impl Format<PyFormatContext<'_>> for FormatFStringExpressionElement<'_> {
+impl Format<PyFormatContext<'_>> for FormatInterpolatedElement<'_> {
     fn fmt(&self, f: &mut PyFormatter) -> FormatResult<()> {
-        let FStringExpressionElement {
+        let InterpolatedElement {
             expression,
             debug_text,
             conversion,
@@ -214,8 +225,8 @@ impl Format<PyFormatContext<'_>> for FormatFStringExpressionElement<'_> {
 
             let item = format_with(|f: &mut PyFormatter| {
                 // Update the context to be inside the f-string expression element.
-                let f = &mut WithFStringState::new(
-                    FStringState::InsideExpressionElement(self.context),
+                let f = &mut WithInterpolatedStringState::new(
+                    InterpolatedStringState::InsideInterpolatedElement(self.context),
                     f,
                 );
 
@@ -233,7 +244,11 @@ impl Format<PyFormatContext<'_>> for FormatFStringExpressionElement<'_> {
                     token(":").fmt(f)?;
 
                     for element in &format_spec.elements {
-                        FormatFStringElement::new(element, self.context.f_string()).fmt(f)?;
+                        FormatInterpolatedStringElement::new(
+                            element,
+                            self.context.interpolated_string(),
+                        )
+                        .fmt(f)?;
                     }
 
                     // These trailing comments can only occur if the format specifier is

crates/ruff_python_formatter/src/other/mod.rs

@@ -7,12 +7,14 @@ pub(crate) mod decorator;
 pub(crate) mod elif_else_clause;
 pub(crate) mod except_handler_except_handler;
 pub(crate) mod f_string;
-pub(crate) mod f_string_element;
 pub(crate) mod identifier;
+pub(crate) mod interpolated_string;
+pub(crate) mod interpolated_string_element;
 pub(crate) mod keyword;
 pub(crate) mod match_case;
 pub(crate) mod parameter;
 pub(crate) mod parameter_with_default;
 pub(crate) mod parameters;
 pub(crate) mod string_literal;
+pub(crate) mod t_string;
 pub(crate) mod with_item;

crates/ruff_python_formatter/src/other/t_string.rs

@@ -0,0 +1,40 @@
+use super::interpolated_string_element::FormatInterpolatedStringElement;
+use crate::other::interpolated_string::{InterpolatedStringContext, InterpolatedStringLayout};
+use crate::prelude::*;
+use crate::string::{StringNormalizer, StringQuotes};
+use ruff_formatter::write;
+use ruff_python_ast::{StringFlags, TString};
+
+/// Formats a t-string which is part of a larger t-string expression.
+///
+/// For example, this would be used to format the t-string part in `"foo" t"bar {x}"`
+/// or the standalone t-string in `t"foo {x} bar"`.
+#[derive(Default)]
+pub struct FormatTString;
+
+impl FormatNodeRule<TString> for FormatTString {
+    fn fmt_fields(&self, item: &TString, f: &mut PyFormatter) -> FormatResult<()> {
+        let normalizer = StringNormalizer::from_context(f.context());
+
+        let string_kind = normalizer.choose_quotes(item.into()).flags();
+
+        let context = InterpolatedStringContext::new(
+            string_kind,
+            InterpolatedStringLayout::from_interpolated_string_elements(
+                &item.elements,
+                f.context().source(),
+            ),
+        );
+
+        // Starting prefix and quote
+        let quotes = StringQuotes::from(string_kind);
+        write!(f, [string_kind.prefix(), quotes])?;
+
+        for element in &item.elements {
+            FormatInterpolatedStringElement::new(element, context).fmt(f)?;
+        }
+
+        // Ending quote
+        quotes.fmt(f)
+    }
+}