Use CommentRanges in backwards lexing (#7360)
## Summary The tokenizer was split into a forward and a backwards tokenizer. The backwards tokenizer uses the same names as the forwards ones (e.g. `next_token`). The backwards tokenizer gets the comment ranges that we already built to skip comments. --------- Co-authored-by: Micha Reiser <micha@reiser.io>
This commit is contained in:
@@ -1,5 +1,5 @@
|
||||
use ruff_python_trivia::{SimpleTokenKind, SimpleTokenizer};
|
||||
use ruff_text_size::{Ranged, TextRange, TextSize};
|
||||
use ruff_python_trivia::{BackwardsTokenizer, CommentRanges, SimpleTokenKind, SimpleTokenizer};
|
||||
use ruff_text_size::{Ranged, TextLen, TextRange};
|
||||
|
||||
use crate::node::AnyNodeRef;
|
||||
use crate::ExpressionRef;
|
||||
@@ -9,6 +9,7 @@ use crate::ExpressionRef;
|
||||
pub fn parenthesized_range(
|
||||
expr: ExpressionRef,
|
||||
parent: AnyNodeRef,
|
||||
comment_ranges: &CommentRanges,
|
||||
source: &str,
|
||||
) -> Option<TextRange> {
|
||||
// If the parent is a node that brings its own parentheses, exclude the closing parenthesis
|
||||
@@ -23,7 +24,7 @@ pub fn parenthesized_range(
|
||||
// - `Tuple`: The elements of a tuple. The only risk is a single-element tuple (e.g., `(x,)`),
|
||||
// which must have a trailing comma anyway.
|
||||
let exclusive_parent_end = if parent.is_arguments() {
|
||||
parent.end() - TextSize::new(1)
|
||||
parent.end() - ")".text_len()
|
||||
} else {
|
||||
parent.end()
|
||||
};
|
||||
@@ -33,9 +34,8 @@ pub fn parenthesized_range(
|
||||
.skip_trivia()
|
||||
.take_while(|token| token.kind == SimpleTokenKind::RParen);
|
||||
|
||||
let left_tokenizer = SimpleTokenizer::up_to_without_back_comment(expr.start(), source)
|
||||
let left_tokenizer = BackwardsTokenizer::up_to(expr.start(), source, comment_ranges)
|
||||
.skip_trivia()
|
||||
.rev()
|
||||
.take_while(|token| token.kind == SimpleTokenKind::LParen);
|
||||
|
||||
// Zip closing parenthesis with opening parenthesis. The order is intentional, as testing for
|
||||
|
||||
Reference in New Issue
Block a user