Build CommentRanges outside the parser (#11792)

## Summary

This PR updates the parser to remove building the `CommentRanges` and
instead it'll be built by the linter and the formatter when it's
required.

For the linter, it'll be built and owned by the `Indexer` while for the
formatter it'll be built from the `Tokens` struct and passed as an
argument.

## Test Plan

`cargo insta test`
This commit is contained in:
Dhruv Manilawala
2024-06-09 15:25:17 +05:30
committed by GitHub
parent 7509a48eab
commit 549cc1e437
28 changed files with 151 additions and 102 deletions

View File

@@ -1,7 +1,7 @@
use insta::assert_debug_snapshot;
use ruff_python_parser::{parse_unchecked, Mode};
use ruff_python_trivia::{lines_after, lines_before, SimpleToken, SimpleTokenizer};
use ruff_python_trivia::{lines_after, lines_before, CommentRanges, SimpleToken, SimpleTokenizer};
use ruff_python_trivia::{BackwardsTokenizer, SimpleTokenKind};
use ruff_text_size::{TextLen, TextRange, TextSize};
@@ -23,7 +23,8 @@ impl TokenizationTestCase {
fn tokenize_reverse(&self) -> Vec<SimpleToken> {
let parsed = parse_unchecked(self.source, Mode::Module);
BackwardsTokenizer::new(self.source, self.range, parsed.comment_ranges()).collect()
let comment_ranges = CommentRanges::from(parsed.tokens());
BackwardsTokenizer::new(self.source, self.range, &comment_ranges).collect()
}
fn tokens(&self) -> &[SimpleToken] {