[ty] AST garbage collection (#18482)
## Summary Garbage collect ASTs once we are done checking a given file. Queries with a cross-file dependency on the AST will reparse the file on demand. This reduces ty's peak memory usage by ~20-30%. The primary change of this PR is adding a `node_index` field to every AST node, that is assigned by the parser. `ParsedModule` can use this to create a flat index of AST nodes any time the file is parsed (or reparsed). This allows `AstNodeRef` to simply index into the current instance of the `ParsedModule`, instead of storing a pointer directly. The indices are somewhat hackily (using an atomic integer) assigned by the `parsed_module` query instead of by the parser directly. Assigning the indices in source-order in the (recursive) parser turns out to be difficult, and collecting the nodes during semantic indexing is impossible as `SemanticIndex` does not hold onto a specific `ParsedModuleRef`, which the pointers in the flat AST are tied to. This means that we have to do an extra AST traversal to assign and collect the nodes into a flat index, but the small performance impact (~3% on cold runs) seems worth it for the memory savings. Part of https://github.com/astral-sh/ty/issues/214.
This commit is contained in:
@@ -184,7 +184,7 @@ mod tests {
|
||||
use insta::assert_debug_snapshot;
|
||||
|
||||
use ruff_formatter::SourceCode;
|
||||
use ruff_python_ast::AnyNodeRef;
|
||||
use ruff_python_ast::{AnyNodeRef, AtomicNodeIndex};
|
||||
use ruff_python_ast::{StmtBreak, StmtContinue};
|
||||
use ruff_python_trivia::{CommentLinePosition, CommentRanges};
|
||||
use ruff_text_size::{TextRange, TextSize};
|
||||
@@ -196,10 +196,12 @@ mod tests {
|
||||
fn debug() {
|
||||
let continue_statement = StmtContinue {
|
||||
range: TextRange::new(TextSize::new(18), TextSize::new(26)),
|
||||
node_index: AtomicNodeIndex::dummy(),
|
||||
};
|
||||
|
||||
let break_statement = StmtBreak {
|
||||
range: TextRange::new(TextSize::new(55), TextSize::new(60)),
|
||||
node_index: AtomicNodeIndex::dummy(),
|
||||
};
|
||||
|
||||
let source = r"# leading comment
|
||||
|
||||
@@ -53,6 +53,7 @@ impl<'a> From<AnyNodeRef<'a>> for NodeRefEqualityKey<'a> {
|
||||
mod tests {
|
||||
use crate::comments::node_key::NodeRefEqualityKey;
|
||||
use ruff_python_ast::AnyNodeRef;
|
||||
use ruff_python_ast::AtomicNodeIndex;
|
||||
use ruff_python_ast::StmtContinue;
|
||||
use ruff_text_size::TextRange;
|
||||
use std::collections::hash_map::DefaultHasher;
|
||||
@@ -68,6 +69,7 @@ mod tests {
|
||||
fn equality() {
|
||||
let continue_statement = StmtContinue {
|
||||
range: TextRange::default(),
|
||||
node_index: AtomicNodeIndex::dummy(),
|
||||
};
|
||||
|
||||
let ref_a = NodeRefEqualityKey::from_ref(AnyNodeRef::from(&continue_statement));
|
||||
@@ -81,6 +83,7 @@ mod tests {
|
||||
fn inequality() {
|
||||
let continue_statement = StmtContinue {
|
||||
range: TextRange::default(),
|
||||
node_index: AtomicNodeIndex::dummy(),
|
||||
};
|
||||
|
||||
let boxed = Box::new(continue_statement.clone());
|
||||
|
||||
@@ -1067,6 +1067,7 @@ fn handle_slice_comments<'a>(
|
||||
) -> CommentPlacement<'a> {
|
||||
let ast::ExprSlice {
|
||||
range: _,
|
||||
node_index: _,
|
||||
lower,
|
||||
upper,
|
||||
step,
|
||||
@@ -1450,6 +1451,7 @@ fn handle_expr_if_comment<'a>(
|
||||
) -> CommentPlacement<'a> {
|
||||
let ast::ExprIf {
|
||||
range: _,
|
||||
node_index: _,
|
||||
test,
|
||||
body,
|
||||
orelse,
|
||||
|
||||
Reference in New Issue
Block a user