Cyclic0007/ruff
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

ast

Browse Source
This commit is contained in:
Douglas Creager
2025-01-14 20:36:54 -05:00
parent ef85c682bd
commit a3b5df8a64
16 changed files with 5898 additions and 1783 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
crates/ruff_python_ast/Cargo.toml
View File

@@ -14,6 +14,7 @@ license = { workspace = true }
[dependencies]
ruff_cache = { workspace = true, optional = true }
ruff_index = { workspace = true }
ruff_macros = { workspace = true, optional = true }
ruff_python_trivia = { workspace = true }
ruff_source_file = { workspace = true }

240
crates/ruff_python_ast/generate.py
View File

@@ -67,6 +67,7 @@ class Ast:
class Group:
name: str
nodes: list[Node]
id_enum_ty: str
owned_enum_ty: str
add_suffix_to_is_methods: bool
@@ -75,6 +76,7 @@ class Group:
def __init__(self, group_name: str, group: dict[str, Any]) -> None:
self.name = group_name
self.id_enum_ty = group_name + "Id"
self.owned_enum_ty = group_name
self.ref_enum_ty = group_name + "Ref"
self.add_suffix_to_is_methods = group.get("add_suffix_to_is_methods", False)
@@ -89,12 +91,16 @@ class Group:
class Node:
name: str
variant: str
id_ty: str
ty: str
storage_field: str
def __init__(self, group: Group, node_name: str, node: dict[str, Any]) -> None:
self.name = node_name
self.variant = node.get("variant", node_name.removeprefix(group.name))
self.id_ty = node_name + "Id"
self.ty = f"crate::{node_name}"
self.storage_field = to_snake_case(node_name)
# ------------------------------------------------------------------------------
@@ -108,6 +114,75 @@ def write_preamble(out: list[str]) -> None:
""")
# ------------------------------------------------------------------------------
# ID enum
def write_ids(out: list[str], ast: Ast) -> None:
"""
Create an ID type for each syntax node, and a per-group enum that contains a
syntax node ID.
```rust
#[newindex_type]
pub struct TypeParamTypeVarId;
#[newindex_type]
pub struct TypeParamTypeVarTuple;
...
pub enum TypeParamId {
TypeVar(TypeParamTypeVarId),
TypeVarTuple(TypeParamTypeVarTupleId),
...
}
```
Also creates:
- `impl From<TypeParamTypeVarId> for TypeParamId`
- `impl Ranged for TypeParamTypeVar`
- `fn TypeParamId::is_type_var() -> bool`
If the `add_suffix_to_is_methods` group option is true, then the
`is_type_var` method will be named `is_type_var_type_param`.
"""
for node in ast.all_nodes:
out.append("")
out.append("#[ruff_index::newtype_index]")
out.append(f"pub struct {node.id_ty};")
out.append(f"""
impl ruff_text_size::Ranged for {node.ty} {{
fn range(&self) -> ruff_text_size::TextRange {{
self.range
}}
}}
""")
for group in ast.groups:
out.append("")
if group.rustdoc is not None:
out.append(group.rustdoc)
out.append("#[derive(Clone, Copy, Debug, PartialEq, is_macro::Is)]")
out.append(f"pub enum {group.id_enum_ty} {{")
for node in group.nodes:
if group.add_suffix_to_is_methods:
is_name = to_snake_case(node.variant + group.name)
out.append(f'#[is(name = "{is_name}")]')
out.append(f"{node.variant}({node.id_ty}),")
out.append("}")
for node in group.nodes:
out.append(f"""
impl From<{node.id_ty}> for {group.id_enum_ty} {{
fn from(id: {node.id_ty}) -> Self {{
Self::{node.variant}(id)
}}
}}
""")
# ------------------------------------------------------------------------------
# Owned enum
@@ -126,7 +201,6 @@ def write_owned_enum(out: list[str], ast: Ast) -> None:
Also creates:
- `impl Ranged for TypeParam`
- `TypeParam::visit_source_order`
- `impl From<TypeParamTypeVar> for TypeParam`
- `impl Ranged for TypeParamTypeVar`
- `fn TypeParam::is_type_var() -> bool`
@@ -170,15 +244,6 @@ def write_owned_enum(out: list[str], ast: Ast) -> None:
}
""")
for node in ast.all_nodes:
out.append(f"""
impl ruff_text_size::Ranged for {node.ty} {{
fn range(&self) -> ruff_text_size::TextRange {{
self.range
}}
}}
""")
for group in ast.groups:
out.append(f"""
impl {group.owned_enum_ty} {{
@@ -210,17 +275,18 @@ def write_ref_enum(out: list[str], ast: Ast) -> None:
```rust
pub enum TypeParamRef<'a> {
TypeVar(&'a TypeParamTypeVar),
TypeVarTuple(&'a TypeParamTypeVarTuple),
TypeVar(Node<'a, &'a TypeParamTypeVar>),
TypeVarTuple(Node<'a, &'a TypeParamTypeVarTuple>),
...
}
```
Also creates:
- `impl<'a> From<&'a TypeParam> for TypeParamRef<'a>`
- `impl<'a> From<&'a TypeParamTypeVar> for TypeParamRef<'a>`
- `impl<'a> From<Node<'a, &'a TypeParam>> for TypeParamRef<'a>`
- `impl<'a> From<Node<'a, &'a TypeParamTypeVar>> for TypeParamRef<'a>`
- `impl Ranged for TypeParamRef<'_>`
- `fn TypeParamRef::is_type_var() -> bool`
- `TypeParamRef::visit_source_order`
The name of each variant can be customized via the `variant` node option. If
the `add_suffix_to_is_methods` group option is true, then the `is_type_var`
@@ -237,17 +303,17 @@ def write_ref_enum(out: list[str], ast: Ast) -> None:
if group.add_suffix_to_is_methods:
is_name = to_snake_case(node.variant + group.name)
out.append(f'#[is(name = "{is_name}")]')
out.append(f"""{node.variant}(&'a {node.ty}),""")
out.append(f"""{node.variant}(crate::Node<'a, &'a {node.ty}>),""")
out.append("}")
out.append(f"""
impl<'a> From<&'a {group.owned_enum_ty}> for {group.ref_enum_ty}<'a> {{
fn from(node: &'a {group.owned_enum_ty}) -> Self {{
match node {{
impl<'a> From<crate::Node<'a, &'a {group.owned_enum_ty}>> for {group.ref_enum_ty}<'a> {{
fn from(node: crate::Node<'a, &'a {group.owned_enum_ty}>) -> Self {{
match node.node {{
""")
for node in group.nodes:
out.append(
f"{group.owned_enum_ty}::{node.variant}(node) => {group.ref_enum_ty}::{node.variant}(node),"
f"""{group.owned_enum_ty}::{node.variant}(n) => {group.ref_enum_ty}::{node.variant}(node.ast.wrap(n)),"""
)
out.append("""
}
@@ -257,8 +323,8 @@ def write_ref_enum(out: list[str], ast: Ast) -> None:
for node in group.nodes:
out.append(f"""
impl<'a> From<&'a {node.ty}> for {group.ref_enum_ty}<'a> {{
fn from(node: &'a {node.ty}) -> Self {{
impl<'a> From<crate::Node<'a, &'a {node.ty}>> for {group.ref_enum_ty}<'a> {{
fn from(node: crate::Node<'a, &'a {node.ty}>) -> Self {{
Self::{node.variant}(node)
}}
}}
@@ -277,6 +343,112 @@ def write_ref_enum(out: list[str], ast: Ast) -> None:
}
""")
for group in ast.groups:
out.append(f"""
impl<'a> {group.ref_enum_ty}<'a> {{
#[allow(unused)]
pub(crate) fn visit_source_order<V>(self, visitor: &mut V)
where
V: crate::visitor::source_order::SourceOrderVisitor<'a> + ?Sized,
{{
match self {{
""")
for node in group.nodes:
out.append(
f"""{group.ref_enum_ty}::{node.variant}(node) => node.visit_source_order(visitor),"""
)
out.append("""
}
}
}
""")
# ------------------------------------------------------------------------------
# AST storage
def write_storage(out: list[str], ast: Ast) -> None:
"""
Create the storage struct for all of the syntax nodes.
```rust
pub(crate) struct Storage {
...
pub(crate) type_param_type_var_id: IndexVec<TypeParamTypeVarId, TypeParamTypeVar>,
pub(crate) type_param_type_var_tuple_id: IndexVec<TypeParamTypeVarTupleId, TypeParamTypeVarTuple>,
...
}
```
Also creates:
- `impl AstId for TypeParamTypeVarId for Ast`
- `impl AstIdMut for TypeParamTypeVarId for Ast`
"""
out.append("")
out.append("#[derive(Clone, Default, PartialEq)]")
out.append("pub(crate) struct Storage {")
for node in ast.all_nodes:
out.append(f"""pub(crate) {node.storage_field}: ruff_index::IndexVec<{node.id_ty}, {node.ty}>,""")
out.append("}")
for node in ast.all_nodes:
out.append(f"""
impl crate::ast::AstId for {node.id_ty} {{
type Output<'a> = crate::Node<'a, &'a {node.ty}>;
#[inline]
fn node<'a>(self, ast: &'a crate::Ast) -> Self::Output<'a> {{
ast.wrap(&ast.storage.{node.storage_field}[self])
}}
}}
""")
out.append(f"""
impl crate::ast::AstIdMut for {node.id_ty} {{
type Output<'a> = crate::Node<'a, &'a mut {node.ty}>;
#[inline]
fn node_mut<'a>(self, ast: &'a mut crate::Ast) -> Self::Output<'a> {{
ast.wrap(&mut ast.storage.{node.storage_field}[self])
}}
}}
""")
out.append(f"""
impl<'a> crate::Node<'a, {node.id_ty}> {{
#[inline]
pub fn node(self) -> crate::Node<'a, &'a {node.ty}> {{
self.ast.node(self.node)
}}
}}
""")
for group in ast.groups:
out.append(f"""
impl crate::ast::AstId for {group.id_enum_ty} {{
type Output<'a> = {group.ref_enum_ty}<'a>;
#[inline]
fn node<'a>(self, ast: &'a crate::Ast) -> Self::Output<'a> {{
match self {{
""")
for node in group.nodes:
out.append(f"""{group.id_enum_ty}::{node.variant}(node) => {group.ref_enum_ty}::{node.variant}(ast.node(node)),""")
out.append(f"""
}}
}}
}}
""")
out.append(f"""
impl<'a> crate::Node<'a, {group.id_enum_ty}> {{
#[inline]
pub fn node(self) -> crate::Node<'a, &'a {node.ty}> {{
self.ast.node(self.node)
}}
}}
""")
# ------------------------------------------------------------------------------
# AnyNodeRef
@@ -289,16 +461,16 @@ def write_anynoderef(out: list[str], ast: Ast) -> None:
```rust
pub enum AnyNodeRef<'a> {
...
TypeParamTypeVar(&'a TypeParamTypeVar),
TypeParamTypeVarTuple(&'a TypeParamTypeVarTuple),
TypeParamTypeVar(Node<'a, &'a TypeParamTypeVar>),
TypeParamTypeVarTuple(Node<'a, &'a TypeParamTypeVarTuple>),
...
}
```
Also creates:
- `impl<'a> From<&'a TypeParam> for AnyNodeRef<'a>`
- `impl<'a> From<TypeParamRef<'a>> for AnyNodeRef<'a>`
- `impl<'a> From<&'a TypeParamTypeVarTuple> for AnyNodeRef<'a>`
- `impl<'a> From<Node<'a, &'a TypeParam>> for AnyNodeRef<'a>`
- `impl<'a> From<Node<'a, &'a TypeParamTypeVarTuple>> for AnyNodeRef<'a>`
- `impl Ranged for AnyNodeRef<'_>`
- `fn AnyNodeRef::as_ptr(&self) -> std::ptr::NonNull<()>`
- `fn AnyNodeRef::visit_preorder(self, visitor &mut impl SourceOrderVisitor)`
@@ -309,20 +481,20 @@ def write_anynoderef(out: list[str], ast: Ast) -> None:
pub enum AnyNodeRef<'a> {
""")
for node in ast.all_nodes:
out.append(f"""{node.name}(&'a {node.ty}),""")
out.append(f"""{node.name}(crate::Node<'a, &'a {node.ty}>),""")
out.append("""
}
""")
for group in ast.groups:
out.append(f"""
impl<'a> From<&'a {group.owned_enum_ty}> for AnyNodeRef<'a> {{
fn from(node: &'a {group.owned_enum_ty}) -> AnyNodeRef<'a> {{
match node {{
impl<'a> From<crate::Node<'a, &'a {group.owned_enum_ty}>> for AnyNodeRef<'a> {{
fn from(node: crate::Node<'a, &'a {group.owned_enum_ty}>) -> AnyNodeRef<'a> {{
match node.node {{
""")
for node in group.nodes:
out.append(
f"{group.owned_enum_ty}::{node.variant}(node) => AnyNodeRef::{node.name}(node),"
f"{group.owned_enum_ty}::{node.variant}(n) => AnyNodeRef::{node.name}(node.ast.wrap(n)),"
)
out.append("""
}
@@ -347,8 +519,8 @@ def write_anynoderef(out: list[str], ast: Ast) -> None:
for node in ast.all_nodes:
out.append(f"""
impl<'a> From<&'a {node.ty}> for AnyNodeRef<'a> {{
fn from(node: &'a {node.ty}) -> AnyNodeRef<'a> {{
impl<'a> From<crate::Node<'a, &'a {node.ty}>> for AnyNodeRef<'a> {{
fn from(node: crate::Node<'a, &'a {node.ty}>) -> AnyNodeRef<'a> {{
AnyNodeRef::{node.name}(node)
}}
}}
@@ -374,7 +546,7 @@ def write_anynoderef(out: list[str], ast: Ast) -> None:
""")
for node in ast.all_nodes:
out.append(
f"AnyNodeRef::{node.name}(node) => std::ptr::NonNull::from(*node).cast(),"
f"AnyNodeRef::{node.name}(node) => std::ptr::NonNull::from(node.as_ref()).cast(),"
)
out.append("""
}
@@ -470,8 +642,10 @@ def write_nodekind(out: list[str], ast: Ast) -> None:
def generate(ast: Ast) -> list[str]:
out = []
write_preamble(out)
write_ids(out, ast)
write_owned_enum(out, ast)
write_ref_enum(out, ast)
write_storage(out, ast)
write_anynoderef(out, ast)
write_nodekind(out, ast)
return out

92
crates/ruff_python_ast/src/ast.rs Normal file
View File

@@ -0,0 +1,92 @@
#![allow(clippy::derive_partial_eq_without_eq)]
use std::ops::{Deref, Index};
use crate as ast;
#[derive(Clone, Default, PartialEq)]
pub struct Ast {
pub(crate) storage: ast::Storage,
}
impl Ast {
#[inline]
pub fn wrap<T>(&self, node: T) -> Node<T> {
Node { ast: self, node }
}
#[inline]
pub fn node<'a, I>(&'a self, id: I) -> <I as AstId>::Output<'a>
where
I: AstId,
{
id.node(self)
}
}
impl std::fmt::Debug for Ast {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("Ast").finish()
}
}
pub trait AstId {
type Output<'a>;
fn node<'a>(self, ast: &'a Ast) -> Self::Output<'a>;
}
pub trait AstIdMut {
type Output<'a>;
fn node_mut<'a>(self, ast: &'a mut Ast) -> Self::Output<'a>;
}
#[derive(Clone, Copy)]
pub struct Node<'ast, T> {
pub ast: &'ast Ast,
pub node: T,
}
impl<T> Node<'_, T> {
pub fn as_ref(&self) -> &T {
&self.node
}
}
impl<T> std::fmt::Debug for Node<'_, T>
where
T: std::fmt::Debug,
{
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_tuple("Node").field(&self.node).finish()
}
}
impl<T> Deref for Node<'_, T> {
type Target = T;
fn deref(&self) -> &Self::Target {
&self.node
}
}
impl<T> Eq for Node<'_, T> where T: Eq {}
impl<T> std::hash::Hash for Node<'_, T>
where
T: std::hash::Hash,
{
fn hash<H>(&self, state: &mut H)
where
H: std::hash::Hasher,
{
self.node.hash(state);
}
}
impl<T> PartialEq for Node<'_, T>
where
T: PartialEq,
{
fn eq(&self, other: &Self) -> bool {
self.node == other.node
}
}

26
crates/ruff_python_ast/src/comparable.rs
View File

@@ -16,7 +16,7 @@
//! have the same shape in that they evaluate to the same value.
use crate as ast;
use crate::{Expr, Number};
use crate::{Expr, Node, Number};
use std::borrow::Cow;
use std::hash::Hash;
@@ -593,18 +593,14 @@ impl<'a> From<ast::LiteralExpressionRef<'a>> for ComparableLiteral<'a> {
match literal {
ast::LiteralExpressionRef::NoneLiteral(_) => Self::None,
ast::LiteralExpressionRef::EllipsisLiteral(_) => Self::Ellipsis,
ast::LiteralExpressionRef::BooleanLiteral(ast::ExprBooleanLiteral {
value, ..
}) => Self::Bool(value),
ast::LiteralExpressionRef::StringLiteral(ast::ExprStringLiteral { value, .. }) => {
Self::Str(value.iter().map(Into::into).collect())
ast::LiteralExpressionRef::BooleanLiteral(node) => Self::Bool(&node.value),
ast::LiteralExpressionRef::StringLiteral(node) => {
Self::Str(node.value.iter().map(Into::into).collect())
}
ast::LiteralExpressionRef::BytesLiteral(ast::ExprBytesLiteral { value, .. }) => {
Self::Bytes(value.iter().map(Into::into).collect())
}
ast::LiteralExpressionRef::NumberLiteral(ast::ExprNumberLiteral { value, .. }) => {
Self::Number(value.into())
ast::LiteralExpressionRef::BytesLiteral(node) => {
Self::Bytes(node.value.iter().map(Into::into).collect())
}
ast::LiteralExpressionRef::NumberLiteral(node) => Self::Number((&node.value).into()),
}
}
}
@@ -1437,9 +1433,9 @@ pub enum ComparableStmt<'a> {
Continue,
}
impl<'a> From<&'a ast::Stmt> for ComparableStmt<'a> {
fn from(stmt: &'a ast::Stmt) -> Self {
match stmt {
impl<'a> From<Node<'a, ast::StmtRef<'a>>> for ComparableStmt<'a> {
fn from(stmt: Node<'a, ast::StmtRef<'a>>) -> Self {
match stmt.node {
ast::Stmt::FunctionDef(ast::StmtFunctionDef {
is_async,
name,
@@ -1451,7 +1447,7 @@ impl<'a> From<&'a ast::Stmt> for ComparableStmt<'a> {
range: _,
}) => Self::FunctionDef(StmtFunctionDef {
is_async: *is_async,
name: name.as_str(),
name: stmt.name().as_str(),
parameters: parameters.into(),
body: body.iter().map(Into::into).collect(),
decorator_list: decorator_list.iter().map(Into::into).collect(),

142
crates/ruff_python_ast/src/expression.rs
View File

@@ -3,26 +3,20 @@ use std::iter::FusedIterator;
use ruff_text_size::{Ranged, TextRange};
use crate::{
self as ast, AnyNodeRef, AnyStringFlags, Expr, ExprBytesLiteral, ExprFString, ExprRef,
ExprStringLiteral, StringFlags,
self as ast, AnyNodeRef, AnyStringFlags, Ast, Expr, ExprBytesLiteral, ExprFString, ExprRef,
ExprStringLiteral, Node, StringFlags,
};
impl<'a> From<&'a Box<Expr>> for ExprRef<'a> {
fn from(value: &'a Box<Expr>) -> Self {
ExprRef::from(value.as_ref())
}
}
/// Unowned pendant to all the literal variants of [`ast::Expr`] that stores a
/// reference instead of an owned value.
#[derive(Copy, Clone, Debug, PartialEq, is_macro::Is)]
pub enum LiteralExpressionRef<'a> {
StringLiteral(&'a ast::ExprStringLiteral),
BytesLiteral(&'a ast::ExprBytesLiteral),
NumberLiteral(&'a ast::ExprNumberLiteral),
BooleanLiteral(&'a ast::ExprBooleanLiteral),
NoneLiteral(&'a ast::ExprNoneLiteral),
EllipsisLiteral(&'a ast::ExprEllipsisLiteral),
StringLiteral(Node<'a, &'a ast::ExprStringLiteral>),
BytesLiteral(Node<'a, &'a ast::ExprBytesLiteral>),
NumberLiteral(Node<'a, &'a ast::ExprNumberLiteral>),
BooleanLiteral(Node<'a, &'a ast::ExprBooleanLiteral>),
NoneLiteral(Node<'a, &'a ast::ExprNoneLiteral>),
EllipsisLiteral(Node<'a, &'a ast::ExprEllipsisLiteral>),
}
impl Ranged for LiteralExpressionRef<'_> {
@@ -83,9 +77,9 @@ impl LiteralExpressionRef<'_> {
/// literals, bytes literals, and f-strings.
#[derive(Copy, Clone, Debug, PartialEq)]
pub enum StringLike<'a> {
String(&'a ast::ExprStringLiteral),
Bytes(&'a ast::ExprBytesLiteral),
FString(&'a ast::ExprFString),
String(Node<'a, &'a ast::ExprStringLiteral>),
Bytes(Node<'a, &'a ast::ExprBytesLiteral>),
FString(Node<'a, &'a ast::ExprFString>),
}
impl<'a> StringLike<'a> {
@@ -96,18 +90,18 @@ impl<'a> StringLike<'a> {
/// Returns an iterator over the [`StringLikePart`] contained in this string-like expression.
pub fn parts(&self) -> StringLikePartIter<'a> {
match self {
StringLike::String(expr) => StringLikePartIter::String(expr.value.iter()),
StringLike::Bytes(expr) => StringLikePartIter::Bytes(expr.value.iter()),
StringLike::FString(expr) => StringLikePartIter::FString(expr.value.iter()),
StringLike::String(expr) => StringLikePartIter::String(expr.ast, expr.value.iter()),
StringLike::Bytes(expr) => StringLikePartIter::Bytes(expr.ast, expr.value.iter()),
StringLike::FString(expr) => StringLikePartIter::FString(expr.ast, expr.value.iter()),
}
}
/// Returns `true` if the string is implicitly concatenated.
pub fn is_implicit_concatenated(self) -> bool {
match self {
Self::String(ExprStringLiteral { value, .. }) => value.is_implicit_concatenated(),
Self::Bytes(ExprBytesLiteral { value, .. }) => value.is_implicit_concatenated(),
Self::FString(ExprFString { value, .. }) => value.is_implicit_concatenated(),
Self::String(node) => node.value.is_implicit_concatenated(),
Self::Bytes(node) => node.value.is_implicit_concatenated(),
Self::FString(node) => node.value.is_implicit_concatenated(),
}
}
@@ -120,26 +114,26 @@ impl<'a> StringLike<'a> {
}
}
impl<'a> From<&'a ast::ExprStringLiteral> for StringLike<'a> {
fn from(value: &'a ast::ExprStringLiteral) -> Self {
impl<'a> From<Node<'a, &'a ast::ExprStringLiteral>> for StringLike<'a> {
fn from(value: Node<'a, &'a ast::ExprStringLiteral>) -> Self {
StringLike::String(value)
}
}
impl<'a> From<&'a ast::ExprBytesLiteral> for StringLike<'a> {
fn from(value: &'a ast::ExprBytesLiteral) -> Self {
impl<'a> From<Node<'a, &'a ast::ExprBytesLiteral>> for StringLike<'a> {
fn from(value: Node<'a, &'a ast::ExprBytesLiteral>) -> Self {
StringLike::Bytes(value)
}
}
impl<'a> From<&'a ast::ExprFString> for StringLike<'a> {
fn from(value: &'a ast::ExprFString) -> Self {
impl<'a> From<Node<'a, &'a ast::ExprFString>> for StringLike<'a> {
fn from(value: Node<'a, &'a ast::ExprFString>) -> Self {
StringLike::FString(value)
}
}
impl<'a> From<&StringLike<'a>> for ExprRef<'a> {
fn from(value: &StringLike<'a>) -> Self {
impl<'a> From<StringLike<'a>> for ExprRef<'a> {
fn from(value: StringLike<'a>) -> Self {
match value {
StringLike::String(expr) => ExprRef::StringLiteral(expr),
StringLike::Bytes(expr) => ExprRef::BytesLiteral(expr),
@@ -150,12 +144,6 @@ impl<'a> From<&StringLike<'a>> for ExprRef<'a> {
impl<'a> From<StringLike<'a>> for AnyNodeRef<'a> {
fn from(value: StringLike<'a>) -> Self {
AnyNodeRef::from(&value)
}
}
impl<'a> From<&StringLike<'a>> for AnyNodeRef<'a> {
fn from(value: &StringLike<'a>) -> Self {
match value {
StringLike::String(expr) => AnyNodeRef::ExprStringLiteral(expr),
StringLike::Bytes(expr) => AnyNodeRef::ExprBytesLiteral(expr),
@@ -164,14 +152,14 @@ impl<'a> From<&StringLike<'a>> for AnyNodeRef<'a> {
}
}
impl<'a> TryFrom<&'a Expr> for StringLike<'a> {
impl<'a> TryFrom<Node<'a, &'a Expr>> for StringLike<'a> {
type Error = ();
fn try_from(value: &'a Expr) -> Result<Self, Self::Error> {
match value {
Expr::StringLiteral(value) => Ok(Self::String(value)),
Expr::BytesLiteral(value) => Ok(Self::Bytes(value)),
Expr::FString(value) => Ok(Self::FString(value)),
fn try_from(value: Node<'a, &'a Expr>) -> Result<Self, Self::Error> {
match value.node {
Expr::StringLiteral(v) => Ok(Self::String(value.ast.wrap(v))),
Expr::BytesLiteral(v) => Ok(Self::Bytes(value.ast.wrap(v))),
Expr::FString(v) => Ok(Self::FString(value.ast.wrap(v))),
_ => Err(()),
}
}
@@ -203,9 +191,9 @@ impl Ranged for StringLike<'_> {
/// An enum that holds a reference to an individual part of a string-like expression.
#[derive(Copy, Clone, Debug, PartialEq)]
pub enum StringLikePart<'a> {
String(&'a ast::StringLiteral),
Bytes(&'a ast::BytesLiteral),
FString(&'a ast::FString),
String(Node<'a, &'a ast::StringLiteral>),
Bytes(Node<'a, &'a ast::BytesLiteral>),
FString(Node<'a, &'a ast::FString>),
}
impl<'a> StringLikePart<'a> {
@@ -231,7 +219,7 @@ impl<'a> StringLikePart<'a> {
matches!(self, Self::String(_))
}
pub const fn as_string_literal(self) -> Option<&'a ast::StringLiteral> {
pub const fn as_string_literal(self) -> Option<Node<'a, &'a ast::StringLiteral>> {
match self {
StringLikePart::String(value) => Some(value),
_ => None,
@@ -243,30 +231,24 @@ impl<'a> StringLikePart<'a> {
}
}
impl<'a> From<&'a ast::StringLiteral> for StringLikePart<'a> {
fn from(value: &'a ast::StringLiteral) -> Self {
impl<'a> From<Node<'a, &'a ast::StringLiteral>> for StringLikePart<'a> {
fn from(value: Node<'a, &'a ast::StringLiteral>) -> Self {
StringLikePart::String(value)
}
}
impl<'a> From<&'a ast::BytesLiteral> for StringLikePart<'a> {
fn from(value: &'a ast::BytesLiteral) -> Self {
impl<'a> From<Node<'a, &'a ast::BytesLiteral>> for StringLikePart<'a> {
fn from(value: Node<'a, &'a ast::BytesLiteral>) -> Self {
StringLikePart::Bytes(value)
}
}
impl<'a> From<&'a ast::FString> for StringLikePart<'a> {
fn from(value: &'a ast::FString) -> Self {
impl<'a> From<Node<'a, &'a ast::FString>> for StringLikePart<'a> {
fn from(value: Node<'a, &'a ast::FString>) -> Self {
StringLikePart::FString(value)
}
}
impl<'a> From<&StringLikePart<'a>> for AnyNodeRef<'a> {
fn from(value: &StringLikePart<'a>) -> Self {
AnyNodeRef::from(*value)
}
}
impl<'a> From<StringLikePart<'a>> for AnyNodeRef<'a> {
fn from(value: StringLikePart<'a>) -> Self {
match value {
@@ -292,9 +274,9 @@ impl Ranged for StringLikePart<'_> {
/// This is created by the [`StringLike::parts`] method.
#[derive(Clone)]
pub enum StringLikePartIter<'a> {
String(std::slice::Iter<'a, ast::StringLiteral>),
Bytes(std::slice::Iter<'a, ast::BytesLiteral>),
FString(std::slice::Iter<'a, ast::FStringPart>),
String(&'a Ast, std::slice::Iter<'a, ast::StringLiteral>),
Bytes(&'a Ast, std::slice::Iter<'a, ast::BytesLiteral>),
FString(&'a Ast, std::slice::Iter<'a, ast::FStringPart>),
}
impl<'a> Iterator for StringLikePartIter<'a> {
@@ -302,15 +284,19 @@ impl<'a> Iterator for StringLikePartIter<'a> {
fn next(&mut self) -> Option<Self::Item> {
let part = match self {
StringLikePartIter::String(inner) => StringLikePart::String(inner.next()?),
StringLikePartIter::Bytes(inner) => StringLikePart::Bytes(inner.next()?),
StringLikePartIter::FString(inner) => {
StringLikePartIter::String(ast, inner) => {
StringLikePart::String(ast.wrap(inner.next()?))
}
StringLikePartIter::Bytes(ast, inner) => StringLikePart::Bytes(ast.wrap(inner.next()?)),
StringLikePartIter::FString(ast, inner) => {
let part = inner.next()?;
match part {
ast::FStringPart::Literal(string_literal) => {
StringLikePart::String(string_literal)
StringLikePart::String(ast.wrap(string_literal))
}
ast::FStringPart::FString(f_string) => {
StringLikePart::FString(ast.wrap(f_string))
}
ast::FStringPart::FString(f_string) => StringLikePart::FString(f_string),
}
}
};
@@ -320,9 +306,9 @@ impl<'a> Iterator for StringLikePartIter<'a> {
fn size_hint(&self) -> (usize, Option<usize>) {
match self {
StringLikePartIter::String(inner) => inner.size_hint(),
StringLikePartIter::Bytes(inner) => inner.size_hint(),
StringLikePartIter::FString(inner) => inner.size_hint(),
StringLikePartIter::String(_, inner) => inner.size_hint(),
StringLikePartIter::Bytes(_, inner) => inner.size_hint(),
StringLikePartIter::FString(_, inner) => inner.size_hint(),
}
}
}
@@ -330,15 +316,21 @@ impl<'a> Iterator for StringLikePartIter<'a> {
impl DoubleEndedIterator for StringLikePartIter<'_> {
fn next_back(&mut self) -> Option<Self::Item> {
let part = match self {
StringLikePartIter::String(inner) => StringLikePart::String(inner.next_back()?),
StringLikePartIter::Bytes(inner) => StringLikePart::Bytes(inner.next_back()?),
StringLikePartIter::FString(inner) => {
StringLikePartIter::String(ast, inner) => {
StringLikePart::String(ast.wrap(inner.next_back()?))
}
StringLikePartIter::Bytes(ast, inner) => {
StringLikePart::Bytes(ast.wrap(inner.next_back()?))
}
StringLikePartIter::FString(ast, inner) => {
let part = inner.next_back()?;
match part {
ast::FStringPart::Literal(string_literal) => {
StringLikePart::String(string_literal)
StringLikePart::String(ast.wrap(string_literal))
}
ast::FStringPart::FString(f_string) => {
StringLikePart::FString(ast.wrap(f_string))
}
ast::FStringPart::FString(f_string) => StringLikePart::FString(f_string),
}
}
};

6116
crates/ruff_python_ast/src/generated.rs generated
View File

File diff suppressed because it is too large Load Diff

2
crates/ruff_python_ast/src/lib.rs
View File

@@ -1,11 +1,13 @@
use std::ffi::OsStr;
use std::path::Path;
pub use ast::{Ast, Node};
pub use expression::*;
pub use generated::*;
pub use int::*;
pub use nodes::*;
pub mod ast;
pub mod comparable;
pub mod docstrings;
mod expression;

546
crates/ruff_python_ast/src/node.rs
View File

File diff suppressed because it is too large Load Diff

89
crates/ruff_python_ast/src/nodes.rs
View File

@@ -13,28 +13,90 @@ use itertools::Itertools;
use ruff_text_size::{Ranged, TextLen, TextRange, TextSize};
use crate::ast::AstId;
use crate::name::Name;
use crate::{
int,
str::Quote,
str_prefix::{AnyStringPrefix, ByteStringPrefix, FStringPrefix, StringLiteralPrefix},
ExceptHandler, Expr, FStringElement, LiteralExpressionRef, Pattern, Stmt, TypeParam,
Ast, DecoratorId, ExceptHandler, Expr, ExprId, FStringElement, IdentifierId,
LiteralExpressionRef, Node, ParametersId, Pattern, Stmt, StmtId, TypeParam, TypeParamsId,
};
macro_rules! accessor {
($ty:ty, $field:ident, $ref_ty:ty) => {
impl $ty {
pub fn $field<'a>(&'a self, ast: &'a Ast) -> Node<'a, <$ref_ty as AstId>::Output<'a>> {
ast.wrap(ast.node(self.$field))
}
}
impl<'a> Node<'a, &'a $ty> {
pub fn $field(self) -> Node<'a, <$ref_ty as AstId>::Output<'a>> {
self.node.$field(self.ast)
}
}
};
}
macro_rules! option_accessor {
($ty:ty, $field:ident, $ref_ty:ty) => {
impl $ty {
pub fn $field<'a>(
&'a self,
ast: &'a Ast,
) -> Option<Node<'a, <$ref_ty as AstId>::Output<'a>>> {
self.$field.map(|id| ast.wrap(ast.node(*id)))
}
}
impl<'a> Node<'a, &'a $ty> {
pub fn $field(self) -> Option<Node<'a, <$ref_ty as AstId>::Output<'a>>> {
self.node.$field(self.ast)
}
}
};
}
macro_rules! vec_accessor {
($ty:ty, $field:ident, $ref_ty:ty) => {
impl $ty {
pub fn $field<'a>(
&'a self,
ast: &'a Ast,
) -> impl Iterator<Item = Node<'a, <$ref_ty as AstId>::Output<'a>>> + 'a {
self.$field.iter().map(|id| ast.wrap(ast.node(*id)))
}
}
impl<'a> Node<'a, &'a $ty> {
pub fn $field(
self,
) -> impl Iterator<Item = Node<'a, <$ref_ty as AstId>::Output<'a>>> + 'a {
self.node.$field(self.ast)
}
}
};
}
/// See also [Module](https://docs.python.org/3/library/ast.html#ast.Module)
#[derive(Clone, Debug, PartialEq)]
pub struct ModModule {
pub range: TextRange,
pub body: Vec<Stmt>,
pub body: Vec<StmtId>,
}
vec_accessor!(ModModule, body, StmtId);
/// See also [Expression](https://docs.python.org/3/library/ast.html#ast.Expression)
#[derive(Clone, Debug, PartialEq)]
pub struct ModExpression {
pub range: TextRange,
pub body: Box<Expr>,
pub body: ExprId,
}
accessor!(ModExpression, body, ExprId);
/// An AST node used to represent a IPython escape command at the statement level.
///
/// For example,
@@ -104,20 +166,27 @@ pub struct StmtIpyEscapeCommand {
pub struct StmtFunctionDef {
pub range: TextRange,
pub is_async: bool,
pub decorator_list: Vec<Decorator>,
pub name: Identifier,
pub type_params: Option<Box<TypeParams>>,
pub parameters: Box<Parameters>,
pub returns: Option<Box<Expr>>,
pub body: Vec<Stmt>,
pub decorator_list: Vec<DecoratorId>,
pub name: IdentifierId,
pub type_params: Option<TypeParamsId>,
pub parameters: ParametersId,
pub returns: Option<ExprId>,
pub body: Vec<StmtId>,
}
vec_accessor!(StmtFunctionDef, decorator_list, DecoratorId);
accessor!(StmtFunctionDef, name, IdentifierId);
option_accessor!(StmtFunctionDef, type_params, TypeParamsId);
accessor!(StmtFunctionDef, parameters, ParametersId);
option_accessor!(StmtFunctionDef, returns, ExprId);
vec_accessor!(StmtFunctionDef, body, StmtId);
/// See also [ClassDef](https://docs.python.org/3/library/ast.html#ast.ClassDef)
#[derive(Clone, Debug, PartialEq)]
pub struct StmtClassDef {
pub range: TextRange,
pub decorator_list: Vec<Decorator>,
pub name: Identifier,
pub name: IdentifierId,
pub type_params: Option<Box<TypeParams>>,
pub arguments: Option<Box<Arguments>>,
pub body: Vec<Stmt>,

12
crates/ruff_python_ast/src/visitor/source_order.rs
View File

@@ -1,9 +1,9 @@
use crate::AnyNodeRef;
use crate::{
Alias, Arguments, BoolOp, BytesLiteral, CmpOp, Comprehension, Decorator, ElifElseClause,
ExceptHandler, Expr, FString, FStringElement, Keyword, MatchCase, Mod, Operator, Parameter,
ParameterWithDefault, Parameters, Pattern, PatternArguments, PatternKeyword, Singleton, Stmt,
StringLiteral, TypeParam, TypeParams, UnaryOp, WithItem,
ExceptHandler, Expr, FString, FStringElement, Keyword, MatchCase, Mod, ModId, Node, Operator,
Parameter, ParameterWithDefault, Parameters, Pattern, PatternArguments, PatternKeyword,
Singleton, Stmt, StringLiteral, TypeParam, TypeParams, UnaryOp, WithItem,
};
/// Visitor that traverses all nodes recursively in the order they appear in the source.
@@ -20,8 +20,8 @@ pub trait SourceOrderVisitor<'a> {
fn leave_node(&mut self, _node: AnyNodeRef<'a>) {}
#[inline]
fn visit_mod(&mut self, module: &'a Mod) {
walk_module(self, module);
fn visit_mod(&mut self, module: Node<'a, ModId>) {
walk_module(self, module.node());
}
#[inline]
@@ -172,7 +172,7 @@ pub trait SourceOrderVisitor<'a> {
}
}
pub fn walk_module<'a, V>(visitor: &mut V, module: &'a Mod)
pub fn walk_module<'a, V>(visitor: &mut V, module: Node<'a, &'a Mod>)
where
V: SourceOrderVisitor<'a> + ?Sized,
{