From 40ab5d353be5937a692be7d757b2d1eb1be150a7 Mon Sep 17 00:00:00 2001 From: Charlie Marsh Date: Sat, 1 Oct 2022 16:36:11 -0400 Subject: [PATCH] Try out tree-sitter --- .gitmodules | 3 + Cargo.lock | 23 +++++++ Cargo.toml | 6 ++ build.rs | 28 +++++++++ examples/parse_python.rs | 130 +++++++++++++++++++++++++++++++++++++++ tree-sitter-python | 1 + 6 files changed, 191 insertions(+) create mode 100644 .gitmodules create mode 100644 build.rs create mode 100644 examples/parse_python.rs create mode 160000 tree-sitter-python diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000000..b548f0248a --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "tree-sitter-python"] + path = tree-sitter-python + url = https://github.com/tree-sitter/tree-sitter-python.git diff --git a/Cargo.lock b/Cargo.lock index 3c6faf5257..378f9d1264 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1912,6 +1912,7 @@ dependencies = [ "anyhow", "bincode", "cacache", + "cc", "chrono", "clap", "clearscreen", @@ -1939,6 +1940,8 @@ dependencies = [ "strum", "strum_macros", "toml", + "tree-sitter", + "tree-sitter-python", "update-informer", "walkdir", ] @@ -2409,6 +2412,26 @@ dependencies = [ "serde", ] +[[package]] +name = "tree-sitter" +version = "0.20.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4423c784fe11398ca91e505cdc71356b07b1a924fc8735cfab5333afe3e18bc" +dependencies = [ + "cc", + "regex", +] + +[[package]] +name = "tree-sitter-python" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dda114f58048f5059dcf158aff691dffb8e113e6d2b50d94263fd68711975287" +dependencies = [ + "cc", + "tree-sitter", +] + [[package]] name = "twox-hash" version = "1.6.3" diff --git a/Cargo.toml b/Cargo.toml index 809b48a9c8..12893b7600 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,10 +6,14 @@ edition = "2021" [lib] name = "ruff" +[build-dependencies] +cc="*" + [dependencies] anyhow = { version = "1.0.60" } bincode = { version = "1.3.3" } cacache = { version = "10.0.1" } +cc = "*" chrono = { version = "0.4.21" } clap = { version = "4.0.1", features = ["derive"] } clearscreen = { version = "1.0.10" } @@ -33,6 +37,8 @@ rustpython-common = { git = "https://github.com/charliermarsh/RustPython.git", r serde = { version = "1.0.143", features = ["derive"] } serde_json = { version = "1.0.83" } toml = { version = "0.5.9" } +tree-sitter = "0.20.9" +tree-sitter-python = "0.20.2" update-informer = { version = "0.5.0", default_features = false, features = ["pypi"], optional = true } walkdir = { version = "2.3.2" } strum = { version = "0.24.1", features = ["strum_macros"] } diff --git a/build.rs b/build.rs new file mode 100644 index 0000000000..fb34def0b6 --- /dev/null +++ b/build.rs @@ -0,0 +1,28 @@ +use std::path::Path; +extern crate cc; + +fn main() { + let src_dir = Path::new("tree-sitter-python").join("src"); + + let mut c_config = cc::Build::new(); + c_config.include(&src_dir); + c_config + .flag_if_supported("-Wno-unused-parameter") + .flag_if_supported("-Wno-unused-but-set-variable") + .flag_if_supported("-Wno-trigraphs"); + let parser_path = src_dir.join("parser.c"); + c_config.file(&parser_path); + println!("cargo:rerun-if-changed={}", parser_path.to_str().unwrap()); + c_config.compile("parser"); + + let mut cpp_config = cc::Build::new(); + cpp_config.cpp(true); + cpp_config.include(&src_dir); + cpp_config + .flag_if_supported("-Wno-unused-parameter") + .flag_if_supported("-Wno-unused-but-set-variable"); + let scanner_path = src_dir.join("scanner.cc"); + cpp_config.file(&scanner_path); + println!("cargo:rerun-if-changed={}", scanner_path.to_str().unwrap()); + cpp_config.compile("scanner"); +} diff --git a/examples/parse_python.rs b/examples/parse_python.rs new file mode 100644 index 0000000000..812e1d63f5 --- /dev/null +++ b/examples/parse_python.rs @@ -0,0 +1,130 @@ +use anyhow::Result; +use tree_sitter::{Parser, Query, QueryCursor}; + +enum Action { + Up, + Down, + Right, +} + +fn main() -> Result<()> { + let src = r#" +def double(x): + # Return a double. + return x * 2 + +x = double(1) +y = (f"{x}" "b") +"#; + let mut parser = Parser::new(); + parser + .set_language(tree_sitter_python::language()) + .expect("Error loading Python grammar"); + let parse_tree = parser.parse(src, None); + + if let Some(parse_tree) = &parse_tree { + // Check for comments. + let query = Query::new(tree_sitter_python::language(), "(comment) @capture")?; + let mut query_cursor = QueryCursor::new(); + let all_matches = query_cursor.matches(&query, parse_tree.root_node(), src.as_bytes()); + + for each_match in all_matches { + for capture in each_match.captures.iter() { + let range = capture.node.range(); + let text = &src[range.start_byte..range.end_byte]; + let line = range.start_point.row; + let col = range.start_point.column; + println!( + "[Line: {}, Col: {}] Offending source code: `{}`", + line, col, text + ); + } + } + + // Check for string concatenations. + let query = Query::new( + tree_sitter_python::language(), + "(concatenated_string) @capture", + )?; + let mut query_cursor = QueryCursor::new(); + let all_matches = query_cursor.matches(&query, parse_tree.root_node(), src.as_bytes()); + + for each_match in all_matches { + for capture in each_match.captures.iter() { + let range = capture.node.range(); + let text = &src[range.start_byte..range.end_byte]; + let line = range.start_point.row; + let col = range.start_point.column; + println!( + "[Line: {}, Col: {}] Offending source code: `{}`", + line, col, text + ); + } + } + + // Walk the tree. + let mut cursor = parse_tree.walk(); + let mut action = Action::Down; + loop { + match action { + Action::Up => { + if cursor.goto_next_sibling() { + action = Action::Right; + } else if cursor.goto_parent() { + action = Action::Up; + } else { + break; + } + } + Action::Down => { + let range = cursor.node().range(); + let text = &src[range.start_byte..range.end_byte]; + let line = range.start_point.row; + let col = range.start_point.column; + println!( + "[Line: {}, Col: {}] {}: `{}`", + line, + col, + cursor.node().kind(), + text + ); + + if cursor.goto_first_child() { + action = Action::Down; + } else if cursor.goto_next_sibling() { + action = Action::Right; + } else if cursor.goto_parent() { + action = Action::Up; + } else { + break; + } + } + Action::Right => { + let range = cursor.node().range(); + let text = &src[range.start_byte..range.end_byte]; + let line = range.start_point.row; + let col = range.start_point.column; + println!( + "[Line: {}, Col: {}] {}: `{}`", + line, + col, + cursor.node().kind(), + text + ); + + if cursor.goto_first_child() { + action = Action::Down; + } else if cursor.goto_next_sibling() { + action = Action::Right; + } else if cursor.goto_parent() { + action = Action::Up; + } else { + break; + } + } + } + } + } + + Ok(()) +} diff --git a/tree-sitter-python b/tree-sitter-python new file mode 160000 index 0000000000..de221eccf9 --- /dev/null +++ b/tree-sitter-python @@ -0,0 +1 @@ +Subproject commit de221eccf9a221f5b85474a553474a69b4b5784d