From 6741ea9790442bc271c8b530350debe180a89d10 Mon Sep 17 00:00:00 2001 From: Charlie Marsh Date: Sat, 5 Nov 2022 15:59:18 -0400 Subject: [PATCH] Create a separate dev crate for development scripts (#607) --- Cargo.lock | 17 +++++++- Cargo.toml | 2 +- ruff_dev/Cargo.toml | 16 ++++++++ .../src}/generate_check_code_prefix.rs | 19 ++++----- .../src}/generate_rules_table.rs | 18 ++++----- .../src}/generate_source_code.rs | 14 +++---- ruff_dev/src/lib.rs | 5 +++ ruff_dev/src/main.rs | 39 +++++++++++++++++++ {examples => ruff_dev/src}/print_ast.rs | 13 +++---- {examples => ruff_dev/src}/print_tokens.rs | 12 +++--- 10 files changed, 110 insertions(+), 45 deletions(-) create mode 100644 ruff_dev/Cargo.toml rename {examples => ruff_dev/src}/generate_check_code_prefix.rs (91%) rename {examples => ruff_dev/src}/generate_rules_table.rs (87%) rename {examples => ruff_dev/src}/generate_source_code.rs (70%) create mode 100644 ruff_dev/src/lib.rs create mode 100644 ruff_dev/src/main.rs rename {examples => ruff_dev/src}/print_ast.rs (72%) rename {examples => ruff_dev/src}/print_tokens.rs (72%) diff --git a/Cargo.lock b/Cargo.lock index eb6d17d0bc..6940e73d3e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2230,7 +2230,6 @@ dependencies = [ "chrono", "clap 4.0.15", "clearscreen", - "codegen", "colored", "common-path", "criterion", @@ -2265,6 +2264,22 @@ dependencies = [ "walkdir", ] +[[package]] +name = "ruff_dev" +version = "0.0.101" +dependencies = [ + "anyhow", + "clap 4.0.15", + "codegen", + "itertools", + "ruff", + "rustpython-ast", + "rustpython-common", + "rustpython-parser", + "strum", + "strum_macros", +] + [[package]] name = "rustls" version = "0.20.6" diff --git a/Cargo.toml b/Cargo.toml index 0d6c7cd903..34c34f0c44 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,7 @@ [workspace] members = [ "flake8_to_ruff", + "ruff_dev", ] [package] @@ -56,7 +57,6 @@ getrandom = { version = "0.2.7", features = ["js"] } [dev-dependencies] assert_cmd = { version = "2.0.4" } -codegen = { version = "0.2.0" } criterion = { version = "0.4.0" } insta = { version = "1.19.1", features = ["yaml"] } test-case = { version = "2.2.2" } diff --git a/ruff_dev/Cargo.toml b/ruff_dev/Cargo.toml new file mode 100644 index 0000000000..7dc26b4608 --- /dev/null +++ b/ruff_dev/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "ruff_dev" +version = "0.0.101" +edition = "2021" + +[dependencies] +anyhow = { version = "1.0.60" } +clap = { version = "4.0.1", features = ["derive"] } +codegen = { version = "0.2.0" } +itertools = { version = "0.10.5" } +ruff = { path = ".." } +rustpython-ast = { features = ["unparse"], git = "https://github.com/RustPython/RustPython.git", rev = "77b821a1941019fe34f73ce17cea013ae1b98fd0" } +rustpython-common = { git = "https://github.com/RustPython/RustPython.git", rev = "77b821a1941019fe34f73ce17cea013ae1b98fd0" } +rustpython-parser = { features = ["lalrpop"], git = "https://github.com/RustPython/RustPython.git", rev = "77b821a1941019fe34f73ce17cea013ae1b98fd0" } +strum = { version = "0.24.1", features = ["strum_macros"] } +strum_macros = { version = "0.24.3" } diff --git a/examples/generate_check_code_prefix.rs b/ruff_dev/src/generate_check_code_prefix.rs similarity index 91% rename from examples/generate_check_code_prefix.rs rename to ruff_dev/src/generate_check_code_prefix.rs index 434a9c41f7..fd8a8924c8 100644 --- a/examples/generate_check_code_prefix.rs +++ b/ruff_dev/src/generate_check_code_prefix.rs @@ -1,3 +1,5 @@ +//! Generate the `CheckCodePrefix` enum. + use std::collections::{BTreeMap, BTreeSet}; use std::fs::OpenOptions; use std::io::Write; @@ -13,17 +15,14 @@ const FILE: &str = "src/checks_gen.rs"; #[derive(Parser)] #[command(author, version, about, long_about = None)] -/// Generate the `CheckCodePrefix` enum. -struct Cli { +pub struct Cli { /// Write the generated source code to stdout (rather than to /// `src/checks_gen.rs`). #[arg(long)] dry_run: bool, } -fn main() -> Result<()> { - let cli = Cli::parse(); - +pub fn main(cli: &Cli) -> Result<()> { // Build up a map from prefix to matching CheckCodes. let mut prefix_to_codes: BTreeMap> = Default::default(); for check_code in CheckCode::iter() { @@ -34,9 +33,7 @@ fn main() -> Result<()> { .count(); for i in prefix_len..=as_ref.len() { let prefix = as_ref[..i].to_string(); - let entry = prefix_to_codes - .entry(prefix) - .or_insert_with(|| Default::default()); + let entry = prefix_to_codes.entry(prefix).or_default(); entry.insert(check_code.clone()); } } @@ -56,7 +53,7 @@ fn main() -> Result<()> { .derive("Clone") .derive("Serialize") .derive("Deserialize"); - for (prefix, _) in &prefix_to_codes { + for prefix in prefix_to_codes.keys() { gen = gen.push_variant(Variant::new(prefix.to_string())); } @@ -100,7 +97,7 @@ fn main() -> Result<()> { .ret(Type::new("PrefixSpecificity")) .vis("pub") .line("match self {"); - for (prefix, _) in &prefix_to_codes { + for prefix in prefix_to_codes.keys() { let num_numeric = prefix.chars().filter(|char| char.is_numeric()).count(); let specificity = match num_numeric { 3 => "Explicit", @@ -129,7 +126,7 @@ fn main() -> Result<()> { output.push_str("use crate::checks::CheckCode;"); output.push('\n'); output.push('\n'); - output.push_str(&format!("{}", scope.to_string())); + output.push_str(&scope.to_string()); // Write the output to `src/checks_gen.rs` (or stdout). if cli.dry_run { diff --git a/examples/generate_rules_table.rs b/ruff_dev/src/generate_rules_table.rs similarity index 87% rename from examples/generate_rules_table.rs rename to ruff_dev/src/generate_rules_table.rs index cfafa2e3fb..e87a8b1368 100644 --- a/examples/generate_rules_table.rs +++ b/ruff_dev/src/generate_rules_table.rs @@ -1,28 +1,26 @@ +//! Generate a Markdown-compatible table of supported lint rules. + use std::fs; use std::fs::OpenOptions; use std::io::Write; use anyhow::Result; -use clap::Parser; +use clap::Args; use ruff::checks::{CheckCategory, CheckCode}; use strum::IntoEnumIterator; -const FILE: &str = "README.md"; +const FILE: &str = "../README.md"; const BEGIN_PRAGMA: &str = ""; const END_PRAGMA: &str = ""; -#[derive(Parser)] -#[command(author, version, about, long_about = None)] -/// Generate a Markdown-compatible table of supported lint rules. -struct Cli { +#[derive(Args)] +pub struct Cli { /// Write the generated table to stdout (rather than to `README.md`). #[arg(long)] dry_run: bool, } -fn main() -> Result<()> { - let cli = Cli::parse(); - +pub fn main(cli: &Cli) -> Result<()> { // Generate the table string. let mut output = String::new(); for check_category in CheckCategory::iter() { @@ -43,7 +41,7 @@ fn main() -> Result<()> { "| {} | {} | {} | {} |", check_kind.code().as_ref(), check_kind.as_ref(), - check_kind.summary().replace("|", r"\|"), + check_kind.summary().replace('|', r"\|"), fix_token )); output.push('\n'); diff --git a/examples/generate_source_code.rs b/ruff_dev/src/generate_source_code.rs similarity index 70% rename from examples/generate_source_code.rs rename to ruff_dev/src/generate_source_code.rs index d8f1de77b3..e2b2fbd946 100644 --- a/examples/generate_source_code.rs +++ b/ruff_dev/src/generate_source_code.rs @@ -1,25 +1,25 @@ +//! Run round-trip source code generation on a given Python file. + use std::path::PathBuf; use anyhow::Result; -use clap::Parser; +use clap::Args; use ruff::code_gen::SourceGenerator; use ruff::fs; use rustpython_parser::parser; -#[derive(Debug, Parser)] -struct Cli { +#[derive(Args)] +pub struct Cli { + /// Python file to round-trip. #[arg(required = true)] file: PathBuf, } -fn main() -> Result<()> { - let cli = Cli::parse(); - +pub fn main(cli: &Cli) -> Result<()> { let contents = fs::read_file(&cli.file)?; let python_ast = parser::parse_program(&contents, &cli.file.to_string_lossy())?; let mut generator = SourceGenerator::new(); generator.unparse_suite(&python_ast)?; println!("{}", generator.generate()?); - Ok(()) } diff --git a/ruff_dev/src/lib.rs b/ruff_dev/src/lib.rs new file mode 100644 index 0000000000..46c90ce233 --- /dev/null +++ b/ruff_dev/src/lib.rs @@ -0,0 +1,5 @@ +pub mod generate_check_code_prefix; +pub mod generate_rules_table; +pub mod generate_source_code; +pub mod print_ast; +pub mod print_tokens; diff --git a/ruff_dev/src/main.rs b/ruff_dev/src/main.rs new file mode 100644 index 0000000000..146bd00444 --- /dev/null +++ b/ruff_dev/src/main.rs @@ -0,0 +1,39 @@ +use anyhow::Result; +use clap::{Parser, Subcommand}; +use ruff_dev::{ + generate_check_code_prefix, generate_rules_table, generate_source_code, print_ast, print_tokens, +}; + +#[derive(Parser)] +#[command(author, version, about, long_about = None)] +#[command(propagate_version = true)] +struct Cli { + #[command(subcommand)] + command: Commands, +} + +#[derive(Subcommand)] +enum Commands { + /// Generate the `CheckCodePrefix` enum. + GenerateCheckCodePrefix(generate_check_code_prefix::Cli), + /// Generate a Markdown-compatible table of supported lint rules. + GenerateRulesTable(generate_rules_table::Cli), + /// Run round-trip source code generation on a given Python file. + GenerateSourceCode(generate_source_code::Cli), + /// Print the AST for a given Python file. + PrintAST(print_ast::Cli), + /// Print the token stream for a given Python file. + PrintTokens(print_tokens::Cli), +} + +fn main() -> Result<()> { + let cli = Cli::parse(); + match &cli.command { + Commands::GenerateCheckCodePrefix(args) => generate_check_code_prefix::main(args)?, + Commands::GenerateRulesTable(args) => generate_rules_table::main(args)?, + Commands::GenerateSourceCode(args) => generate_source_code::main(args)?, + Commands::PrintAST(args) => print_ast::main(args)?, + Commands::PrintTokens(args) => print_tokens::main(args)?, + } + Ok(()) +} diff --git a/examples/print_ast.rs b/ruff_dev/src/print_ast.rs similarity index 72% rename from examples/print_ast.rs rename to ruff_dev/src/print_ast.rs index f62f17f406..33e4a4712b 100644 --- a/examples/print_ast.rs +++ b/ruff_dev/src/print_ast.rs @@ -3,23 +3,20 @@ use std::path::PathBuf; use anyhow::Result; -use clap::Parser; +use clap::Args; use ruff::fs; use rustpython_parser::parser; -#[derive(Debug, Parser)] -struct Cli { +#[derive(Args)] +pub struct Cli { + /// Python file for which to generate the AST. #[arg(required = true)] file: PathBuf, } -fn main() -> Result<()> { - let cli = Cli::parse(); - +pub fn main(cli: &Cli) -> Result<()> { let contents = fs::read_file(&cli.file)?; let python_ast = parser::parse_program(&contents, &cli.file.to_string_lossy())?; - println!("{:#?}", python_ast); - Ok(()) } diff --git a/examples/print_tokens.rs b/ruff_dev/src/print_tokens.rs similarity index 72% rename from examples/print_tokens.rs rename to ruff_dev/src/print_tokens.rs index 27d1d4af80..12f796dbd2 100644 --- a/examples/print_tokens.rs +++ b/ruff_dev/src/print_tokens.rs @@ -3,23 +3,21 @@ use std::path::PathBuf; use anyhow::Result; -use clap::Parser; +use clap::Args; use ruff::fs; use rustpython_parser::lexer; -#[derive(Debug, Parser)] -struct Cli { +#[derive(Args)] +pub struct Cli { + /// Python file for which to generate the AST. #[arg(required = true)] file: PathBuf, } -fn main() -> Result<()> { - let cli = Cli::parse(); - +pub fn main(cli: &Cli) -> Result<()> { let contents = fs::read_file(&cli.file)?; for (_, tok, _) in lexer::make_tokenizer(&contents).flatten() { println!("{:#?}", tok); } - Ok(()) }