Compare commits

...

15 Commits

Author SHA1 Message Date
Charlie Marsh
f3e11a30cb Bump version to 0.0.178 2022-12-12 22:06:04 -05:00
Charlie Marsh
2f3b5367ff Add a note on extends to README 2022-12-12 21:36:39 -05:00
Charlie Marsh
92bc417e4e Add support for glob patterns in src (#1225) 2022-12-12 21:35:03 -05:00
Charlie Marsh
9853b0728b Enable configuration files to "extend" other configuration files (#1219) 2022-12-12 20:28:22 -05:00
Charlie Marsh
77709dcc41 Remove underscore from extend_exclude 2022-12-12 16:34:16 -05:00
Charlie Marsh
b0cb5fc7ef Document current behavior around pyproject.toml discovery (#1213) 2022-12-12 11:49:21 -05:00
Charlie Marsh
d6f51e55dd Remove extraneous test_project 2022-12-12 10:53:12 -05:00
Charlie Marsh
4bb6b4851a Rename p to path 2022-12-12 10:51:24 -05:00
Charlie Marsh
54c5ded938 Move settings path discovery into its own function 2022-12-12 10:50:08 -05:00
Charlie Marsh
0157fedab5 Move Python file resolution into resolver.rs (#1211) 2022-12-12 10:43:50 -05:00
Charlie Marsh
cd69610741 Use --config everywhere if provided (#1210) 2022-12-12 10:28:00 -05:00
Charlie Marsh
a3d06d0005 Move more commands into commands.rs (#1209) 2022-12-12 10:22:47 -05:00
Charlie Marsh
ac6fa1dc88 Simplify some logic around configuration detection (#1197) 2022-12-12 10:15:05 -05:00
Charlie Marsh
73794fc299 Resolve hierarchical settings and Python files in a single filesystem pass (#1205) 2022-12-12 10:13:52 -05:00
Charlie Marsh
0adc9ed259 Support hierarchical settings for nested directories (#1190) 2022-12-12 10:12:23 -05:00
39 changed files with 1335 additions and 748 deletions

View File

@@ -1,6 +1,6 @@
repos:
- repo: https://github.com/charliermarsh/ruff-pre-commit
rev: v0.0.177
rev: v0.0.178
hooks:
- id: ruff

10
BREAKING_CHANGES.md Normal file
View File

@@ -0,0 +1,10 @@
# Breaking Changes
## 0.0.178
### Configuration files are now resolved hierarchically ([#1190](https://github.com/charliermarsh/ruff/pull/1190))
`pyproject.toml` files are now resolved hierarchically, such that for each Python file, we find
the first `pyproject.toml` file in its path, and use that to determine its lint settings.
See the [README](https://github.com/charliermarsh/ruff#pyprojecttoml-discovery) for more.

15
Cargo.lock generated
View File

@@ -724,7 +724,7 @@ checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80"
[[package]]
name = "flake8-to-ruff"
version = "0.0.177-dev.0"
version = "0.0.178-dev.0"
dependencies = [
"anyhow",
"clap 4.0.29",
@@ -796,6 +796,12 @@ dependencies = [
"wasm-bindgen",
]
[[package]]
name = "glob"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574"
[[package]]
name = "globset"
version = "0.4.9"
@@ -1821,7 +1827,7 @@ dependencies = [
[[package]]
name = "ruff"
version = "0.0.177"
version = "0.0.178"
dependencies = [
"annotate-snippets 0.9.1",
"anyhow",
@@ -1841,6 +1847,7 @@ dependencies = [
"fern",
"filetime",
"getrandom 0.2.8",
"glob",
"globset",
"insta",
"itertools",
@@ -1874,7 +1881,7 @@ dependencies = [
[[package]]
name = "ruff_dev"
version = "0.0.177"
version = "0.0.178"
dependencies = [
"anyhow",
"clap 4.0.29",
@@ -1892,7 +1899,7 @@ dependencies = [
[[package]]
name = "ruff_macros"
version = "0.0.177"
version = "0.0.178"
dependencies = [
"proc-macro2",
"quote",

View File

@@ -6,7 +6,7 @@ members = [
[package]
name = "ruff"
version = "0.0.177"
version = "0.0.178"
edition = "2021"
rust-version = "1.65.0"
@@ -28,6 +28,7 @@ common-path = { version = "1.0.0" }
dirs = { version = "4.0.0" }
fern = { version = "0.6.1" }
filetime = { version = "0.2.17" }
glob = { version = "0.3.0" }
globset = { version = "0.4.9" }
itertools = { version = "0.10.5" }
libcst = { git = "https://github.com/charliermarsh/LibCST", rev = "f2f0b7a487a8725d161fe8b3ed73a6758b21e177" }
@@ -41,7 +42,7 @@ quick-junit = { version = "0.3.2" }
rayon = { version = "1.5.3" }
regex = { version = "1.6.0" }
ropey = { version = "1.5.0", features = ["cr_lines", "simd"], default-features = false }
ruff_macros = { version = "0.0.177", path = "ruff_macros" }
ruff_macros = { version = "0.0.178", path = "ruff_macros" }
rustc-hash = { version = "1.1.0" }
rustpython-ast = { features = ["unparse"], git = "https://github.com/RustPython/RustPython.git", rev = "2edd0d264c50c7807bcff03a52a6509e8b7f187f" }
rustpython-common = { git = "https://github.com/RustPython/RustPython.git", rev = "2edd0d264c50c7807bcff03a52a6509e8b7f187f" }

View File

@@ -155,7 +155,7 @@ Ruff also works with [pre-commit](https://pre-commit.com):
```yaml
repos:
- repo: https://github.com/charliermarsh/ruff-pre-commit
rev: v0.0.177
rev: v0.0.178
hooks:
- id: ruff
```
@@ -331,6 +331,40 @@ Options:
Print version information
```
### `pyproject.toml` discovery
Similar to [ESLint](https://eslint.org/docs/latest/user-guide/configuring/configuration-files#cascading-and-hierarchy),
Ruff supports hierarchical configuration, such that the "closest" `pyproject.toml` file in the
directory hierarchy is used for every individual file, with all paths in the `pyproject.toml` file
(e.g., `exclude` globs, `src` paths) being resolved relative to the directory containing the
`pyproject.toml` file.
There are a few exceptions to these rules:
1. If a configuration file is passed directly via `--config`, those settings are used for across
files. Any relative paths in that configuration file (like `exclude` globs or `src` paths) are
resolved relative to the _current working directory_.
2. If no `pyproject.toml` file is found in the filesystem hierarchy, Ruff will fall back to using
a default configuration. If a user-specific configuration file exists at `${config_dir}/ruff/pyproject.toml`,
that file will be used instead of the default configuration, with `${config_dir}` being determined
via the [`dirs](https://docs.rs/dirs/4.0.0/dirs/fn.config_dir.html) crate, and all relative paths
being again resolved relative to the _current working directory_.
3. Any `pyproject.toml`-supported settings that are provided on the command-line (e.g., via
`--select`) will override the settings in _every_ resolved configuration file.
Unlike [ESLint](https://eslint.org/docs/latest/user-guide/configuring/configuration-files#cascading-and-hierarchy),
Ruff does not merge settings across configuration files; instead, the "closest" configuration file
is used, and any parent configuration files are ignored. In lieu of this implicit cascade, Ruff
supports an [`extend`](#extend) field, which allows you to inherit the settings from another
`pyproject.toml` file, like so:
```toml
# Extend the `pyproject.toml` file in the parent directory.
extend = "../pyproject.toml"
# But use a different line length.
line-length = 100
```
### Ignoring errors
To omit a lint check entirely, add it to the "ignore" list via [`ignore`](#ignore) or
@@ -1422,7 +1456,7 @@ Exclusions are based on globs, and can be either:
(to exclude any Python files in `directory`). Note that these paths are relative to the
project root (e.g., the directory containing your `pyproject.toml`).
Note that you'll typically want to use [`extend_exclude`](#extend_exclude) to modify
Note that you'll typically want to use [`extend-exclude`](#extend-exclude) to modify
the excluded paths.
**Default value**: `[".bzr", ".direnv", ".eggs", ".git", ".hg", ".mypy_cache", ".nox", ".pants.d", ".ruff_cache", ".svn", ".tox", ".venv", "__pypackages__", "_build", "buck-out", "build", "dist", "node_modules", "venv"]`
@@ -1438,6 +1472,30 @@ exclude = [".venv"]
---
#### [`extend`](#extend)
A path to a local `pyproject.toml` file to merge into this configuration.
To resolve the current `pyproject.toml` file, Ruff will first resolve this base
configuration file, then merge in any properties defined in the current configuration
file.
**Default value**: `None`
**Type**: `Path`
**Example usage**:
```toml
[tool.ruff]
# Extend the `pyproject.toml` file in the parent directory.
extend = "../pyproject.toml"
# But use a different line length.
line-length = 100
```
---
#### [`extend-exclude`](#extend-exclude)
A list of file patterns to omit from linting, in addition to those specified by `exclude`.
@@ -1695,6 +1753,25 @@ show-source = true
The source code paths to consider, e.g., when resolving first- vs. third-party imports.
As an example: given a Python package structure like:
```text
my_package/
pyproject.toml
src/
my_package/
__init__.py
foo.py
bar.py
```
The `src` directory should be included in `source` (e.g., `source = ["src"]`), such that
when resolving imports, `my_package.foo` is considered a first-party import.
This field supports globs. For example, if you have a series of Python packages in
a `python_modules` directory, `src = ["python_modules/*"]` would expand to incorporate
all of the packages in that directory.
**Default value**: `["."]`
**Type**: `Vec<PathBuf>`

View File

@@ -771,7 +771,7 @@ checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80"
[[package]]
name = "flake8_to_ruff"
version = "0.0.177"
version = "0.0.178"
dependencies = [
"anyhow",
"clap",
@@ -1975,7 +1975,7 @@ dependencies = [
[[package]]
name = "ruff"
version = "0.0.177"
version = "0.0.178"
dependencies = [
"anyhow",
"bincode",

View File

@@ -1,6 +1,6 @@
[package]
name = "flake8-to-ruff"
version = "0.0.177-dev.0"
version = "0.0.178-dev.0"
edition = "2021"
[lib]

View File

@@ -246,6 +246,7 @@ mod tests {
allowed_confusables: None,
dummy_variable_rgx: None,
exclude: None,
extend: None,
extend_exclude: None,
extend_ignore: None,
extend_select: None,
@@ -291,6 +292,7 @@ mod tests {
allowed_confusables: None,
dummy_variable_rgx: None,
exclude: None,
extend: None,
extend_exclude: None,
extend_ignore: None,
extend_select: None,
@@ -336,6 +338,7 @@ mod tests {
allowed_confusables: None,
dummy_variable_rgx: None,
exclude: None,
extend: None,
extend_exclude: None,
extend_ignore: None,
extend_select: None,
@@ -381,6 +384,7 @@ mod tests {
allowed_confusables: None,
dummy_variable_rgx: None,
exclude: None,
extend: None,
extend_exclude: None,
extend_ignore: None,
extend_select: None,
@@ -426,6 +430,7 @@ mod tests {
allowed_confusables: None,
dummy_variable_rgx: None,
exclude: None,
extend: None,
extend_exclude: None,
extend_ignore: None,
extend_select: None,
@@ -479,6 +484,7 @@ mod tests {
allowed_confusables: None,
dummy_variable_rgx: None,
exclude: None,
extend: None,
extend_exclude: None,
extend_ignore: None,
extend_select: None,
@@ -560,6 +566,7 @@ mod tests {
allowed_confusables: None,
dummy_variable_rgx: None,
exclude: None,
extend: None,
extend_exclude: None,
extend_ignore: None,
extend_select: None,

3
resources/test/fixtures/README.md vendored Normal file
View File

@@ -0,0 +1,3 @@
# fixtures
Fixture files used for snapshot testing.

View File

@@ -0,0 +1,73 @@
# project
An example multi-package Python project used to test setting resolution and other complex
behaviors.
## Expected behavior
Running from the repo root should pick up and enforce the appropriate settings for each package:
```
∴ cargo run resources/test/project/
Found 5 error(s).
resources/test/project/examples/docs/docs/file.py:1:1: I001 Import block is un-sorted or un-formatted
resources/test/project/examples/docs/docs/file.py:8:5: F841 Local variable `x` is assigned to but never used
resources/test/project/src/file.py:1:8: F401 `os` imported but unused
resources/test/project/src/file.py:5:5: F841 Local variable `x` is assigned to but never used
resources/test/project/src/import_file.py:1:1: I001 Import block is un-sorted or un-formatted
3 potentially fixable with the --fix option.
```
Running from the project directory itself should exhibit the same behavior:
```
∴ cd resources/test/project/ && cargo run .
Found 5 error(s).
examples/docs/docs/file.py:1:1: I001 Import block is un-sorted or un-formatted
examples/docs/docs/file.py:8:5: F841 Local variable `x` is assigned to but never used
src/file.py:1:8: F401 `os` imported but unused
src/file.py:5:5: F841 Local variable `x` is assigned to but never used
src/import_file.py:1:1: I001 Import block is un-sorted or un-formatted
3 potentially fixable with the --fix option.
```
Running from the sub-package directory should exhibit the same behavior, but omit the top-level
files:
```
∴ cd resources/test/project/examples/docs && cargo run .
Found 2 error(s).
docs/file.py:1:1: I001 Import block is un-sorted or un-formatted
docs/file.py:8:5: F841 Local variable `x` is assigned to but never used
1 potentially fixable with the --fix option.
```
`--config` should force Ruff to use the specified `pyproject.toml` for all files, and resolve
file paths from the current working directory:
```
∴ cargo run -- --config=resources/test/project/pyproject.toml resources/test/project/
Found 9 error(s).
resources/test/project/examples/docs/docs/concepts/file.py:1:8: F401 `os` imported but unused
resources/test/project/examples/docs/docs/concepts/file.py:5:5: F841 Local variable `x` is assigned to but never used
resources/test/project/examples/docs/docs/file.py:1:8: F401 `os` imported but unused
resources/test/project/examples/docs/docs/file.py:3:8: F401 `numpy` imported but unused
resources/test/project/examples/docs/docs/file.py:4:27: F401 `docs.concepts.file` imported but unused
resources/test/project/examples/docs/docs/file.py:8:5: F841 Local variable `x` is assigned to but never used
resources/test/project/src/file.py:1:8: F401 `os` imported but unused
resources/test/project/src/file.py:5:5: F841 Local variable `x` is assigned to but never used
resources/test/project/src/import_file.py:1:1: I001 Import block is un-sorted or un-formatted
6 potentially fixable with the --fix option.
```
Running from a parent directory should this "ignore" the `exclude` (hence, `concepts/file.py` gets
included in the output):
```
∴ cd resources/test/project/examples && cargo run -- --config=docs/pyproject.toml .
Found 3 error(s).
docs/docs/concepts/file.py:5:5: F841 Local variable `x` is assigned to but never used
docs/docs/file.py:1:1: I001 Import block is un-sorted or un-formatted
docs/docs/file.py:8:5: F841 Local variable `x` is assigned to but never used
1 potentially fixable with the --fix option.
```

View File

@@ -0,0 +1,5 @@
import os
def f():
x = 1

View File

@@ -0,0 +1,8 @@
import os
import numpy as np
from docs.concepts import file
def f():
x = 1

View File

@@ -0,0 +1,6 @@
[tool.ruff]
extend = "../../pyproject.toml"
src = ["."]
extend-select = ["I001"]
extend-ignore = ["F401"]
extend-exclude = ["./docs/concepts/file.py"]

View File

@@ -0,0 +1,3 @@
[tool.ruff]
src = [".", "python_modules/*"]
extend-select = ["I001"]

View File

View File

@@ -0,0 +1,5 @@
import os
def f():
x = 1

View File

@@ -0,0 +1,7 @@
import numpy as np
from app import app_file
from core import core_file
np.array([1, 2, 3])
app_file()
core_file()

View File

@@ -1,6 +1,6 @@
[package]
name = "ruff_dev"
version = "0.0.177"
version = "0.0.178"
edition = "2021"
[dependencies]

View File

@@ -1,6 +1,6 @@
[package]
name = "ruff_macros"
version = "0.0.177"
version = "0.0.178"
edition = "2021"
[lib]

View File

@@ -10,7 +10,7 @@ use crate::autofix::Fix;
use crate::checks::Check;
use crate::source_code_locator::SourceCodeLocator;
#[derive(Hash)]
#[derive(Debug, Hash)]
pub enum Mode {
Generate,
Apply,

View File

@@ -1,4 +1,4 @@
use std::path::PathBuf;
use std::path::{Path, PathBuf};
use clap::{command, Parser};
use regex::Regex;
@@ -6,6 +6,7 @@ use rustc_hash::FxHashMap;
use crate::checks::CheckCode;
use crate::checks_gen::CheckCodePrefix;
use crate::fs;
use crate::logging::LogLevel;
use crate::settings::types::{
FilePattern, PatternPrefixPair, PerFileIgnore, PythonVersion, SerializationFormat,
@@ -88,7 +89,7 @@ pub struct Cli {
/// See the files Ruff will be run against with the current settings.
#[arg(long)]
pub show_files: bool,
/// See Ruff's settings.
/// See the settings Ruff used for the first matching file.
#[arg(long)]
pub show_settings: bool,
/// Enable automatic additions of noqa directives to failing lines.
@@ -198,6 +199,7 @@ pub struct Arguments {
}
/// CLI settings that function as configuration overrides.
#[derive(Clone)]
#[allow(clippy::struct_excessive_bools)]
pub struct Overrides {
pub dummy_variable_rgx: Option<Regex>,
@@ -243,6 +245,9 @@ pub fn collect_per_file_ignores(pairs: Vec<PatternPrefixPair>) -> Vec<PerFileIgn
}
per_file_ignores
.into_iter()
.map(|(pattern, prefixes)| PerFileIgnore::new(pattern, &prefixes))
.map(|(pattern, prefixes)| {
let absolute = fs::normalize_path(Path::new(&pattern));
PerFileIgnore::new(pattern, absolute, &prefixes)
})
.collect()
}

View File

@@ -1,36 +1,207 @@
use std::path::PathBuf;
use std::io::{self, Read};
use std::path::{Path, PathBuf};
use std::time::Instant;
use anyhow::{bail, Result};
use itertools::Itertools;
use log::{debug, error};
#[cfg(not(target_family = "wasm"))]
use rayon::prelude::*;
use rustpython_ast::Location;
use serde::Serialize;
use walkdir::DirEntry;
use crate::checks::CheckCode;
use crate::fs::iter_python_files;
use crate::autofix::fixer;
use crate::checks::{CheckCode, CheckKind};
use crate::cli::Overrides;
use crate::iterators::par_iter;
use crate::linter::{add_noqa_to_path, autoformat_path, lint_path, lint_stdin, Diagnostics};
use crate::message::Message;
use crate::resolver;
use crate::resolver::Strategy;
use crate::settings::types::SerializationFormat;
use crate::{Configuration, Settings};
/// Run the linter over a collection of files.
pub fn run(
files: &[PathBuf],
strategy: &Strategy,
overrides: &Overrides,
cache: bool,
autofix: &fixer::Mode,
) -> Result<Diagnostics> {
// Collect all the files to check.
let start = Instant::now();
let (paths, resolver) = resolver::resolve_python_files(files, strategy, overrides)?;
let duration = start.elapsed();
debug!("Identified files to lint in: {:?}", duration);
let start = Instant::now();
let mut diagnostics: Diagnostics = par_iter(&paths)
.map(|entry| {
match entry {
Ok(entry) => {
let path = entry.path();
let settings = resolver.resolve(path, strategy);
lint_path(path, settings, &cache.into(), autofix)
.map_err(|e| (Some(path.to_owned()), e.to_string()))
}
Err(e) => Err((
e.path().map(Path::to_owned),
e.io_error()
.map_or_else(|| e.to_string(), io::Error::to_string),
)),
}
.unwrap_or_else(|(path, message)| {
if let Some(path) = &path {
let settings = resolver.resolve(path, strategy);
if settings.enabled.contains(&CheckCode::E902) {
Diagnostics::new(vec![Message {
kind: CheckKind::IOError(message),
location: Location::default(),
end_location: Location::default(),
fix: None,
filename: path.to_string_lossy().to_string(),
source: None,
}])
} else {
error!("Failed to check {}: {message}", path.to_string_lossy());
Diagnostics::default()
}
} else {
error!("{message}");
Diagnostics::default()
}
})
})
.reduce(Diagnostics::default, |mut acc, item| {
acc += item;
acc
});
diagnostics.messages.sort_unstable();
let duration = start.elapsed();
debug!("Checked files in: {:?}", duration);
Ok(diagnostics)
}
/// Read a `String` from `stdin`.
fn read_from_stdin() -> Result<String> {
let mut buffer = String::new();
io::stdin().lock().read_to_string(&mut buffer)?;
Ok(buffer)
}
/// Run the linter over a single file, read from `stdin`.
pub fn run_stdin(
strategy: &Strategy,
filename: &Path,
autofix: &fixer::Mode,
) -> Result<Diagnostics> {
let stdin = read_from_stdin()?;
let settings = match strategy {
Strategy::Fixed(settings) => settings,
Strategy::Hierarchical(settings) => settings,
};
let mut diagnostics = lint_stdin(filename, &stdin, settings, autofix)?;
diagnostics.messages.sort_unstable();
Ok(diagnostics)
}
/// Add `noqa` directives to a collection of files.
pub fn add_noqa(files: &[PathBuf], strategy: &Strategy, overrides: &Overrides) -> Result<usize> {
// Collect all the files to check.
let start = Instant::now();
let (paths, resolver) = resolver::resolve_python_files(files, strategy, overrides)?;
let duration = start.elapsed();
debug!("Identified files to lint in: {:?}", duration);
let start = Instant::now();
let modifications: usize = par_iter(&paths)
.flatten()
.filter_map(|entry| {
let path = entry.path();
let settings = resolver.resolve(path, strategy);
match add_noqa_to_path(path, settings) {
Ok(count) => Some(count),
Err(e) => {
error!("Failed to add noqa to {}: {e}", path.to_string_lossy());
None
}
}
})
.sum();
let duration = start.elapsed();
debug!("Added noqa to files in: {:?}", duration);
Ok(modifications)
}
/// Automatically format a collection of files.
pub fn autoformat(files: &[PathBuf], strategy: &Strategy, overrides: &Overrides) -> Result<usize> {
// Collect all the files to format.
let start = Instant::now();
let (paths, resolver) = resolver::resolve_python_files(files, strategy, overrides)?;
let duration = start.elapsed();
debug!("Identified files to lint in: {:?}", duration);
let start = Instant::now();
let modifications = par_iter(&paths)
.flatten()
.filter_map(|entry| {
let path = entry.path();
let settings = resolver.resolve(path, strategy);
match autoformat_path(path, settings) {
Ok(()) => Some(()),
Err(e) => {
error!("Failed to autoformat {}: {e}", path.to_string_lossy());
None
}
}
})
.count();
let duration = start.elapsed();
debug!("Auto-formatted files in: {:?}", duration);
Ok(modifications)
}
/// Print the user-facing configuration settings.
pub fn show_settings(
configuration: &Configuration,
project_root: Option<&PathBuf>,
pyproject: Option<&PathBuf>,
) {
println!("Resolved configuration: {configuration:#?}");
println!("Found project root at: {project_root:?}");
println!("Found pyproject.toml at: {pyproject:?}");
pub fn show_settings(files: &[PathBuf], strategy: &Strategy, overrides: &Overrides) -> Result<()> {
// Collect all files in the hierarchy.
let (paths, resolver) = resolver::resolve_python_files(files, strategy, overrides)?;
// Print the list of files.
let Some(entry) = paths
.iter()
.flatten()
.sorted_by(|a, b| a.path().cmp(b.path())).next() else {
bail!("No files found under the given path");
};
let path = entry.path();
let settings = resolver.resolve(path, strategy);
println!("Resolved settings for: {path:?}");
println!("{settings:#?}");
Ok(())
}
/// Show the list of files to be checked based on current settings.
pub fn show_files(files: &[PathBuf], settings: &Settings) {
let mut entries: Vec<DirEntry> = files
pub fn show_files(files: &[PathBuf], strategy: &Strategy, overrides: &Overrides) -> Result<()> {
// Collect all files in the hierarchy.
let (paths, _resolver) = resolver::resolve_python_files(files, strategy, overrides)?;
// Print the list of files.
for entry in paths
.iter()
.flat_map(|path| iter_python_files(path, &settings.exclude, &settings.extend_exclude))
.flatten()
.collect();
entries.sort_by(|a, b| a.path().cmp(b.path()));
for entry in entries {
.sorted_by(|a, b| a.path().cmp(b.path()))
{
println!("{}", entry.path().to_string_lossy());
}
Ok(())
}
#[derive(Serialize)]
@@ -41,7 +212,7 @@ struct Explanation<'a> {
}
/// Explain a `CheckCode` to the user.
pub fn explain(code: &CheckCode, format: SerializationFormat) -> Result<()> {
pub fn explain(code: &CheckCode, format: &SerializationFormat) -> Result<()> {
match format {
SerializationFormat::Text | SerializationFormat::Grouped => {
println!(

172
src/fs.rs
View File

@@ -5,15 +5,13 @@ use std::path::{Path, PathBuf};
use anyhow::{anyhow, Result};
use globset::GlobMatcher;
use log::debug;
use path_absolutize::{path_dedot, Absolutize};
use rustc_hash::FxHashSet;
use walkdir::{DirEntry, WalkDir};
use crate::checks::CheckCode;
/// Extract the absolute path and basename (as strings) from a Path.
fn extract_path_names(path: &Path) -> Result<(&str, &str)> {
pub fn extract_path_names(path: &Path) -> Result<(&str, &str)> {
let file_path = path
.to_str()
.ok_or_else(|| anyhow!("Unable to parse filename: {:?}", path))?;
@@ -25,62 +23,7 @@ fn extract_path_names(path: &Path) -> Result<(&str, &str)> {
Ok((file_path, file_basename))
}
fn is_excluded(file_path: &str, file_basename: &str, exclude: &globset::GlobSet) -> bool {
exclude.is_match(file_path) || exclude.is_match(file_basename)
}
fn is_included(path: &Path) -> bool {
let file_name = path.to_string_lossy();
file_name.ends_with(".py") || file_name.ends_with(".pyi")
}
pub fn iter_python_files<'a>(
path: &'a Path,
exclude: &'a globset::GlobSet,
extend_exclude: &'a globset::GlobSet,
) -> impl Iterator<Item = Result<DirEntry, walkdir::Error>> + 'a {
// Run some checks over the provided patterns, to enable optimizations below.
let has_exclude = !exclude.is_empty();
let has_extend_exclude = !extend_exclude.is_empty();
WalkDir::new(normalize_path(path))
.into_iter()
.filter_entry(move |entry| {
if !has_exclude && !has_extend_exclude {
return true;
}
let path = entry.path();
match extract_path_names(path) {
Ok((file_path, file_basename)) => {
if has_exclude && is_excluded(file_path, file_basename, exclude) {
debug!("Ignored path via `exclude`: {:?}", path);
false
} else if has_extend_exclude
&& is_excluded(file_path, file_basename, extend_exclude)
{
debug!("Ignored path via `extend-exclude`: {:?}", path);
false
} else {
true
}
}
Err(e) => {
debug!("Ignored path due to error in parsing: {:?}: {}", path, e);
true
}
}
})
.filter(|entry| {
entry.as_ref().map_or(true, |entry| {
(entry.depth() == 0 || is_included(entry.path()))
&& !entry.file_type().is_dir()
&& !(entry.file_type().is_symlink() && entry.path().is_dir())
})
})
}
/// Create tree set with codes matching the pattern/code pairs.
/// Create a set with codes matching the pattern/code pairs.
pub(crate) fn ignores_from_path<'a>(
path: &Path,
pattern_code_pairs: &'a [(GlobMatcher, GlobMatcher, FxHashSet<CheckCode>)],
@@ -128,114 +71,3 @@ pub(crate) fn read_file(path: &Path) -> Result<String> {
buf_reader.read_to_string(&mut contents)?;
Ok(contents)
}
#[cfg(test)]
mod tests {
use std::path::{Path, PathBuf};
use anyhow::Result;
use globset::GlobSet;
use path_absolutize::Absolutize;
use crate::fs::{extract_path_names, is_excluded, is_included};
use crate::settings::types::FilePattern;
#[test]
fn inclusions() {
let path = Path::new("foo/bar/baz.py").absolutize().unwrap();
assert!(is_included(&path));
let path = Path::new("foo/bar/baz.pyi").absolutize().unwrap();
assert!(is_included(&path));
let path = Path::new("foo/bar/baz.js").absolutize().unwrap();
assert!(!is_included(&path));
let path = Path::new("foo/bar/baz").absolutize().unwrap();
assert!(!is_included(&path));
}
fn make_exclusion(file_pattern: FilePattern, project_root: Option<&PathBuf>) -> GlobSet {
let mut builder = globset::GlobSetBuilder::new();
file_pattern.add_to(&mut builder, project_root).unwrap();
builder.build().unwrap()
}
#[test]
fn exclusions() -> Result<()> {
let project_root = Path::new("/tmp/");
let path = Path::new("foo").absolutize_from(project_root).unwrap();
let exclude = FilePattern::User("foo".to_string());
let (file_path, file_basename) = extract_path_names(&path)?;
assert!(is_excluded(
file_path,
file_basename,
&make_exclusion(exclude, Some(&project_root.to_path_buf()))
));
let path = Path::new("foo/bar").absolutize_from(project_root).unwrap();
let exclude = FilePattern::User("bar".to_string());
let (file_path, file_basename) = extract_path_names(&path)?;
assert!(is_excluded(
file_path,
file_basename,
&make_exclusion(exclude, Some(&project_root.to_path_buf()))
));
let path = Path::new("foo/bar/baz.py")
.absolutize_from(project_root)
.unwrap();
let exclude = FilePattern::User("baz.py".to_string());
let (file_path, file_basename) = extract_path_names(&path)?;
assert!(is_excluded(
file_path,
file_basename,
&make_exclusion(exclude, Some(&project_root.to_path_buf()))
));
let path = Path::new("foo/bar").absolutize_from(project_root).unwrap();
let exclude = FilePattern::User("foo/bar".to_string());
let (file_path, file_basename) = extract_path_names(&path)?;
assert!(is_excluded(
file_path,
file_basename,
&make_exclusion(exclude, Some(&project_root.to_path_buf()))
));
let path = Path::new("foo/bar/baz.py")
.absolutize_from(project_root)
.unwrap();
let exclude = FilePattern::User("foo/bar/baz.py".to_string());
let (file_path, file_basename) = extract_path_names(&path)?;
assert!(is_excluded(
file_path,
file_basename,
&make_exclusion(exclude, Some(&project_root.to_path_buf()))
));
let path = Path::new("foo/bar/baz.py")
.absolutize_from(project_root)
.unwrap();
let exclude = FilePattern::User("foo/bar/*.py".to_string());
let (file_path, file_basename) = extract_path_names(&path)?;
assert!(is_excluded(
file_path,
file_basename,
&make_exclusion(exclude, Some(&project_root.to_path_buf()))
));
let path = Path::new("foo/bar/baz.py")
.absolutize_from(project_root)
.unwrap();
let exclude = FilePattern::User("baz".to_string());
let (file_path, file_basename) = extract_path_names(&path)?;
assert!(!is_excluded(
file_path,
file_basename,
&make_exclusion(exclude, Some(&project_root.to_path_buf()))
));
Ok(())
}
}

16
src/iterators.rs Normal file
View File

@@ -0,0 +1,16 @@
#[cfg(not(target_family = "wasm"))]
use rayon::prelude::*;
/// Shim that calls `par_iter` except for wasm because there's no wasm support
/// in rayon yet (there is a shim to be used for the web, but it requires js
/// cooperation) Unfortunately, `ParallelIterator` does not implement `Iterator`
/// so the signatures diverge
#[cfg(not(target_family = "wasm"))]
pub fn par_iter<T: Sync>(iterable: &[T]) -> impl ParallelIterator<Item = &T> {
iterable.par_iter()
}
#[cfg(target_family = "wasm")]
pub fn par_iter<T: Sync>(iterable: &[T]) -> impl Iterator<Item = &T> {
iterable.iter()
}

View File

@@ -14,13 +14,14 @@
use std::path::Path;
use anyhow::Result;
use log::debug;
use path_absolutize::path_dedot;
use rustpython_helpers::tokenize;
use rustpython_parser::lexer::LexResult;
use settings::{pyproject, Settings};
use crate::checks::Check;
use crate::linter::check_path;
use crate::resolver::Relativity;
use crate::settings::configuration::Configuration;
use crate::source_code_locator::SourceCodeLocator;
@@ -58,6 +59,7 @@ pub mod flake8_tidy_imports;
mod flake8_unused_arguments;
pub mod fs;
mod isort;
pub mod iterators;
mod lex;
pub mod linter;
pub mod logging;
@@ -73,6 +75,7 @@ mod pygrep_hooks;
mod pylint;
mod python;
mod pyupgrade;
pub mod resolver;
mod ruff;
mod rustpython_helpers;
pub mod settings;
@@ -82,24 +85,24 @@ pub mod updates;
mod vendored;
pub mod visibility;
/// Load the relevant `Settings` for a given `Path`.
fn resolve(path: &Path) -> Result<Settings> {
if let Some(pyproject) = pyproject::find_pyproject_toml(path) {
// First priority: `pyproject.toml` in the current `Path`.
resolver::resolve_settings(&pyproject, &Relativity::Parent, None)
} else if let Some(pyproject) = pyproject::find_user_pyproject_toml() {
// Second priority: user-specific `pyproject.toml`.
resolver::resolve_settings(&pyproject, &Relativity::Cwd, None)
} else {
// Fallback: default settings.
Settings::from_configuration(Configuration::default(), &path_dedot::CWD)
}
}
/// Run Ruff over Python source code directly.
pub fn check(path: &Path, contents: &str, autofix: bool) -> Result<Vec<Check>> {
// Find the project root and pyproject.toml.
let project_root = pyproject::find_project_root(&[path.to_path_buf()]);
match &project_root {
Some(path) => debug!("Found project root at: {:?}", path),
None => debug!("Unable to identify project root; assuming current directory..."),
};
let pyproject = pyproject::find_pyproject_toml(project_root.as_ref());
match &pyproject {
Some(path) => debug!("Found pyproject.toml at: {:?}", path),
None => debug!("Unable to find pyproject.toml; using default settings..."),
};
let settings = Settings::from_configuration(
Configuration::from_pyproject(pyproject.as_ref(), project_root.as_ref())?,
project_root.as_ref(),
)?;
// Load the relevant `Settings` for the given `Path`.
let settings = resolve(path)?;
// Tokenize once.
let tokens: Vec<LexResult> = tokenize(contents);

View File

@@ -211,7 +211,7 @@ pub fn add_noqa_to_path(path: &Path, settings: &Settings) -> Result<usize> {
}
/// Apply autoformatting to the source code at the given `Path`.
pub fn autoformat_path(path: &Path) -> Result<()> {
pub fn autoformat_path(path: &Path, _settings: &Settings) -> Result<()> {
// Read the file from disk.
let contents = fs::read_file(path)?;

View File

@@ -11,190 +11,61 @@
clippy::too_many_lines
)]
use std::io::{self, Read};
use std::io::{self};
use std::path::{Path, PathBuf};
use std::process::ExitCode;
use std::sync::mpsc::channel;
use std::time::Instant;
use ::ruff::autofix::fixer;
use ::ruff::checks::{CheckCode, CheckKind};
use ::ruff::cli::{extract_log_level, Cli};
use ::ruff::fs::iter_python_files;
use ::ruff::linter::{add_noqa_to_path, autoformat_path, lint_path, lint_stdin, Diagnostics};
use ::ruff::logging::{set_up_logging, LogLevel};
use ::ruff::message::Message;
use ::ruff::printer::Printer;
use ::ruff::resolver::Strategy;
use ::ruff::settings::configuration::Configuration;
use ::ruff::settings::types::SerializationFormat;
use ::ruff::settings::{pyproject, Settings};
#[cfg(feature = "update-informer")]
use ::ruff::updates;
use ::ruff::{cache, commands, fs};
use ::ruff::{cache, commands};
use anyhow::Result;
use clap::{CommandFactory, Parser};
use colored::Colorize;
use log::{debug, error};
use notify::{recommended_watcher, RecursiveMode, Watcher};
#[cfg(not(target_family = "wasm"))]
use rayon::prelude::*;
use rustpython_ast::Location;
use walkdir::DirEntry;
use path_absolutize::path_dedot;
use ruff::cli::Overrides;
use ruff::resolver::{resolve_settings, Relativity};
/// Shim that calls `par_iter` except for wasm because there's no wasm support
/// in rayon yet (there is a shim to be used for the web, but it requires js
/// cooperation) Unfortunately, `ParallelIterator` does not implement `Iterator`
/// so the signatures diverge
#[cfg(not(target_family = "wasm"))]
fn par_iter<T: Sync>(iterable: &Vec<T>) -> impl ParallelIterator<Item = &T> {
iterable.par_iter()
}
#[cfg(target_family = "wasm")]
fn par_iter<T: Sync>(iterable: &Vec<T>) -> impl Iterator<Item = &T> {
iterable.iter()
}
fn read_from_stdin() -> Result<String> {
let mut buffer = String::new();
io::stdin().lock().read_to_string(&mut buffer)?;
Ok(buffer)
}
fn run_once_stdin(
settings: &Settings,
filename: &Path,
autofix: &fixer::Mode,
) -> Result<Diagnostics> {
let stdin = read_from_stdin()?;
let mut diagnostics = lint_stdin(filename, &stdin, settings, autofix)?;
diagnostics.messages.sort_unstable();
Ok(diagnostics)
}
fn run_once(
files: &[PathBuf],
settings: &Settings,
cache: bool,
autofix: &fixer::Mode,
) -> Diagnostics {
// Collect all the files to check.
let start = Instant::now();
let paths: Vec<Result<DirEntry, walkdir::Error>> = files
.iter()
.flat_map(|path| iter_python_files(path, &settings.exclude, &settings.extend_exclude))
.collect();
let duration = start.elapsed();
debug!("Identified files to lint in: {:?}", duration);
let start = Instant::now();
let mut diagnostics: Diagnostics = par_iter(&paths)
.map(|entry| {
match entry {
Ok(entry) => {
let path = entry.path();
lint_path(path, settings, &cache.into(), autofix)
.map_err(|e| (Some(path.to_owned()), e.to_string()))
}
Err(e) => Err((
e.path().map(Path::to_owned),
e.io_error()
.map_or_else(|| e.to_string(), io::Error::to_string),
)),
}
.unwrap_or_else(|(path, message)| {
if let Some(path) = path {
if settings.enabled.contains(&CheckCode::E902) {
Diagnostics::new(vec![Message {
kind: CheckKind::IOError(message),
location: Location::default(),
end_location: Location::default(),
fix: None,
filename: path.to_string_lossy().to_string(),
source: None,
}])
} else {
error!("Failed to check {}: {message}", path.to_string_lossy());
Diagnostics::default()
}
} else {
error!("{message}");
Diagnostics::default()
}
})
})
.reduce(Diagnostics::default, |mut acc, item| {
acc += item;
acc
});
diagnostics.messages.sort_unstable();
let duration = start.elapsed();
debug!("Checked files in: {:?}", duration);
diagnostics
}
fn add_noqa(files: &[PathBuf], settings: &Settings) -> usize {
// Collect all the files to check.
let start = Instant::now();
let paths: Vec<DirEntry> = files
.iter()
.flat_map(|path| iter_python_files(path, &settings.exclude, &settings.extend_exclude))
.flatten()
.collect();
let duration = start.elapsed();
debug!("Identified files to lint in: {:?}", duration);
let start = Instant::now();
let modifications: usize = par_iter(&paths)
.filter_map(|entry| {
let path = entry.path();
match add_noqa_to_path(path, settings) {
Ok(count) => Some(count),
Err(e) => {
error!("Failed to add noqa to {}: {e}", path.to_string_lossy());
None
}
}
})
.sum();
let duration = start.elapsed();
debug!("Added noqa to files in: {:?}", duration);
modifications
}
fn autoformat(files: &[PathBuf], settings: &Settings) -> usize {
// Collect all the files to format.
let start = Instant::now();
let paths: Vec<DirEntry> = files
.iter()
.flat_map(|path| iter_python_files(path, &settings.exclude, &settings.extend_exclude))
.flatten()
.collect();
let duration = start.elapsed();
debug!("Identified files to lint in: {:?}", duration);
let start = Instant::now();
let modifications = par_iter(&paths)
.filter_map(|entry| {
let path = entry.path();
match autoformat_path(path) {
Ok(()) => Some(()),
Err(e) => {
error!("Failed to autoformat {}: {e}", path.to_string_lossy());
None
}
}
})
.count();
let duration = start.elapsed();
debug!("Auto-formatted files in: {:?}", duration);
modifications
/// Resolve the relevant settings strategy and defaults for the current
/// invocation.
fn resolve(config: Option<PathBuf>, overrides: &Overrides) -> Result<Strategy> {
if let Some(pyproject) = config {
// First priority: the user specified a `pyproject.toml` file. Use that
// `pyproject.toml` for _all_ configuration, and resolve paths relative to the
// current working directory. (This matches ESLint's behavior.)
let settings = resolve_settings(&pyproject, &Relativity::Cwd, Some(overrides))?;
Ok(Strategy::Fixed(settings))
} else if let Some(pyproject) = pyproject::find_pyproject_toml(path_dedot::CWD.as_path()) {
// Second priority: find a `pyproject.toml` file in the current working path,
// and resolve all paths relative to that directory. (With
// `Strategy::Hierarchical`, we'll end up finding the "closest" `pyproject.toml`
// file for every Python file later on, so these act as the "default" settings.)
let settings = resolve_settings(&pyproject, &Relativity::Parent, Some(overrides))?;
Ok(Strategy::Hierarchical(settings))
} else if let Some(pyproject) = pyproject::find_user_pyproject_toml() {
// Third priority: find a user-specific `pyproject.toml`, but resolve all paths
// relative the current working directory. (With `Strategy::Hierarchical`, we'll
// end up the "closest" `pyproject.toml` file for every Python file later on, so
// these act as the "default" settings.)
let settings = resolve_settings(&pyproject, &Relativity::Cwd, Some(overrides))?;
Ok(Strategy::Hierarchical(settings))
} else {
// Fallback: load Ruff's default settings, and resolve all paths relative to the
// current working directory. (With `Strategy::Hierarchical`, we'll end up the
// "closest" `pyproject.toml` file for every Python file later on, so these act
// as the "default" settings.)
let settings = Settings::from_configuration(Configuration::default(), &path_dedot::CWD)?;
Ok(Strategy::Hierarchical(settings))
}
}
fn inner_main() -> Result<ExitCode> {
@@ -203,63 +74,42 @@ fn inner_main() -> Result<ExitCode> {
let log_level = extract_log_level(&cli);
set_up_logging(&log_level)?;
if let Some(shell) = cli.generate_shell_completion {
shell.generate(&mut Cli::command(), &mut std::io::stdout());
return Ok(ExitCode::SUCCESS);
}
// Find the project root and pyproject.toml.
let config: Option<PathBuf> = cli.config;
let project_root = config.as_ref().map_or_else(
|| pyproject::find_project_root(&cli.files),
|config| config.parent().map(fs::normalize_path),
);
let pyproject = config.or_else(|| pyproject::find_pyproject_toml(project_root.as_ref()));
// Reconcile configuration from pyproject.toml and command-line arguments.
let mut configuration =
Configuration::from_pyproject(pyproject.as_ref(), project_root.as_ref())?;
configuration.merge(overrides);
if cli.show_settings && cli.show_files {
eprintln!("Error: specify --show-settings or show-files (not both).");
return Ok(ExitCode::FAILURE);
anyhow::bail!("specify --show-settings or show-files (not both)")
}
if cli.show_settings {
commands::show_settings(&configuration, project_root.as_ref(), pyproject.as_ref());
if let Some(shell) = cli.generate_shell_completion {
shell.generate(&mut Cli::command(), &mut io::stdout());
return Ok(ExitCode::SUCCESS);
}
// TODO(charlie): Included in `pyproject.toml`, but not inherited.
let fix = if configuration.fix {
// Construct the "default" settings. These are used when no `pyproject.toml`
// files are present, or files are injected from outside of the hierarchy.
let strategy = resolve(cli.config, &overrides)?;
// Extract options that are included in `Settings`, but only apply at the top
// level.
let (fix, format) = match &strategy {
Strategy::Fixed(settings) => (settings.fix, settings.format),
Strategy::Hierarchical(settings) => (settings.fix, settings.format),
};
let autofix = if fix {
fixer::Mode::Apply
} else if matches!(configuration.format, SerializationFormat::Json) {
} else if matches!(format, SerializationFormat::Json) {
fixer::Mode::Generate
} else {
fixer::Mode::None
};
let format = configuration.format;
let settings = Settings::from_configuration(configuration, project_root.as_ref())?;
// Now that we've inferred the appropriate log level, add some debug
// information.
match &project_root {
Some(path) => debug!("Found project root at: {:?}", path),
None => debug!("Unable to identify project root; assuming current directory..."),
};
match &pyproject {
Some(path) => debug!("Found pyproject.toml at: {:?}", path),
None => debug!("Unable to find pyproject.toml; using default settings..."),
};
if let Some(code) = cli.explain {
commands::explain(&code, format)?;
commands::explain(&code, &format)?;
return Ok(ExitCode::SUCCESS);
}
if cli.show_settings {
commands::show_settings(&cli.files, &strategy, &overrides)?;
return Ok(ExitCode::SUCCESS);
}
if cli.show_files {
commands::show_files(&cli.files, &settings);
commands::show_files(&cli.files, &strategy, &overrides)?;
return Ok(ExitCode::SUCCESS);
}
@@ -272,7 +122,7 @@ fn inner_main() -> Result<ExitCode> {
let printer = Printer::new(&format, &log_level);
if cli.watch {
if matches!(fix, fixer::Mode::Generate | fixer::Mode::Apply) {
if matches!(autofix, fixer::Mode::Generate | fixer::Mode::Apply) {
eprintln!("Warning: --fix is not enabled in watch mode.");
}
if cli.add_noqa {
@@ -289,7 +139,13 @@ fn inner_main() -> Result<ExitCode> {
printer.clear_screen()?;
printer.write_to_user("Starting linter in watch mode...\n");
let messages = run_once(&cli.files, &settings, cache_enabled, &fixer::Mode::None);
let messages = commands::run(
&cli.files,
&strategy,
&overrides,
cache_enabled,
&fixer::Mode::None,
)?;
printer.write_continuously(&messages)?;
// Configure the file watcher.
@@ -301,32 +157,37 @@ fn inner_main() -> Result<ExitCode> {
loop {
match rx.recv() {
Ok(e) => {
let paths = e?.paths;
let py_changed = paths.iter().any(|p| {
p.extension()
.map(|ext| ext.eq_ignore_ascii_case("py"))
Ok(event) => {
let paths = event?.paths;
let py_changed = paths.iter().any(|path| {
path.extension()
.map(|ext| ext == "py" || ext == "pyi")
.unwrap_or_default()
});
if py_changed {
printer.clear_screen()?;
printer.write_to_user("File change detected...\n");
let messages =
run_once(&cli.files, &settings, cache_enabled, &fixer::Mode::None);
let messages = commands::run(
&cli.files,
&strategy,
&overrides,
cache_enabled,
&fixer::Mode::None,
)?;
printer.write_continuously(&messages)?;
}
}
Err(e) => return Err(e.into()),
Err(err) => return Err(err.into()),
}
}
} else if cli.add_noqa {
let modifications = add_noqa(&cli.files, &settings);
let modifications = commands::add_noqa(&cli.files, &strategy, &overrides)?;
if modifications > 0 && log_level >= LogLevel::Default {
println!("Added {modifications} noqa directives.");
}
} else if cli.autoformat {
let modifications = autoformat(&cli.files, &settings);
let modifications = commands::autoformat(&cli.files, &strategy, &overrides)?;
if modifications > 0 && log_level >= LogLevel::Default {
println!("Formatted {modifications} files.");
}
@@ -337,16 +198,16 @@ fn inner_main() -> Result<ExitCode> {
let diagnostics = if is_stdin {
let filename = cli.stdin_filename.unwrap_or_else(|| "-".to_string());
let path = Path::new(&filename);
run_once_stdin(&settings, path, &fix)?
commands::run_stdin(&strategy, path, &autofix)?
} else {
run_once(&cli.files, &settings, cache_enabled, &fix)
commands::run(&cli.files, &strategy, &overrides, cache_enabled, &autofix)?
};
// Always try to print violations (the printer itself may suppress output),
// unless we're writing fixes via stdin (in which case, the transformed
// source code goes to stdout).
if !(is_stdin && matches!(fix, fixer::Mode::Apply)) {
printer.write_once(&diagnostics, &fix)?;
if !(is_stdin && matches!(autofix, fixer::Mode::Apply)) {
printer.write_once(&diagnostics, &autofix)?;
}
// Check for updates if we're in a non-silent log level.

414
src/resolver.rs Normal file
View File

@@ -0,0 +1,414 @@
//! Discover Python files, and their corresponding `Settings`, from the
//! filesystem.
use std::collections::BTreeMap;
use std::path::{Path, PathBuf};
use anyhow::{bail, Result};
use log::debug;
use path_absolutize::path_dedot;
use rustc_hash::FxHashSet;
use walkdir::{DirEntry, WalkDir};
use crate::cli::Overrides;
use crate::fs;
use crate::settings::configuration::Configuration;
use crate::settings::{pyproject, Settings};
/// The strategy for discovering a `pyproject.toml` file for each Python file.
#[derive(Debug)]
pub enum Strategy {
/// Use a fixed `pyproject.toml` file for all Python files (i.e., one
/// provided on the command-line).
Fixed(Settings),
/// Use the closest `pyproject.toml` file in the filesystem hierarchy, or
/// the default settings.
Hierarchical(Settings),
}
/// The strategy for resolving file paths in a `pyproject.toml`.
pub enum Relativity {
/// Resolve file paths relative to the current working directory.
Cwd,
/// Resolve file paths relative to the directory containing the
/// `pyproject.toml`.
Parent,
}
impl Relativity {
pub fn resolve(&self, path: &Path) -> PathBuf {
match self {
Relativity::Parent => path
.parent()
.expect("Expected pyproject.toml file to be in parent directory")
.to_path_buf(),
Relativity::Cwd => path_dedot::CWD.clone(),
}
}
}
#[derive(Default)]
pub struct Resolver {
settings: BTreeMap<PathBuf, Settings>,
}
impl Resolver {
/// Merge a `Resolver` into the current `Resolver`.
pub fn merge(&mut self, resolver: Resolver) {
self.settings.extend(resolver.settings);
}
/// Add a resolved `Settings` under a given `PathBuf` scope.
pub fn add(&mut self, path: PathBuf, settings: Settings) {
self.settings.insert(path, settings);
}
/// Return the appropriate `Settings` for a given `Path`.
pub fn resolve<'a>(&'a self, path: &Path, strategy: &'a Strategy) -> &'a Settings {
match strategy {
Strategy::Fixed(settings) => settings,
Strategy::Hierarchical(default) => self
.settings
.iter()
.rev()
.find_map(|(root, settings)| {
if path.starts_with(root) {
Some(settings)
} else {
None
}
})
.unwrap_or(default),
}
}
}
/// Recursively resolve a `Configuration` from a `pyproject.toml` file at the
/// specified `Path`.
// TODO(charlie): This whole system could do with some caching. Right now, if a
// configuration file extends another in the same path, we'll re-parse the same
// file at least twice (possibly more than twice, since we'll also parse it when
// resolving the "default" configuration).
pub fn resolve_configuration(
pyproject: &Path,
relativity: &Relativity,
overrides: Option<&Overrides>,
) -> Result<Configuration> {
let mut seen = FxHashSet::default();
let mut stack = vec![];
let mut next = Some(fs::normalize_path(pyproject));
while let Some(path) = next {
if seen.contains(&path) {
bail!("Circular dependency detected in pyproject.toml");
}
// Resolve the current path.
let options = pyproject::load_options(&path)?;
let project_root = relativity.resolve(&path);
let configuration = Configuration::from_options(options, &project_root)?;
// If extending, continue to collect.
next = configuration.extend.as_ref().map(|extend| {
fs::normalize_path_to(
extend,
path.parent()
.expect("Expected pyproject.toml file to be in parent directory"),
)
});
// Keep track of (1) the paths we've already resolved (to avoid cycles), and (2)
// the base configuration for every path.
seen.insert(path);
stack.push(configuration);
}
// Merge the configurations, in order.
stack.reverse();
let mut configuration = stack.pop().unwrap();
while let Some(extend) = stack.pop() {
configuration = configuration.combine(extend);
}
if let Some(overrides) = overrides {
configuration.apply(overrides.clone());
}
Ok(configuration)
}
/// Extract the project root (scope) and `Settings` from a given
/// `pyproject.toml`.
pub fn resolve_scoped_settings(
pyproject: &Path,
relativity: &Relativity,
overrides: Option<&Overrides>,
) -> Result<(PathBuf, Settings)> {
let project_root = relativity.resolve(pyproject);
let configuration = resolve_configuration(pyproject, relativity, overrides)?;
let settings = Settings::from_configuration(configuration, &project_root)?;
Ok((project_root, settings))
}
/// Extract the `Settings` from a given `pyproject.toml`.
pub fn resolve_settings(
pyproject: &Path,
relativity: &Relativity,
overrides: Option<&Overrides>,
) -> Result<Settings> {
let (_project_root, settings) = resolve_scoped_settings(pyproject, relativity, overrides)?;
Ok(settings)
}
/// Return `true` if the given file should be ignored based on the exclusion
/// criteria.
fn is_excluded(file_path: &str, file_basename: &str, exclude: &globset::GlobSet) -> bool {
exclude.is_match(file_path) || exclude.is_match(file_basename)
}
/// Return `true` if the `Path` appears to be that of a Python file.
fn is_python_file(path: &Path) -> bool {
path.extension()
.map_or(false, |ext| ext == "py" || ext == "pyi")
}
/// Find all Python (`.py` and `.pyi` files) in a set of `PathBuf`s.
pub fn resolve_python_files(
paths: &[PathBuf],
strategy: &Strategy,
overrides: &Overrides,
) -> Result<(Vec<Result<DirEntry, walkdir::Error>>, Resolver)> {
let mut files = Vec::new();
let mut resolver = Resolver::default();
for path in paths {
let (files_in_path, file_resolver) = python_files_in_path(path, strategy, overrides)?;
files.extend(files_in_path);
resolver.merge(file_resolver);
}
Ok((files, resolver))
}
/// Find all Python (`.py` and `.pyi` files) in a given `Path`.
fn python_files_in_path(
path: &Path,
strategy: &Strategy,
overrides: &Overrides,
) -> Result<(Vec<Result<DirEntry, walkdir::Error>>, Resolver)> {
let path = fs::normalize_path(path);
// Search for `pyproject.toml` files in all parent directories.
let mut resolver = Resolver::default();
for path in path.ancestors() {
if path.is_dir() {
let pyproject = path.join("pyproject.toml");
if pyproject.is_file() {
let (root, settings) =
resolve_scoped_settings(&pyproject, &Relativity::Parent, Some(overrides))?;
resolver.add(root, settings);
}
}
}
// Collect all Python files.
let files: Vec<Result<DirEntry, walkdir::Error>> = WalkDir::new(path)
.into_iter()
.filter_entry(|entry| {
// Search for the `pyproject.toml` file in this directory, before we visit any
// of its contents.
if entry.file_type().is_dir() {
let pyproject = entry.path().join("pyproject.toml");
if pyproject.is_file() {
// TODO(charlie): Return a `Result` here.
let (root, settings) =
resolve_scoped_settings(&pyproject, &Relativity::Parent, Some(overrides))
.unwrap();
resolver.add(root, settings);
}
}
let path = entry.path();
let settings = resolver.resolve(path, strategy);
match fs::extract_path_names(path) {
Ok((file_path, file_basename)) => {
if !settings.exclude.is_empty()
&& is_excluded(file_path, file_basename, &settings.exclude)
{
debug!("Ignored path via `exclude`: {:?}", path);
false
} else if !settings.extend_exclude.is_empty()
&& is_excluded(file_path, file_basename, &settings.extend_exclude)
{
debug!("Ignored path via `extend-exclude`: {:?}", path);
false
} else {
true
}
}
Err(e) => {
debug!("Ignored path due to error in parsing: {:?}: {}", path, e);
true
}
}
})
.filter(|entry| {
entry.as_ref().map_or(true, |entry| {
(entry.depth() == 0 || is_python_file(entry.path()))
&& !entry.file_type().is_dir()
&& !(entry.file_type().is_symlink() && entry.path().is_dir())
})
})
.collect::<Vec<_>>();
Ok((files, resolver))
}
#[cfg(test)]
mod tests {
use std::path::Path;
use anyhow::Result;
use globset::GlobSet;
use path_absolutize::Absolutize;
use crate::fs;
use crate::resolver::{is_excluded, is_python_file};
use crate::settings::types::FilePattern;
#[test]
fn inclusions() {
let path = Path::new("foo/bar/baz.py").absolutize().unwrap();
assert!(is_python_file(&path));
let path = Path::new("foo/bar/baz.pyi").absolutize().unwrap();
assert!(is_python_file(&path));
let path = Path::new("foo/bar/baz.js").absolutize().unwrap();
assert!(!is_python_file(&path));
let path = Path::new("foo/bar/baz").absolutize().unwrap();
assert!(!is_python_file(&path));
}
fn make_exclusion(file_pattern: FilePattern) -> GlobSet {
let mut builder = globset::GlobSetBuilder::new();
file_pattern.add_to(&mut builder).unwrap();
builder.build().unwrap()
}
#[test]
fn exclusions() -> Result<()> {
let project_root = Path::new("/tmp/");
let path = Path::new("foo").absolutize_from(project_root).unwrap();
let exclude = FilePattern::User(
"foo".to_string(),
Path::new("foo")
.absolutize_from(project_root)
.unwrap()
.to_path_buf(),
);
let (file_path, file_basename) = fs::extract_path_names(&path)?;
assert!(is_excluded(
file_path,
file_basename,
&make_exclusion(exclude,)
));
let path = Path::new("foo/bar").absolutize_from(project_root).unwrap();
let exclude = FilePattern::User(
"bar".to_string(),
Path::new("bar")
.absolutize_from(project_root)
.unwrap()
.to_path_buf(),
);
let (file_path, file_basename) = fs::extract_path_names(&path)?;
assert!(is_excluded(
file_path,
file_basename,
&make_exclusion(exclude,)
));
let path = Path::new("foo/bar/baz.py")
.absolutize_from(project_root)
.unwrap();
let exclude = FilePattern::User(
"baz.py".to_string(),
Path::new("baz.py")
.absolutize_from(project_root)
.unwrap()
.to_path_buf(),
);
let (file_path, file_basename) = fs::extract_path_names(&path)?;
assert!(is_excluded(
file_path,
file_basename,
&make_exclusion(exclude,)
));
let path = Path::new("foo/bar").absolutize_from(project_root).unwrap();
let exclude = FilePattern::User(
"foo/bar".to_string(),
Path::new("foo/bar")
.absolutize_from(project_root)
.unwrap()
.to_path_buf(),
);
let (file_path, file_basename) = fs::extract_path_names(&path)?;
assert!(is_excluded(
file_path,
file_basename,
&make_exclusion(exclude,)
));
let path = Path::new("foo/bar/baz.py")
.absolutize_from(project_root)
.unwrap();
let exclude = FilePattern::User(
"foo/bar/baz.py".to_string(),
Path::new("foo/bar/baz.py")
.absolutize_from(project_root)
.unwrap()
.to_path_buf(),
);
let (file_path, file_basename) = fs::extract_path_names(&path)?;
assert!(is_excluded(
file_path,
file_basename,
&make_exclusion(exclude,)
));
let path = Path::new("foo/bar/baz.py")
.absolutize_from(project_root)
.unwrap();
let exclude = FilePattern::User(
"foo/bar/*.py".to_string(),
Path::new("foo/bar/*.py")
.absolutize_from(project_root)
.unwrap()
.to_path_buf(),
);
let (file_path, file_basename) = fs::extract_path_names(&path)?;
assert!(is_excluded(
file_path,
file_basename,
&make_exclusion(exclude,)
));
let path = Path::new("foo/bar/baz.py")
.absolutize_from(project_root)
.unwrap();
let exclude = FilePattern::User(
"baz".to_string(),
Path::new("baz")
.absolutize_from(project_root)
.unwrap()
.to_path_buf(),
);
let (file_path, file_basename) = fs::extract_path_names(&path)?;
assert!(!is_excluded(
file_path,
file_basename,
&make_exclusion(exclude,)
));
Ok(())
}
}

View File

@@ -5,13 +5,12 @@
use std::path::{Path, PathBuf};
use anyhow::{anyhow, Result};
use once_cell::sync::Lazy;
use path_absolutize::path_dedot;
use glob::{glob, GlobError, Paths, PatternError};
use regex::Regex;
use rustc_hash::FxHashSet;
use crate::checks_gen::{CheckCodePrefix, CATEGORIES};
use crate::checks_gen::CheckCodePrefix;
use crate::cli::{collect_per_file_ignores, Overrides};
use crate::settings::options::Options;
use crate::settings::pyproject::load_options;
use crate::settings::types::{FilePattern, PerFileIgnore, PythonVersion, SerializationFormat};
use crate::{
@@ -19,222 +18,217 @@ use crate::{
flake8_tidy_imports, fs, isort, mccabe, pep8_naming, pyupgrade,
};
#[derive(Debug)]
#[derive(Debug, Default)]
pub struct Configuration {
pub allowed_confusables: FxHashSet<char>,
pub dummy_variable_rgx: Regex,
pub exclude: Vec<FilePattern>,
pub extend_exclude: Vec<FilePattern>,
pub extend_ignore: Vec<CheckCodePrefix>,
pub extend_select: Vec<CheckCodePrefix>,
pub external: Vec<String>,
pub fix: bool,
pub fixable: Vec<CheckCodePrefix>,
pub format: SerializationFormat,
pub ignore: Vec<CheckCodePrefix>,
pub ignore_init_module_imports: bool,
pub line_length: usize,
pub per_file_ignores: Vec<PerFileIgnore>,
pub select: Vec<CheckCodePrefix>,
pub show_source: bool,
pub src: Vec<PathBuf>,
pub target_version: PythonVersion,
pub unfixable: Vec<CheckCodePrefix>,
pub allowed_confusables: Option<Vec<char>>,
pub dummy_variable_rgx: Option<Regex>,
pub exclude: Option<Vec<FilePattern>>,
pub extend: Option<PathBuf>,
pub extend_exclude: Option<Vec<FilePattern>>,
pub extend_ignore: Option<Vec<CheckCodePrefix>>,
pub extend_select: Option<Vec<CheckCodePrefix>>,
pub external: Option<Vec<String>>,
pub fix: Option<bool>,
pub fixable: Option<Vec<CheckCodePrefix>>,
pub format: Option<SerializationFormat>,
pub ignore: Option<Vec<CheckCodePrefix>>,
pub ignore_init_module_imports: Option<bool>,
pub line_length: Option<usize>,
pub per_file_ignores: Option<Vec<PerFileIgnore>>,
pub select: Option<Vec<CheckCodePrefix>>,
pub show_source: Option<bool>,
pub src: Option<Vec<PathBuf>>,
pub target_version: Option<PythonVersion>,
pub unfixable: Option<Vec<CheckCodePrefix>>,
// Plugins
pub flake8_annotations: flake8_annotations::settings::Settings,
pub flake8_bugbear: flake8_bugbear::settings::Settings,
pub flake8_import_conventions: flake8_import_conventions::settings::Settings,
pub flake8_quotes: flake8_quotes::settings::Settings,
pub flake8_tidy_imports: flake8_tidy_imports::settings::Settings,
pub isort: isort::settings::Settings,
pub mccabe: mccabe::settings::Settings,
pub pep8_naming: pep8_naming::settings::Settings,
pub pyupgrade: pyupgrade::settings::Settings,
pub flake8_annotations: Option<flake8_annotations::settings::Options>,
pub flake8_bugbear: Option<flake8_bugbear::settings::Options>,
pub flake8_import_conventions: Option<flake8_import_conventions::settings::Options>,
pub flake8_quotes: Option<flake8_quotes::settings::Options>,
pub flake8_tidy_imports: Option<flake8_tidy_imports::settings::Options>,
pub isort: Option<isort::settings::Options>,
pub mccabe: Option<mccabe::settings::Options>,
pub pep8_naming: Option<pep8_naming::settings::Options>,
pub pyupgrade: Option<pyupgrade::settings::Options>,
}
static DEFAULT_EXCLUDE: Lazy<Vec<FilePattern>> = Lazy::new(|| {
vec![
FilePattern::Builtin(".bzr"),
FilePattern::Builtin(".direnv"),
FilePattern::Builtin(".eggs"),
FilePattern::Builtin(".git"),
FilePattern::Builtin(".hg"),
FilePattern::Builtin(".mypy_cache"),
FilePattern::Builtin(".nox"),
FilePattern::Builtin(".pants.d"),
FilePattern::Builtin(".ruff_cache"),
FilePattern::Builtin(".svn"),
FilePattern::Builtin(".tox"),
FilePattern::Builtin(".venv"),
FilePattern::Builtin("__pypackages__"),
FilePattern::Builtin("_build"),
FilePattern::Builtin("buck-out"),
FilePattern::Builtin("build"),
FilePattern::Builtin("dist"),
FilePattern::Builtin("node_modules"),
FilePattern::Builtin("venv"),
]
});
static DEFAULT_DUMMY_VARIABLE_RGX: Lazy<Regex> =
Lazy::new(|| Regex::new("^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$").unwrap());
impl Configuration {
pub fn from_pyproject(
pyproject: Option<&PathBuf>,
project_root: Option<&PathBuf>,
) -> Result<Self> {
let options = load_options(pyproject)?;
pub fn from_pyproject(pyproject: &Path, project_root: &Path) -> Result<Self> {
Self::from_options(load_options(pyproject)?, project_root)
}
pub fn from_options(options: Options, project_root: &Path) -> Result<Self> {
Ok(Configuration {
allowed_confusables: FxHashSet::from_iter(
options.allowed_confusables.unwrap_or_default(),
),
dummy_variable_rgx: match options.dummy_variable_rgx {
Some(pattern) => Regex::new(&pattern)
.map_err(|e| anyhow!("Invalid `dummy-variable-rgx` value: {e}"))?,
None => DEFAULT_DUMMY_VARIABLE_RGX.clone(),
},
src: options.src.map_or_else(
|| {
vec![match project_root {
Some(project_root) => project_root.clone(),
None => path_dedot::CWD.clone(),
}]
},
|src| {
src.iter()
.map(|path| {
let path = Path::new(path);
match project_root {
Some(project_root) => fs::normalize_path_to(path, project_root),
None => fs::normalize_path(path),
}
})
.collect()
},
),
target_version: options.target_version.unwrap_or(PythonVersion::Py310),
exclude: options.exclude.map_or_else(
|| DEFAULT_EXCLUDE.clone(),
|paths| paths.into_iter().map(FilePattern::User).collect(),
),
extend_exclude: options
.extend_exclude
.map(|paths| paths.into_iter().map(FilePattern::User).collect())
.unwrap_or_default(),
extend_ignore: options.extend_ignore.unwrap_or_default(),
select: options
.select
.unwrap_or_else(|| vec![CheckCodePrefix::E, CheckCodePrefix::F]),
extend_select: options.extend_select.unwrap_or_default(),
external: options.external.unwrap_or_default(),
fix: options.fix.unwrap_or_default(),
fixable: options.fixable.unwrap_or_else(|| CATEGORIES.to_vec()),
unfixable: options.unfixable.unwrap_or_default(),
format: options.format.unwrap_or_default(),
ignore: options.ignore.unwrap_or_default(),
ignore_init_module_imports: options.ignore_init_module_imports.unwrap_or_default(),
line_length: options.line_length.unwrap_or(88),
per_file_ignores: options
.per_file_ignores
.map(|per_file_ignores| {
per_file_ignores
.into_iter()
.map(|(pattern, prefixes)| PerFileIgnore::new(pattern, &prefixes))
.collect()
})
.unwrap_or_default(),
show_source: options.show_source.unwrap_or_default(),
extend: options.extend.map(PathBuf::from),
allowed_confusables: options.allowed_confusables,
dummy_variable_rgx: options
.dummy_variable_rgx
.map(|pattern| Regex::new(&pattern))
.transpose()
.map_err(|e| anyhow!("Invalid `dummy-variable-rgx` value: {e}"))?,
src: options
.src
.map(|src| resolve_src(&src, project_root))
.transpose()?,
target_version: options.target_version,
exclude: options.exclude.map(|paths| {
paths
.into_iter()
.map(|pattern| {
let absolute = fs::normalize_path_to(Path::new(&pattern), project_root);
FilePattern::User(pattern, absolute)
})
.collect()
}),
extend_exclude: options.extend_exclude.map(|paths| {
paths
.into_iter()
.map(|pattern| {
let absolute = fs::normalize_path_to(Path::new(&pattern), project_root);
FilePattern::User(pattern, absolute)
})
.collect()
}),
extend_ignore: options.extend_ignore,
select: options.select,
extend_select: options.extend_select,
external: options.external,
fix: options.fix,
fixable: options.fixable,
unfixable: options.unfixable,
format: options.format,
ignore: options.ignore,
ignore_init_module_imports: options.ignore_init_module_imports,
line_length: options.line_length,
per_file_ignores: options.per_file_ignores.map(|per_file_ignores| {
per_file_ignores
.into_iter()
.map(|(pattern, prefixes)| {
let absolute = fs::normalize_path_to(Path::new(&pattern), project_root);
PerFileIgnore::new(pattern, absolute, &prefixes)
})
.collect()
}),
show_source: options.show_source,
// Plugins
flake8_annotations: options
.flake8_annotations
.map(flake8_annotations::settings::Settings::from_options)
.unwrap_or_default(),
flake8_bugbear: options
.flake8_bugbear
.map(flake8_bugbear::settings::Settings::from_options)
.unwrap_or_default(),
flake8_import_conventions: options
.flake8_import_conventions
.map(flake8_import_conventions::settings::Settings::from_options)
.unwrap_or_default(),
flake8_quotes: options
.flake8_quotes
.map(flake8_quotes::settings::Settings::from_options)
.unwrap_or_default(),
flake8_tidy_imports: options
.flake8_tidy_imports
.map(flake8_tidy_imports::settings::Settings::from_options)
.unwrap_or_default(),
isort: options
.isort
.map(isort::settings::Settings::from_options)
.unwrap_or_default(),
mccabe: options
.mccabe
.as_ref()
.map(mccabe::settings::Settings::from_options)
.unwrap_or_default(),
pep8_naming: options
.pep8_naming
.map(pep8_naming::settings::Settings::from_options)
.unwrap_or_default(),
pyupgrade: options
.pyupgrade
.as_ref()
.map(pyupgrade::settings::Settings::from_options)
.unwrap_or_default(),
flake8_annotations: options.flake8_annotations,
flake8_bugbear: options.flake8_bugbear,
flake8_import_conventions: options.flake8_import_conventions,
flake8_quotes: options.flake8_quotes,
flake8_tidy_imports: options.flake8_tidy_imports,
isort: options.isort,
mccabe: options.mccabe,
pep8_naming: options.pep8_naming,
pyupgrade: options.pyupgrade,
})
}
pub fn merge(&mut self, overrides: Overrides) {
#[must_use]
pub fn combine(self, config: Configuration) -> Self {
Self {
allowed_confusables: self.allowed_confusables.or(config.allowed_confusables),
dummy_variable_rgx: self.dummy_variable_rgx.or(config.dummy_variable_rgx),
exclude: self.exclude.or(config.exclude),
extend: self.extend.or(config.extend),
extend_exclude: self.extend_exclude.or(config.extend_exclude),
extend_ignore: self.extend_ignore.or(config.extend_ignore),
extend_select: self.extend_select.or(config.extend_select),
external: self.external.or(config.external),
fix: self.fix.or(config.fix),
fixable: self.fixable.or(config.fixable),
format: self.format.or(config.format),
ignore: self.ignore.or(config.ignore),
ignore_init_module_imports: self
.ignore_init_module_imports
.or(config.ignore_init_module_imports),
line_length: self.line_length.or(config.line_length),
per_file_ignores: self.per_file_ignores.or(config.per_file_ignores),
select: self.select.or(config.select),
show_source: self.show_source.or(config.show_source),
src: self.src.or(config.src),
target_version: self.target_version.or(config.target_version),
unfixable: self.unfixable.or(config.unfixable),
// Plugins
flake8_annotations: self.flake8_annotations.or(config.flake8_annotations),
flake8_bugbear: self.flake8_bugbear.or(config.flake8_bugbear),
flake8_import_conventions: self
.flake8_import_conventions
.or(config.flake8_import_conventions),
flake8_quotes: self.flake8_quotes.or(config.flake8_quotes),
flake8_tidy_imports: self.flake8_tidy_imports.or(config.flake8_tidy_imports),
isort: self.isort.or(config.isort),
mccabe: self.mccabe.or(config.mccabe),
pep8_naming: self.pep8_naming.or(config.pep8_naming),
pyupgrade: self.pyupgrade.or(config.pyupgrade),
}
}
pub fn apply(&mut self, overrides: Overrides) {
if let Some(dummy_variable_rgx) = overrides.dummy_variable_rgx {
self.dummy_variable_rgx = dummy_variable_rgx;
self.dummy_variable_rgx = Some(dummy_variable_rgx);
}
if let Some(exclude) = overrides.exclude {
self.exclude = exclude;
self.exclude = Some(exclude);
}
if let Some(extend_exclude) = overrides.extend_exclude {
self.extend_exclude = extend_exclude;
self.extend_exclude = Some(extend_exclude);
}
if let Some(extend_ignore) = overrides.extend_ignore {
self.extend_ignore = extend_ignore;
self.extend_ignore = Some(extend_ignore);
}
if let Some(extend_select) = overrides.extend_select {
self.extend_select = extend_select;
self.extend_select = Some(extend_select);
}
if let Some(fix) = overrides.fix {
self.fix = fix;
self.fix = Some(fix);
}
if let Some(fixable) = overrides.fixable {
self.fixable = fixable;
self.fixable = Some(fixable);
}
if let Some(format) = overrides.format {
self.format = format;
self.format = Some(format);
}
if let Some(ignore) = overrides.ignore {
self.ignore = ignore;
self.ignore = Some(ignore);
}
if let Some(line_length) = overrides.line_length {
self.line_length = line_length;
self.line_length = Some(line_length);
}
if let Some(max_complexity) = overrides.max_complexity {
self.mccabe.max_complexity = max_complexity;
self.mccabe = Some(mccabe::settings::Options {
max_complexity: Some(max_complexity),
});
}
if let Some(per_file_ignores) = overrides.per_file_ignores {
self.per_file_ignores = collect_per_file_ignores(per_file_ignores);
self.per_file_ignores = Some(collect_per_file_ignores(per_file_ignores));
}
if let Some(select) = overrides.select {
self.select = select;
self.select = Some(select);
}
if let Some(show_source) = overrides.show_source {
self.show_source = show_source;
self.show_source = Some(show_source);
}
if let Some(target_version) = overrides.target_version {
self.target_version = target_version;
self.target_version = Some(target_version);
}
if let Some(unfixable) = overrides.unfixable {
self.unfixable = unfixable;
self.unfixable = Some(unfixable);
}
}
}
/// Given a list of source paths, which could include glob patterns, resolve the
/// matching paths.
pub fn resolve_src(src: &[String], project_root: &Path) -> Result<Vec<PathBuf>> {
let globs = src
.iter()
.map(Path::new)
.map(|path| fs::normalize_path_to(path, project_root))
.map(|path| glob(&path.to_string_lossy()))
.collect::<Result<Vec<Paths>, PatternError>>()?;
let paths: Vec<PathBuf> = globs
.into_iter()
.flatten()
.collect::<Result<Vec<PathBuf>, GlobError>>()?;
Ok(paths)
}

View File

@@ -8,17 +8,18 @@ use std::path::{Path, PathBuf};
use anyhow::Result;
use globset::{Glob, GlobMatcher, GlobSet};
use itertools::Itertools;
use once_cell::sync::Lazy;
use path_absolutize::path_dedot;
use regex::Regex;
use rustc_hash::FxHashSet;
use crate::checks::CheckCode;
use crate::checks_gen::{CheckCodePrefix, SuffixLength};
use crate::checks_gen::{CheckCodePrefix, SuffixLength, CATEGORIES};
use crate::settings::configuration::Configuration;
use crate::settings::types::{FilePattern, PerFileIgnore, PythonVersion};
use crate::settings::types::{FilePattern, PerFileIgnore, PythonVersion, SerializationFormat};
use crate::{
flake8_annotations, flake8_bugbear, flake8_import_conventions, flake8_quotes,
flake8_tidy_imports, fs, isort, mccabe, pep8_naming, pyupgrade,
flake8_tidy_imports, isort, mccabe, pep8_naming, pyupgrade,
};
pub mod configuration;
@@ -35,7 +36,9 @@ pub struct Settings {
pub exclude: GlobSet,
pub extend_exclude: GlobSet,
pub external: FxHashSet<String>,
pub fix: bool,
pub fixable: FxHashSet<CheckCode>,
pub format: SerializationFormat,
pub ignore_init_module_imports: bool,
pub line_length: usize,
pub per_file_ignores: Vec<(GlobMatcher, GlobMatcher, FxHashSet<CheckCode>)>,
@@ -54,45 +57,115 @@ pub struct Settings {
pub pyupgrade: pyupgrade::settings::Settings,
}
static DEFAULT_EXCLUDE: Lazy<Vec<FilePattern>> = Lazy::new(|| {
vec![
FilePattern::Builtin(".bzr"),
FilePattern::Builtin(".direnv"),
FilePattern::Builtin(".eggs"),
FilePattern::Builtin(".git"),
FilePattern::Builtin(".hg"),
FilePattern::Builtin(".mypy_cache"),
FilePattern::Builtin(".nox"),
FilePattern::Builtin(".pants.d"),
FilePattern::Builtin(".ruff_cache"),
FilePattern::Builtin(".svn"),
FilePattern::Builtin(".tox"),
FilePattern::Builtin(".venv"),
FilePattern::Builtin("__pypackages__"),
FilePattern::Builtin("_build"),
FilePattern::Builtin("buck-out"),
FilePattern::Builtin("build"),
FilePattern::Builtin("dist"),
FilePattern::Builtin("node_modules"),
FilePattern::Builtin("venv"),
]
});
static DEFAULT_DUMMY_VARIABLE_RGX: Lazy<Regex> =
Lazy::new(|| Regex::new("^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$").unwrap());
impl Settings {
pub fn from_configuration(
config: Configuration,
project_root: Option<&PathBuf>,
) -> Result<Self> {
pub fn from_configuration(config: Configuration, project_root: &Path) -> Result<Self> {
Ok(Self {
allowed_confusables: config.allowed_confusables,
dummy_variable_rgx: config.dummy_variable_rgx,
allowed_confusables: config
.allowed_confusables
.map(FxHashSet::from_iter)
.unwrap_or_default(),
dummy_variable_rgx: config
.dummy_variable_rgx
.unwrap_or_else(|| DEFAULT_DUMMY_VARIABLE_RGX.clone()),
enabled: resolve_codes(
&config
.select
.unwrap_or_else(|| vec![CheckCodePrefix::E, CheckCodePrefix::F])
.into_iter()
.chain(config.extend_select.into_iter())
.chain(config.extend_select.unwrap_or_default().into_iter())
.collect::<Vec<_>>(),
&config
.ignore
.unwrap_or_default()
.into_iter()
.chain(config.extend_ignore.into_iter())
.chain(config.extend_ignore.unwrap_or_default().into_iter())
.collect::<Vec<_>>(),
),
exclude: resolve_globset(config.exclude, project_root)?,
extend_exclude: resolve_globset(config.extend_exclude, project_root)?,
external: FxHashSet::from_iter(config.external),
fixable: resolve_codes(&config.fixable, &config.unfixable),
flake8_annotations: config.flake8_annotations,
flake8_bugbear: config.flake8_bugbear,
flake8_import_conventions: config.flake8_import_conventions,
flake8_quotes: config.flake8_quotes,
flake8_tidy_imports: config.flake8_tidy_imports,
ignore_init_module_imports: config.ignore_init_module_imports,
isort: config.isort,
mccabe: config.mccabe,
line_length: config.line_length,
pep8_naming: config.pep8_naming,
pyupgrade: config.pyupgrade,
per_file_ignores: resolve_per_file_ignores(config.per_file_ignores, project_root)?,
src: config.src,
target_version: config.target_version,
show_source: config.show_source,
exclude: resolve_globset(config.exclude.unwrap_or_else(|| DEFAULT_EXCLUDE.clone()))?,
extend_exclude: resolve_globset(config.extend_exclude.unwrap_or_default())?,
external: FxHashSet::from_iter(config.external.unwrap_or_default()),
fix: config.fix.unwrap_or(false),
fixable: resolve_codes(
&config.fixable.unwrap_or_else(|| CATEGORIES.to_vec()),
&config.unfixable.unwrap_or_default(),
),
format: config.format.unwrap_or(SerializationFormat::Text),
ignore_init_module_imports: config.ignore_init_module_imports.unwrap_or_default(),
line_length: config.line_length.unwrap_or(88),
per_file_ignores: resolve_per_file_ignores(
config.per_file_ignores.unwrap_or_default(),
)?,
src: config
.src
.unwrap_or_else(|| vec![project_root.to_path_buf()]),
target_version: config.target_version.unwrap_or(PythonVersion::Py310),
show_source: config.show_source.unwrap_or_default(),
// Plugins
flake8_annotations: config
.flake8_annotations
.map(flake8_annotations::settings::Settings::from_options)
.unwrap_or_default(),
flake8_bugbear: config
.flake8_bugbear
.map(flake8_bugbear::settings::Settings::from_options)
.unwrap_or_default(),
flake8_import_conventions: config
.flake8_import_conventions
.map(flake8_import_conventions::settings::Settings::from_options)
.unwrap_or_default(),
flake8_quotes: config
.flake8_quotes
.map(flake8_quotes::settings::Settings::from_options)
.unwrap_or_default(),
flake8_tidy_imports: config
.flake8_tidy_imports
.map(flake8_tidy_imports::settings::Settings::from_options)
.unwrap_or_default(),
isort: config
.isort
.map(isort::settings::Settings::from_options)
.unwrap_or_default(),
mccabe: config
.mccabe
.as_ref()
.map(mccabe::settings::Settings::from_options)
.unwrap_or_default(),
pep8_naming: config
.pep8_naming
.map(pep8_naming::settings::Settings::from_options)
.unwrap_or_default(),
pyupgrade: config
.pyupgrade
.as_ref()
.map(pyupgrade::settings::Settings::from_options)
.unwrap_or_default(),
})
}
@@ -104,7 +177,9 @@ impl Settings {
exclude: GlobSet::empty(),
extend_exclude: GlobSet::empty(),
external: FxHashSet::default(),
fix: false,
fixable: FxHashSet::from_iter([check_code]),
format: SerializationFormat::Text,
ignore_init_module_imports: false,
line_length: 88,
per_file_ignores: vec![],
@@ -131,7 +206,9 @@ impl Settings {
exclude: GlobSet::empty(),
extend_exclude: GlobSet::empty(),
external: FxHashSet::default(),
fix: false,
fixable: FxHashSet::from_iter(check_codes),
format: SerializationFormat::Text,
ignore_init_module_imports: false,
line_length: 88,
per_file_ignores: vec![],
@@ -192,13 +269,10 @@ impl Hash for Settings {
}
/// Given a list of patterns, create a `GlobSet`.
pub fn resolve_globset(
patterns: Vec<FilePattern>,
project_root: Option<&PathBuf>,
) -> Result<GlobSet> {
pub fn resolve_globset(patterns: Vec<FilePattern>) -> Result<GlobSet> {
let mut builder = globset::GlobSetBuilder::new();
for pattern in patterns {
pattern.add_to(&mut builder, project_root)?;
pattern.add_to(&mut builder)?;
}
builder.build().map_err(std::convert::Into::into)
}
@@ -206,21 +280,16 @@ pub fn resolve_globset(
/// Given a list of patterns, create a `GlobSet`.
pub fn resolve_per_file_ignores(
per_file_ignores: Vec<PerFileIgnore>,
project_root: Option<&PathBuf>,
) -> Result<Vec<(GlobMatcher, GlobMatcher, FxHashSet<CheckCode>)>> {
per_file_ignores
.into_iter()
.map(|per_file_ignore| {
// Construct absolute path matcher.
let path = Path::new(&per_file_ignore.pattern);
let absolute_path = match project_root {
Some(project_root) => fs::normalize_path_to(path, project_root),
None => fs::normalize_path(path),
};
let absolute = Glob::new(&absolute_path.to_string_lossy())?.compile_matcher();
let absolute =
Glob::new(&per_file_ignore.absolute.to_string_lossy())?.compile_matcher();
// Construct basename matcher.
let basename = Glob::new(&per_file_ignore.pattern)?.compile_matcher();
let basename = Glob::new(&per_file_ignore.basename)?.compile_matcher();
Ok((absolute, basename, per_file_ignore.codes))
})

View File

@@ -55,7 +55,7 @@ pub struct Options {
(to exclude any Python files in `directory`). Note that these paths are relative to the
project root (e.g., the directory containing your `pyproject.toml`).
Note that you'll typically want to use [`extend_exclude`](#extend_exclude) to modify
Note that you'll typically want to use [`extend-exclude`](#extend-exclude) to modify
the excluded paths.
"#,
default = r#"[".bzr", ".direnv", ".eggs", ".git", ".hg", ".mypy_cache", ".nox", ".pants.d", ".ruff_cache", ".svn", ".tox", ".venv", "__pypackages__", "_build", "buck-out", "build", "dist", "node_modules", "venv"]"#,
@@ -65,6 +65,24 @@ pub struct Options {
"#
)]
pub exclude: Option<Vec<String>>,
#[option(
doc = r#"
A path to a local `pyproject.toml` file to merge into this configuration.
To resolve the current `pyproject.toml` file, Ruff will first resolve this base
configuration file, then merge in any properties defined in the current configuration
file.
"#,
default = r#"None"#,
value_type = "Path",
example = r#"
# Extend the `pyproject.toml` file in the parent directory.
extend = "../pyproject.toml"
# But use a different line length.
line-length = 100
"#
)]
pub extend: Option<String>,
#[option(
doc = "A list of file patterns to omit from linting, in addition to those specified by \
`exclude`.",
@@ -217,8 +235,28 @@ pub struct Options {
)]
pub show_source: Option<bool>,
#[option(
doc = "The source code paths to consider, e.g., when resolving first- vs. third-party \
imports.",
doc = r#"
The source code paths to consider, e.g., when resolving first- vs. third-party imports.
As an example: given a Python package structure like:
```text
my_package/
pyproject.toml
src/
my_package/
__init__.py
foo.py
bar.py
```
The `src` directory should be included in `source` (e.g., `source = ["src"]`), such that
when resolving imports, `my_package.foo` is considered a first-party import.
This field supports globs. For example, if you have a series of Python packages in
a `python_modules` directory, `src = ["python_modules/*"]` would expand to incorporate
all of the packages in that directory.
"#,
default = r#"["."]"#,
value_type = "Vec<PathBuf>",
example = r#"

View File

@@ -3,9 +3,6 @@
use std::path::{Path, PathBuf};
use anyhow::{anyhow, Result};
use common_path::common_path_all;
use log::debug;
use path_absolutize::Absolutize;
use serde::{Deserialize, Serialize};
use crate::fs;
@@ -33,21 +30,21 @@ impl Pyproject {
fn parse_pyproject_toml(path: &Path) -> Result<Pyproject> {
let contents = fs::read_file(path)?;
Ok(toml::from_str(&contents)?)
toml::from_str(&contents).map_err(std::convert::Into::into)
}
pub fn find_pyproject_toml(path: Option<&PathBuf>) -> Option<PathBuf> {
if let Some(path) = path {
let path_pyproject_toml = path.join("pyproject.toml");
if path_pyproject_toml.is_file() {
return Some(path_pyproject_toml);
/// Find the nearest `pyproject.toml` file.
pub fn find_pyproject_toml(path: &Path) -> Option<PathBuf> {
for directory in path.ancestors() {
let pyproject = directory.join("pyproject.toml");
if pyproject.is_file() {
return Some(pyproject);
}
}
find_user_pyproject_toml()
None
}
fn find_user_pyproject_toml() -> Option<PathBuf> {
pub fn find_user_pyproject_toml() -> Option<PathBuf> {
let mut path = dirs::config_dir()?;
path.push("ruff");
path.push("pyproject.toml");
@@ -58,46 +55,17 @@ fn find_user_pyproject_toml() -> Option<PathBuf> {
}
}
pub fn find_project_root(sources: &[PathBuf]) -> Option<PathBuf> {
let absolute_sources: Vec<PathBuf> = sources
.iter()
.flat_map(|source| source.absolutize().map(|path| path.to_path_buf()))
.collect();
if let Some(prefix) = common_path_all(absolute_sources.iter().map(PathBuf::as_path)) {
for directory in prefix.ancestors() {
if directory.join(".git").is_dir() {
return Some(directory.to_path_buf());
}
if directory.join(".hg").is_dir() {
return Some(directory.to_path_buf());
}
if directory.join("pyproject.toml").is_file() {
return Some(directory.to_path_buf());
}
}
}
None
}
pub fn load_options(pyproject: Option<&PathBuf>) -> Result<Options> {
if let Some(pyproject) = pyproject {
Ok(parse_pyproject_toml(pyproject)
.map_err(|err| anyhow!("Failed to parse `{}`: {}", pyproject.to_string_lossy(), err))?
.tool
.and_then(|tool| tool.ruff)
.unwrap_or_default())
} else {
debug!("No pyproject.toml found.");
debug!("Falling back to default configuration...");
Ok(Options::default())
}
pub fn load_options(pyproject: &Path) -> Result<Options> {
Ok(parse_pyproject_toml(pyproject)
.map_err(|err| anyhow!("Failed to parse `{}`: {}", pyproject.to_string_lossy(), err))?
.tool
.and_then(|tool| tool.ruff)
.unwrap_or_default())
}
#[cfg(test)]
mod tests {
use std::env::current_dir;
use std::path::PathBuf;
use std::str::FromStr;
use anyhow::Result;
@@ -107,7 +75,7 @@ mod tests {
use crate::flake8_quotes::settings::Quote;
use crate::flake8_tidy_imports::settings::Strictness;
use crate::settings::pyproject::{
find_project_root, find_pyproject_toml, parse_pyproject_toml, Options, Pyproject, Tools,
find_pyproject_toml, parse_pyproject_toml, Options, Pyproject, Tools,
};
use crate::settings::types::PatternPrefixPair;
use crate::{
@@ -140,6 +108,7 @@ mod tests {
allowed_confusables: None,
dummy_variable_rgx: None,
exclude: None,
extend: None,
extend_exclude: None,
extend_ignore: None,
extend_select: None,
@@ -183,6 +152,7 @@ line-length = 79
allowed_confusables: None,
dummy_variable_rgx: None,
exclude: None,
extend: None,
extend_exclude: None,
extend_ignore: None,
extend_select: None,
@@ -226,6 +196,7 @@ exclude = ["foo.py"]
allowed_confusables: None,
line_length: None,
fix: None,
extend: None,
exclude: Some(vec!["foo.py".to_string()]),
extend_exclude: None,
select: None,
@@ -269,6 +240,7 @@ select = ["E501"]
allowed_confusables: None,
dummy_variable_rgx: None,
exclude: None,
extend: None,
extend_exclude: None,
extend_ignore: None,
extend_select: None,
@@ -313,6 +285,7 @@ ignore = ["E501"]
allowed_confusables: None,
dummy_variable_rgx: None,
exclude: None,
extend: None,
extend_exclude: None,
extend_ignore: None,
extend_select: Some(vec![CheckCodePrefix::RUF100]),
@@ -376,14 +349,14 @@ other-attribute = 1
#[test]
fn find_and_parse_pyproject_toml() -> Result<()> {
let cwd = current_dir()?;
let project_root =
find_project_root(&[PathBuf::from("resources/test/fixtures/__init__.py")]).unwrap();
assert_eq!(project_root, cwd.join("resources/test/fixtures"));
let pyproject =
find_pyproject_toml(&cwd.join("resources/test/fixtures/__init__.py")).unwrap();
assert_eq!(
pyproject,
cwd.join("resources/test/fixtures/pyproject.toml")
);
let path = find_pyproject_toml(Some(&project_root)).unwrap();
assert_eq!(path, cwd.join("resources/test/fixtures/pyproject.toml"));
let pyproject = parse_pyproject_toml(&path)?;
let pyproject = parse_pyproject_toml(&pyproject)?;
let config = pyproject.tool.and_then(|tool| tool.ruff).unwrap();
assert_eq!(
config,
@@ -392,6 +365,7 @@ other-attribute = 1
line_length: Some(88),
fix: None,
exclude: None,
extend: None,
extend_exclude: Some(vec![
"excluded_file.py".to_string(),
"migrations".to_string(),

View File

@@ -49,27 +49,18 @@ impl FromStr for PythonVersion {
#[derive(Debug, Clone)]
pub enum FilePattern {
Builtin(&'static str),
User(String),
User(String, PathBuf),
}
impl FilePattern {
pub fn add_to(
self,
builder: &mut GlobSetBuilder,
project_root: Option<&PathBuf>,
) -> Result<()> {
pub fn add_to(self, builder: &mut GlobSetBuilder) -> Result<()> {
match self {
FilePattern::Builtin(pattern) => {
builder.add(Glob::from_str(pattern)?);
}
FilePattern::User(pattern) => {
// Add absolute path.
let path = Path::new(&pattern);
let absolute_path = match project_root {
Some(project_root) => fs::normalize_path_to(path, project_root),
None => fs::normalize_path(path),
};
builder.add(Glob::new(&absolute_path.to_string_lossy())?);
FilePattern::User(pattern, absolute) => {
// Add the absolute path.
builder.add(Glob::new(&absolute.to_string_lossy())?);
// Add basename path.
if !pattern.contains(std::path::MAIN_SEPARATOR) {
@@ -85,20 +76,27 @@ impl FromStr for FilePattern {
type Err = anyhow::Error;
fn from_str(s: &str) -> Result<Self, Self::Err> {
Ok(Self::User(s.into()))
let pattern = s.to_string();
let absolute = fs::normalize_path(Path::new(&pattern));
Ok(Self::User(pattern, absolute))
}
}
#[derive(Debug, Clone)]
pub struct PerFileIgnore {
pub pattern: String,
pub basename: String,
pub absolute: PathBuf,
pub codes: FxHashSet<CheckCode>,
}
impl PerFileIgnore {
pub fn new(pattern: String, prefixes: &[CheckCodePrefix]) -> Self {
pub fn new(basename: String, absolute: PathBuf, prefixes: &[CheckCodePrefix]) -> Self {
let codes = prefixes.iter().flat_map(CheckCodePrefix::codes).collect();
Self { pattern, codes }
Self {
basename,
absolute,
codes,
}
}
}