Compare commits

...

19 Commits

Author SHA1 Message Date
Charlie Marsh
556ae00078 Bump version to 0.0.21 2022-08-31 11:25:46 -04:00
Charlie Marsh
adad214619 Handle submodule imports for F401 (#58) 2022-08-31 11:24:25 -04:00
Charlie Marsh
0ebed13e67 Sort messages prior to display (#56) 2022-08-31 10:53:35 -04:00
Charlie Marsh
3afedcd48b Upgrade parser to handle more F821 cases (#57) 2022-08-31 10:52:54 -04:00
Charlie Marsh
875e812188 Disable build-on-push for now 2022-08-31 10:45:41 -04:00
Charlie Marsh
80f3cd0ef7 Don't assume errors appear in-order with line contents 2022-08-31 08:40:38 -04:00
Dmitry Dygalo
9e3c35e6dc Improve search for duplicates (#53) 2022-08-31 08:23:43 -04:00
Charlie Marsh
1e67ce229f Increment to v0.0.20 2022-08-30 14:41:28 -04:00
Charlie Marsh
643797d922 Support builtins (#50) 2022-08-30 14:41:17 -04:00
Charlie Marsh
e9a3484edf Implement F821 (#49) 2022-08-30 14:37:06 -04:00
Charlie Marsh
dd759e4730 Update graph 2022-08-30 13:41:30 -04:00
Adrian Garcia Badaracco
e16fd39bb5 Build on every push but only publish on tags (#46) 2022-08-30 13:19:36 -04:00
Charlie Marsh
7ed5b3d3a2 Avoid re-reading + iterating over lines for ignores (#48) 2022-08-30 13:19:22 -04:00
Charlie Marsh
ae27793b86 Remove extraneous Dockerfile 2022-08-30 09:24:57 -04:00
Charlie Marsh
641ff8452e Tweak README 2022-08-30 09:23:15 -04:00
Charlie Marsh
53554a2bf1 Remove lint step from release.yaml 2022-08-30 09:16:35 -04:00
Adrian Garcia Badaracco
bd0ed1b96c Build ABI3 wheels and expand supported platforms (#45) 2022-08-30 09:16:07 -04:00
Charlie Marsh
0cbcb982eb Implement F823 (#44) 2022-08-29 23:04:44 -04:00
Charlie Marsh
6c5845922f Remove caveat from README 2022-08-29 22:09:22 -04:00
21 changed files with 866 additions and 299 deletions

View File

@@ -5,36 +5,46 @@ on:
tags:
- v*
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
env:
PACKAGE_NAME: ruff
PYTHON_VERSION: "3.7" # to build abi3 wheels
jobs:
macos:
runs-on: macos-latest
steps:
- uses: actions/checkout@v2
- uses: actions/setup-python@v2
- uses: actions/checkout@v3
- uses: actions/setup-python@v4
with:
python-version: "3.10"
python-version: ${{ env.PYTHON_VERSION }}
architecture: x64
- name: Install Rust toolchain
uses: actions-rs/toolchain@v1
with:
toolchain: stable
target: aarch64-apple-darwin
profile: minimal
default: true
- name: Build wheels - x86_64
uses: messense/maturin-action@v1
with:
target: x86_64
args: -i python --release --out dist --sdist
args: --release --out dist --sdist
maturin-version: "v0.13.0"
- name: Install built wheel - x86_64
run: |
pip install ruff --no-index --find-links dist --force-reinstall
pip install dist/${{ env.PACKAGE_NAME }}-*.whl --force-reinstall
- name: Build wheels - universal2
uses: messense/maturin-action@v1
with:
args: -i python --release --universal2 --out dist
args: --release --universal2 --out dist
maturin-version: "v0.13.0"
- name: Install built wheel - universal2
run: |
pip install ruff --no-index --find-links dist --force-reinstall
pip install dist/${{ env.PACKAGE_NAME }}-*universal2.whl --force-reinstall
- name: Upload wheels
uses: actions/upload-artifact@v2
with:
@@ -47,13 +57,11 @@ jobs:
matrix:
target: [x64, x86]
steps:
- uses: actions/checkout@v2
- uses: actions/setup-python@v2
- uses: actions/checkout@v3
- uses: actions/setup-python@v4
with:
python-version: "3.10"
python-version: ${{ env.PYTHON_VERSION }}
architecture: ${{ matrix.target }}
- name: Update rustup
run: rustup self update
- name: Install Rust toolchain
uses: actions-rs/toolchain@v1
with:
@@ -64,10 +72,12 @@ jobs:
uses: messense/maturin-action@v1
with:
target: ${{ matrix.target }}
args: -i python --release --out dist
args: --release --out dist
maturin-version: "v0.13.0"
- name: Install built wheel
shell: bash
run: |
pip install ruff --no-index --find-links dist --force-reinstall
python -m pip install dist/${{ env.PACKAGE_NAME }}-*.whl --force-reinstall
- name: Upload wheels
uses: actions/upload-artifact@v2
with:
@@ -80,87 +90,192 @@ jobs:
matrix:
target: [x86_64, i686]
steps:
- uses: actions/checkout@v2
- name: Install Rust toolchain
uses: actions-rs/toolchain@v1
with:
toolchain: stable
profile: minimal
default: true
- uses: actions/setup-python@v2
with:
python-version: "3.10"
- name: Build Wheels
uses: messense/maturin-action@v1
with:
target: ${{ matrix.target }}
manylinux: auto
args: -i 3.10 --release --out dist
- name: Install built wheel
if: matrix.target == 'x86_64'
run: |
pip install ruff --no-index --find-links dist --force-reinstall
- name: Upload wheels
uses: actions/upload-artifact@v2
with:
name: wheels
path: dist
- uses: actions/checkout@v3
- uses: actions/setup-python@v4
with:
python-version: ${{ env.PYTHON_VERSION }}
architecture: x64
- name: Build wheels
uses: messense/maturin-action@v1
with:
target: ${{ matrix.target }}
manylinux: auto
args: --release --out dist
maturin-version: "v0.13.0"
- name: Install built wheel
if: matrix.target == 'x86_64'
run: |
pip install dist/${{ env.PACKAGE_NAME }}-*.whl --force-reinstall
- name: Upload wheels
uses: actions/upload-artifact@v2
with:
name: wheels
path: dist
linux-cross:
runs-on: ubuntu-latest
strategy:
matrix:
target: [aarch64, armv7, s390x, ppc64le]
target: [aarch64, armv7, s390x, ppc64le, ppc64]
steps:
- uses: actions/checkout@v2
- name: Build Wheels
uses: messense/maturin-action@v1
with:
target: ${{ matrix.target }}
manylinux: auto
args: -i 3.10 --release --out dist
- uses: uraimo/run-on-arch-action@v2.2.0
name: Install built wheel
with:
arch: ${{ matrix.target }}
distro: ubuntu20.04
githubToken: ${{ github.token }}
dockerRunArgs: |
--volume "${PWD}/dist:/artifacts"
install: |
apt-get update
apt-get install -y --no-install-recommends python3 python3-venv software-properties-common
add-apt-repository ppa:deadsnakes/ppa
apt-get update
apt-get install -y curl python3.7-venv python3.9-venv python3.10-venv
- uses: actions/checkout@v3
- uses: actions/setup-python@v4
with:
python-version: ${{ env.PYTHON_VERSION }}
- name: Build wheels
uses: messense/maturin-action@v1
with:
target: ${{ matrix.target }}
manylinux: auto
args: --release --out dist
maturin-version: "v0.13.0"
- uses: uraimo/run-on-arch-action@v2.0.5
if: matrix.target != 'ppc64'
name: Install built wheel
with:
arch: ${{ matrix.target }}
distro: ubuntu20.04
githubToken: ${{ github.token }}
install: |
apt-get update
apt-get install -y --no-install-recommends python3 python3-pip
pip3 install -U pip
run: |
pip3 install ${{ env.PACKAGE_NAME }} --no-index --find-links dist/ --force-reinstall
- name: Upload wheels
uses: actions/upload-artifact@v2
with:
name: wheels
path: dist
musllinux:
runs-on: ubuntu-latest
strategy:
matrix:
target:
- x86_64-unknown-linux-musl
- i686-unknown-linux-musl
steps:
- uses: actions/checkout@v3
- uses: actions/setup-python@v4
with:
python-version: ${{ env.PYTHON_VERSION }}
architecture: x64
- name: Build wheels
uses: messense/maturin-action@v1
with:
target: ${{ matrix.target }}
manylinux: musllinux_1_2
args: --release --out dist
maturin-version: "v0.13.0"
- name: Install built wheel
if: matrix.target == 'x86_64-unknown-linux-musl'
uses: addnab/docker-run-action@v3
with:
image: alpine:latest
options: -v ${{ github.workspace }}:/io -w /io
run: |
apk add py3-pip
pip3 install ${{ env.PACKAGE_NAME }} --no-index --find-links /io/dist/ --force-reinstall
- name: Upload wheels
uses: actions/upload-artifact@v2
with:
name: wheels
path: dist
musllinux-cross:
runs-on: ubuntu-latest
strategy:
matrix:
platform:
- target: aarch64-unknown-linux-musl
arch: aarch64
- target: armv7-unknown-linux-musleabihf
arch: armv7
steps:
- uses: actions/checkout@v3
- uses: actions/setup-python@v4
with:
python-version: ${{ env.PYTHON_VERSION }}
- name: Build wheels
uses: messense/maturin-action@v1
with:
target: ${{ matrix.platform.target }}
manylinux: musllinux_1_2
args: --release --out dist
maturin-version: "v0.13.0"
- uses: uraimo/run-on-arch-action@master
name: Install built wheel
with:
arch: ${{ matrix.platform.arch }}
distro: alpine_latest
githubToken: ${{ github.token }}
install: |
apk add py3-pip
run: |
pip3 install ${{ env.PACKAGE_NAME }} --no-index --find-links dist/ --force-reinstall
- name: Upload wheels
uses: actions/upload-artifact@v2
with:
name: wheels
path: dist
pypy:
runs-on: ${{ matrix.os }}
strategy:
matrix:
os: [ubuntu-latest, macos-latest]
target: [x86_64, aarch64]
python-version:
- '3.7'
- '3.8'
- '3.9'
exclude:
- os: macos-latest
target: aarch64
steps:
- uses: actions/checkout@v3
- uses: actions/setup-python@v4
with:
python-version: pypy${{ matrix.python-version }}
- name: Build wheels
uses: messense/maturin-action@v1
with:
maturin-version: "v0.13.0"
target: ${{ matrix.target }}
manylinux: auto
args: --release --out dist -i pypy${{ matrix.python-version }}
- name: Install built wheel
if: matrix.target == 'x86_64'
run: |
ls -lrth /artifacts
PYTHON=python3.10
$PYTHON -m venv venv
venv/bin/pip install -U pip
venv/bin/pip install ruff --no-index --find-links /artifacts --force-reinstall
- name: Upload wheels
uses: actions/upload-artifact@v2
with:
name: wheels
path: dist
pip install dist/${{ env.PACKAGE_NAME }}-*.whl --force-reinstall
- name: Upload wheels
uses: actions/upload-artifact@v2
with:
name: wheels
path: dist
release:
name: Release
runs-on: ubuntu-latest
needs:
- macos
- windows
- linux
- linux-cross
- musllinux
- musllinux-cross
- pypy
if: "startsWith(github.ref, 'refs/tags/')"
needs: [ macos, windows, linux, linux-cross ]
steps:
- uses: actions/download-artifact@v2
with:
name: wheels
- uses: actions/setup-python@v2
with:
python-version: "3.10"
- name: Publish to PyPI
- uses: actions/setup-python@v4
- name: Publish to PyPi
env:
TWINE_USERNAME: __token__
TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }}
run: |
pip install --upgrade wheel pip setuptools twine
pip install --upgrade twine
twine upload --skip-existing *

8
Cargo.lock generated
View File

@@ -1635,7 +1635,7 @@ dependencies = [
[[package]]
name = "ruff"
version = "0.0.19"
version = "0.0.21"
dependencies = [
"anyhow",
"bincode",
@@ -1662,7 +1662,7 @@ dependencies = [
[[package]]
name = "rustpython-ast"
version = "0.1.0"
source = "git+https://github.com/RustPython/RustPython.git?rev=7a688e0b6c2904f286acac1e4af3f1628dd38589#7a688e0b6c2904f286acac1e4af3f1628dd38589"
source = "git+https://github.com/charliermarsh/RustPython.git?rev=1613f6c6990011a4bc559e79aaf28d715f9f729b#1613f6c6990011a4bc559e79aaf28d715f9f729b"
dependencies = [
"num-bigint",
"rustpython-compiler-core",
@@ -1671,7 +1671,7 @@ dependencies = [
[[package]]
name = "rustpython-compiler-core"
version = "0.1.2"
source = "git+https://github.com/RustPython/RustPython.git?rev=7a688e0b6c2904f286acac1e4af3f1628dd38589#7a688e0b6c2904f286acac1e4af3f1628dd38589"
source = "git+https://github.com/charliermarsh/RustPython.git?rev=1613f6c6990011a4bc559e79aaf28d715f9f729b#1613f6c6990011a4bc559e79aaf28d715f9f729b"
dependencies = [
"bincode",
"bitflags",
@@ -1688,7 +1688,7 @@ dependencies = [
[[package]]
name = "rustpython-parser"
version = "0.1.2"
source = "git+https://github.com/RustPython/RustPython.git?rev=7a688e0b6c2904f286acac1e4af3f1628dd38589#7a688e0b6c2904f286acac1e4af3f1628dd38589"
source = "git+https://github.com/charliermarsh/RustPython.git?rev=1613f6c6990011a4bc559e79aaf28d715f9f729b#1613f6c6990011a4bc559e79aaf28d715f9f729b"
dependencies = [
"ahash",
"anyhow",

View File

@@ -1,6 +1,6 @@
[package]
name = "ruff"
version = "0.0.19"
version = "0.0.21"
edition = "2021"
[lib]
@@ -22,12 +22,14 @@ log = { version = "0.4.17" }
notify = { version = "4.0.17" }
rayon = { version = "1.5.3" }
regex = { version = "1.6.0" }
rustpython-parser = { features = ["lalrpop"], git = "https://github.com/RustPython/RustPython.git", rev = "7a688e0b6c2904f286acac1e4af3f1628dd38589" }
rustpython-parser = { features = ["lalrpop"], git = "https://github.com/charliermarsh/RustPython.git", rev = "1613f6c6990011a4bc559e79aaf28d715f9f729b" }
serde = { version = "1.0.143", features = ["derive"] }
serde_json = { version = "1.0.83" }
toml = { version = "0.5.9" }
walkdir = { version = "2.3.2" }
[profile.release]
lto = true
panic = "abort"
lto = "thin"
codegen-units = 1
opt-level = 3

View File

@@ -6,25 +6,25 @@
An extremely fast Python linter, written in Rust.
<p align="center">
<img alt="Bar chart with benchmark results" src="https://user-images.githubusercontent.com/1309177/187330134-ac05076c-8d16-4451-a300-986692b34abf.svg">
<img alt="Bar chart with benchmark results" src="https://user-images.githubusercontent.com/1309177/187504482-6d9df992-a81d-4e86-9f6a-d958741c8182.svg">
</p>
<p align="center">
<i>Linting the CPython codebase from scratch.</i>
</p>
Major features:
- 10-100x faster than existing linters.
- Installable via `pip`.
- Python 3.10 compatibility.
- [ESLint](https://eslint.org/docs/latest/user-guide/command-line-interface#caching)-inspired cache semantics.
- [TypeScript](https://www.typescriptlang.org/docs/handbook/configuring-watch.html)-inspired `--watch` semantics.
- `pyproject.toml` support.
- ⚡️ 10-100x faster than existing linters
- 🐍 Installable via `pip`
- 🤝 Python 3.10 compatibility
- 🛠️ `pyproject.toml` support
- 📦 [ESLint](https://eslint.org/docs/latest/user-guide/command-line-interface#caching)-inspired cache semantics
- 👀 [TypeScript](https://www.typescriptlang.org/docs/handbook/configuring-watch.html)-inspired `--watch` semantics
_ruff is a proof-of-concept and not yet intended for production use. It supports only a small subset
of the Flake8 rules, and may crash on your codebase._
Read the [launch blog post](https://notes.crmarsh.com/python-tooling-could-be-much-much-faster).
## Installation and usage
### Installation
@@ -35,11 +35,6 @@ Available as [ruff](https://pypi.org/project/ruff/) on PyPI:
pip install ruff
```
For now, wheels are available for Python 3.7, 3.8, 3.9, and 3.10 on macOS, Windows, and Linux. If a
wheel isn't available for your Python version or platform, you'll need to install the [Rust toolchain](https://www.rust-lang.org/tools/install)
prior to running `pip install ruff`. (This is an effort limitation on my part, not a technical
limitation.)
### Usage
To run ruff, try any of the following:
@@ -81,7 +76,7 @@ See `ruff --help` for more:
```shell
ruff
A Python linter written in Rust
An extremely fast Python linter.
USAGE:
ruff [OPTIONS] <FILES>...

View File

@@ -1,5 +0,0 @@
FROM python:3.10.6-buster
RUN pip install ruff
RUN touch foo.py
RUN ruff foo.py

View File

@@ -5,10 +5,14 @@ from collections import (
OrderedDict,
namedtuple,
)
import multiprocessing.pool
import multiprocessing.process
import logging.config
import logging.handlers
class X:
def a(self) -> "namedtuple":
x = os.environ["1"]
y = Counter()
return X
z = multiprocessing.pool.ThreadPool()

View File

@@ -0,0 +1,48 @@
def get_name():
return self.name
def get_name():
return (self.name,)
def get_name():
del self.name
def get_name(self):
return self.name
x = list()
def randdec(maxprec, maxexp):
return numeric_string(maxprec, maxexp)
def ternary_optarg(prec, exp_range, itr):
for _ in range(100):
a = randdec(prec, 2 * exp_range)
b = randdec(prec, 2 * exp_range)
c = randdec(prec, 2 * exp_range)
yield a, b, c, None
yield a, b, c, None, None
class Foo:
CLASS_VAR = 1
REFERENCES_CLASS_VAR = {"CLASS_VAR": CLASS_VAR}
class Class:
def __init__(self):
# TODO(charlie): This should be recognized as a defined variable.
Class # noqa: F821
try:
x = 1 / 0
except Exception as e:
# TODO(charlie): This should be recognized as a defined variable.
print(e) # noqa: F821

View File

@@ -0,0 +1,17 @@
my_dict = {}
my_var = 0
def foo():
my_var += 1
def bar():
global my_var
my_var += 1
def baz():
global my_var
global my_dict
my_dict[my_var] += 1

View File

@@ -8,6 +8,8 @@ select = [
"F541",
"F634",
"F706",
"F821",
"F831",
"F832",
"F901",
]

163
src/builtins.rs Normal file
View File

@@ -0,0 +1,163 @@
pub const BUILTINS: &[&str] = &[
"ArithmeticError",
"AssertionError",
"AttributeError",
"BaseException",
"BlockingIOError",
"BrokenPipeError",
"BufferError",
"BytesWarning",
"ChildProcessError",
"ConnectionAbortedError",
"ConnectionError",
"ConnectionRefusedError",
"ConnectionResetError",
"DeprecationWarning",
"EOFError",
"Ellipsis",
"EnvironmentError",
"Exception",
"False",
"FileExistsError",
"FileNotFoundError",
"FloatingPointError",
"FutureWarning",
"GeneratorExit",
"IOError",
"ImportError",
"ImportWarning",
"IndentationError",
"IndexError",
"InterruptedError",
"IsADirectoryError",
"KeyError",
"KeyboardInterrupt",
"LookupError",
"MemoryError",
"ModuleNotFoundError",
"NameError",
"None",
"NotADirectoryError",
"NotImplemented",
"NotImplementedError",
"OSError",
"OverflowError",
"PendingDeprecationWarning",
"PermissionError",
"ProcessLookupError",
"RecursionError",
"ReferenceError",
"ResourceWarning",
"RuntimeError",
"RuntimeWarning",
"StopAsyncIteration",
"StopIteration",
"SyntaxError",
"SyntaxWarning",
"SystemError",
"SystemExit",
"TabError",
"TimeoutError",
"True",
"TypeError",
"UnboundLocalError",
"UnicodeDecodeError",
"UnicodeEncodeError",
"UnicodeError",
"UnicodeTranslateError",
"UnicodeWarning",
"UserWarning",
"ValueError",
"Warning",
"ZeroDivisionError",
"__build_class__",
"__debug__",
"__doc__",
"__import__",
"__loader__",
"__name__",
"__package__",
"__spec__",
"abs",
"all",
"any",
"ascii",
"bin",
"bool",
"breakpoint",
"bytearray",
"bytes",
"callable",
"chr",
"classmethod",
"compile",
"complex",
"copyright",
"credits",
"delattr",
"dict",
"dir",
"divmod",
"enumerate",
"eval",
"exec",
"exit",
"filter",
"float",
"format",
"frozenset",
"getattr",
"globals",
"hasattr",
"hash",
"help",
"hex",
"id",
"input",
"int",
"isinstance",
"issubclass",
"iter",
"len",
"license",
"list",
"locals",
"map",
"max",
"memoryview",
"min",
"next",
"object",
"oct",
"open",
"ord",
"pow",
"print",
"property",
"quit",
"range",
"repr",
"reversed",
"round",
"set",
"setattr",
"slice",
"sorted",
"staticmethod",
"str",
"sum",
"super",
"tuple",
"type",
"vars",
"zip",
];
// Globally defined names which are not attributes of the builtins module, or are only present on
// some platforms.
pub const MAGIC_GLOBALS: &[&str] = &[
"__file__",
"__builtins__",
"__annotations__",
"WindowsError",
];

View File

@@ -1,5 +1,7 @@
use std::collections::{BTreeMap, BTreeSet};
use std::sync::atomic::{AtomicUsize, Ordering};
use crate::builtins::{BUILTINS, MAGIC_GLOBALS};
use rustpython_parser::ast::{
Arg, Arguments, Constant, Expr, ExprContext, ExprKind, Location, Stmt, StmtKind, Suite,
};
@@ -11,6 +13,11 @@ use crate::settings::Settings;
use crate::visitor;
use crate::visitor::Visitor;
fn id() -> usize {
static COUNTER: AtomicUsize = AtomicUsize::new(1);
COUNTER.fetch_add(1, Ordering::Relaxed)
}
enum ScopeKind {
Class,
Function,
@@ -19,26 +26,37 @@ enum ScopeKind {
}
struct Scope {
id: usize,
kind: ScopeKind,
values: BTreeMap<String, Binding>,
}
impl Scope {
fn new(kind: ScopeKind) -> Self {
Scope {
id: id(),
kind,
values: BTreeMap::new(),
}
}
}
enum BindingKind {
Argument,
Assignment,
ClassDefinition,
Definition,
ClassDefinition,
Builtin,
FutureImportation,
Importation(String),
StarImportation,
SubmoduleImportation,
SubmoduleImportation(String),
}
struct Binding {
kind: BindingKind,
name: String,
location: Location,
used: bool,
used: Option<usize>,
}
struct Checker<'a> {
@@ -68,29 +86,49 @@ impl Checker<'_> {
impl Visitor for Checker<'_> {
fn visit_stmt(&mut self, stmt: &Stmt) {
match &stmt.node {
StmtKind::Global { names } | StmtKind::Nonlocal { names } => {
// TODO(charlie): Handle doctests.
let global_scope_index = 0;
let global_scope_id = self.scopes[global_scope_index].id;
let current_scope_id = self.scopes.last().expect("No current scope found.").id;
if current_scope_id != global_scope_id {
for name in names {
for scope in self.scopes.iter_mut().skip(global_scope_index + 1) {
scope.values.insert(
name.to_string(),
Binding {
kind: BindingKind::Assignment,
used: Some(global_scope_id),
location: stmt.location,
},
);
}
}
}
}
StmtKind::FunctionDef { name, .. } => {
self.push_scope(Scope {
kind: Function,
values: BTreeMap::new(),
});
self.add_binding(Binding {
kind: BindingKind::ClassDefinition,
name: name.clone(),
used: false,
location: stmt.location,
})
self.add_binding(
name.to_string(),
Binding {
kind: BindingKind::Definition,
used: None,
location: stmt.location,
},
);
self.push_scope(Scope::new(Function));
}
StmtKind::AsyncFunctionDef { name, .. } => {
self.push_scope(Scope {
kind: Function,
values: BTreeMap::new(),
});
self.add_binding(Binding {
kind: BindingKind::ClassDefinition,
name: name.clone(),
used: false,
location: stmt.location,
})
self.add_binding(
name.to_string(),
Binding {
kind: BindingKind::Definition,
used: None,
location: stmt.location,
},
);
self.push_scope(Scope::new(Function));
}
StmtKind::Return { .. } => {
if self
@@ -111,36 +149,39 @@ impl Visitor for Checker<'_> {
}
}
}
StmtKind::ClassDef { .. } => self.push_scope(Scope {
kind: Class,
values: BTreeMap::new(),
}),
StmtKind::ClassDef { .. } => self.push_scope(Scope::new(Class)),
StmtKind::Import { names } => {
for alias in names {
if alias.node.name.contains('.') && alias.node.asname.is_none() {
self.add_binding(Binding {
kind: BindingKind::SubmoduleImportation,
name: alias.node.name.clone(),
used: false,
location: stmt.location,
})
self.add_binding(
alias.node.name.split('.').next().unwrap().to_string(),
Binding {
kind: BindingKind::SubmoduleImportation(
alias.node.name.to_string(),
),
used: None,
location: stmt.location,
},
)
} else {
self.add_binding(Binding {
kind: BindingKind::Importation(
alias
.node
.asname
.clone()
.unwrap_or_else(|| alias.node.name.clone()),
),
name: alias
self.add_binding(
alias
.node
.asname
.clone()
.unwrap_or_else(|| alias.node.name.clone()),
used: false,
location: stmt.location,
})
Binding {
kind: BindingKind::Importation(
alias
.node
.asname
.clone()
.unwrap_or_else(|| alias.node.name.clone()),
),
used: None,
location: stmt.location,
},
)
}
}
}
@@ -152,19 +193,25 @@ impl Visitor for Checker<'_> {
.clone()
.unwrap_or_else(|| alias.node.name.clone());
if let Some("future") = module.as_deref() {
self.add_binding(Binding {
kind: BindingKind::FutureImportation,
self.add_binding(
name,
used: true,
location: stmt.location,
});
Binding {
kind: BindingKind::FutureImportation,
used: Some(self.scopes.last().expect("No current scope found.").id),
location: stmt.location,
},
);
} else if alias.node.name == "*" {
self.add_binding(Binding {
kind: BindingKind::StarImportation,
self.add_binding(
name,
used: false,
location: stmt.location,
});
Binding {
kind: BindingKind::StarImportation,
used: None,
location: stmt.location,
},
);
if self
.settings
@@ -177,15 +224,15 @@ impl Visitor for Checker<'_> {
});
}
} else {
self.add_binding(Binding {
let binding = Binding {
kind: BindingKind::Importation(match module {
None => name.clone(),
Some(parent) => format!("{}.{}", parent, name),
Some(parent) => format!("{}.{}", parent, name.clone()),
}),
name,
used: false,
used: None,
location: stmt.location,
})
};
self.add_binding(name, binding)
}
}
}
@@ -248,12 +295,15 @@ impl Visitor for Checker<'_> {
};
if let StmtKind::ClassDef { name, .. } = &stmt.node {
self.add_binding(Binding {
kind: BindingKind::Definition,
name: name.clone(),
used: false,
location: stmt.location,
});
self.add_binding(
name.to_string(),
Binding {
kind: BindingKind::ClassDefinition,
used: None,
location: stmt.location,
},
);
}
}
@@ -270,16 +320,13 @@ impl Visitor for Checker<'_> {
ExprKind::Name { ctx, .. } => match ctx {
ExprContext::Load => self.handle_node_load(expr),
ExprContext::Store => self.handle_node_store(expr),
ExprContext::Del => {}
ExprContext::Del => self.handle_node_delete(expr),
},
ExprKind::GeneratorExp { .. } => self.push_scope(Scope {
kind: Generator,
values: BTreeMap::new(),
}),
ExprKind::Lambda { .. } => self.push_scope(Scope {
kind: Function,
values: BTreeMap::new(),
}),
ExprKind::GeneratorExp { .. }
| ExprKind::ListComp { .. }
| ExprKind::DictComp { .. }
| ExprKind::SetComp { .. } => self.push_scope(Scope::new(Generator)),
ExprKind::Lambda { .. } => self.push_scope(Scope::new(Function)),
ExprKind::JoinedStr { values } => {
if !self.in_f_string
&& self
@@ -307,7 +354,11 @@ impl Visitor for Checker<'_> {
visitor::walk_expr(self, expr);
match &expr.node {
ExprKind::GeneratorExp { .. } | ExprKind::Lambda { .. } => {
ExprKind::GeneratorExp { .. }
| ExprKind::ListComp { .. }
| ExprKind::DictComp { .. }
| ExprKind::SetComp { .. }
| ExprKind::Lambda { .. } => {
if let Some(scope) = self.scopes.pop() {
self.dead_scopes.push(scope);
}
@@ -340,17 +391,17 @@ impl Visitor for Checker<'_> {
}
// Search for duplicates.
let mut idents: BTreeSet<String> = BTreeSet::new();
let mut idents: BTreeSet<&str> = BTreeSet::new();
for arg in all_arguments {
let ident = &arg.node.arg;
if idents.contains(ident) {
if idents.contains(ident.as_str()) {
self.checks.push(Check {
kind: CheckKind::DuplicateArgumentName,
location: arg.location,
});
break;
}
idents.insert(ident.clone());
idents.insert(ident);
}
}
@@ -358,12 +409,14 @@ impl Visitor for Checker<'_> {
}
fn visit_arg(&mut self, arg: &Arg) {
self.add_binding(Binding {
kind: BindingKind::Argument,
name: arg.node.arg.clone(),
used: false,
location: arg.location,
});
self.add_binding(
arg.node.arg.to_string(),
Binding {
kind: BindingKind::Argument,
used: None,
location: arg.location,
},
);
visitor::walk_arg(self, arg);
}
}
@@ -378,50 +431,124 @@ impl Checker<'_> {
.push(self.scopes.pop().expect("Attempted to pop without scope."));
}
fn add_binding(&mut self, binding: Binding) {
// TODO(charlie): Don't treat annotations as assignments if there is an existing value.
fn bind_builtins(&mut self) {
for builtin in BUILTINS {
self.add_binding(
builtin.to_string(),
Binding {
kind: BindingKind::Builtin,
location: Default::default(),
used: None,
},
)
}
for builtin in MAGIC_GLOBALS {
self.add_binding(
builtin.to_string(),
Binding {
kind: BindingKind::Builtin,
location: Default::default(),
used: None,
},
)
}
}
fn add_binding(&mut self, name: String, binding: Binding) {
let scope = self.scopes.last_mut().expect("No current scope found.");
scope.values.insert(
binding.name.clone(),
match scope.values.get(&binding.name) {
None => binding,
Some(existing) => Binding {
kind: binding.kind,
name: binding.name,
location: binding.location,
used: existing.used,
},
// TODO(charlie): Don't treat annotations as assignments if there is an existing value.
let binding = match scope.values.get(&name) {
None => binding,
Some(existing) => Binding {
kind: binding.kind,
location: binding.location,
used: existing.used,
},
);
};
scope.values.insert(name, binding);
}
fn handle_node_load(&mut self, expr: &Expr) {
if let ExprKind::Name { id, .. } = &expr.node {
let scope_id = self.scopes.last_mut().expect("No current scope found.").id;
let mut first_iter = true;
let mut in_generators = false;
for scope in self.scopes.iter_mut().rev() {
if matches!(scope.kind, Class) {
if id == "__class__" {
return;
} else {
} else if !first_iter && !in_generators {
continue;
}
}
if let Some(binding) = scope.values.get_mut(id) {
binding.used = true;
binding.used = Some(scope_id);
return;
}
first_iter = false;
in_generators = matches!(scope.kind, Generator);
}
if self.settings.select.contains(&CheckCode::F821) {
self.checks.push(Check {
kind: CheckKind::UndefinedName(id.clone()),
location: expr.location,
})
}
}
}
fn handle_node_store(&mut self, expr: &Expr) {
if let ExprKind::Name { id, .. } = &expr.node {
if self.settings.select.contains(&CheckCode::F832) {
let current = self.scopes.last().expect("No current scope found.");
if matches!(current.kind, ScopeKind::Function) && !current.values.contains_key(id) {
for scope in self.scopes.iter().rev().skip(1) {
if matches!(scope.kind, ScopeKind::Function) || matches!(scope.kind, Module)
{
let used = scope
.values
.get(id)
.map(|binding| binding.used)
.unwrap_or_default();
if let Some(scope_id) = used {
if scope_id == current.id {
self.checks.push(Check {
kind: CheckKind::UndefinedLocal(id.clone()),
location: expr.location,
});
}
}
}
}
}
}
// TODO(charlie): Handle alternate binding types (like `Annotation`).
self.add_binding(Binding {
kind: BindingKind::Assignment,
name: id.to_string(),
used: false,
location: expr.location,
});
self.add_binding(
id.to_string(),
Binding {
kind: BindingKind::Assignment,
used: None,
location: expr.location,
},
);
}
}
fn handle_node_delete(&mut self, expr: &Expr) {
if let ExprKind::Name { id, .. } = &expr.node {
let current = self.scopes.last_mut().expect("No current scope found.");
if current.values.remove(id).is_none()
&& self.settings.select.contains(&CheckCode::F821)
{
self.checks.push(Check {
kind: CheckKind::UndefinedName(id.clone()),
location: expr.location,
})
}
}
}
@@ -438,12 +565,16 @@ impl Checker<'_> {
// TODO(charlie): Handle `__all__`.
for scope in &self.dead_scopes {
for (_, binding) in scope.values.iter().rev() {
if !binding.used {
if let BindingKind::Importation(name) = &binding.kind {
self.checks.push(Check {
kind: CheckKind::UnusedImport(name.clone()),
location: binding.location,
});
if binding.used.is_none() {
match &binding.kind {
BindingKind::Importation(full_name)
| BindingKind::SubmoduleImportation(full_name) => {
self.checks.push(Check {
kind: CheckKind::UnusedImport(full_name.to_string()),
location: binding.location,
});
}
_ => {}
}
}
}
@@ -454,10 +585,8 @@ impl Checker<'_> {
pub fn check_ast(python_ast: &Suite, settings: &Settings, path: &str) -> Vec<Check> {
let mut checker = Checker::new(settings);
checker.push_scope(Scope {
kind: Module,
values: BTreeMap::new(),
});
checker.push_scope(Scope::new(Module));
checker.bind_builtins();
for stmt in python_ast {
checker.visit_stmt(stmt);

View File

@@ -3,24 +3,37 @@ use rustpython_parser::ast::Location;
use crate::checks::{Check, CheckKind};
use crate::settings::Settings;
pub fn check_lines(contents: &str, settings: &Settings) -> Vec<Check> {
contents
.lines()
.enumerate()
.filter_map(|(row, line)| {
if settings.select.contains(CheckKind::LineTooLong.code())
&& line.len() > settings.line_length
{
let chunks: Vec<&str> = line.split_whitespace().collect();
if !(chunks.len() == 1 || (chunks.len() == 2 && chunks[0] == "#")) {
return Some(Check {
kind: CheckKind::LineTooLong,
location: Location::new(row + 1, settings.line_length + 1),
});
pub fn check_lines(checks: &mut Vec<Check>, contents: &str, settings: &Settings) {
let enforce_line_too_ling = settings.select.contains(CheckKind::LineTooLong.code());
let mut line_checks = vec![];
let mut ignored = vec![];
for (row, line) in contents.lines().enumerate() {
// Remove any ignored checks.
// TODO(charlie): Only validate checks for the current line.
for (index, check) in checks.iter().enumerate() {
if check.location.row() == row + 1 && check.is_inline_ignored(line) {
ignored.push(index);
}
}
// Enforce line length.
if enforce_line_too_ling && line.len() > settings.line_length {
let chunks: Vec<&str> = line.split_whitespace().collect();
if !(chunks.len() == 1 || (chunks.len() == 2 && chunks[0] == "#")) {
let check = Check {
kind: CheckKind::LineTooLong,
location: Location::new(row + 1, settings.line_length + 1),
};
if !check.is_inline_ignored(line) {
line_checks.push(check);
}
}
None
})
.collect()
}
}
ignored.sort();
for index in ignored.iter().rev() {
checks.swap_remove(*index);
}
checks.extend(line_checks);
}

View File

@@ -1,6 +1,7 @@
use std::str::FromStr;
use anyhow::Result;
use regex::Regex;
use rustpython_parser::ast::Location;
use serde::{Deserialize, Serialize};
@@ -12,7 +13,9 @@ pub enum CheckCode {
F541,
F634,
F706,
F821,
F831,
F832,
F901,
}
@@ -27,7 +30,9 @@ impl FromStr for CheckCode {
"F541" => Ok(CheckCode::F541),
"F634" => Ok(CheckCode::F634),
"F706" => Ok(CheckCode::F706),
"F821" => Ok(CheckCode::F821),
"F831" => Ok(CheckCode::F831),
"F832" => Ok(CheckCode::F832),
"F901" => Ok(CheckCode::F901),
_ => Err(anyhow::anyhow!("Unknown check code: {s}")),
}
@@ -43,7 +48,9 @@ impl CheckCode {
CheckCode::F541 => "F541",
CheckCode::F634 => "F634",
CheckCode::F706 => "F706",
CheckCode::F821 => "F821",
CheckCode::F831 => "F831",
CheckCode::F832 => "F832",
CheckCode::F901 => "F901",
}
}
@@ -57,7 +64,9 @@ impl CheckCode {
CheckCode::F541 => &LintSource::AST,
CheckCode::F634 => &LintSource::AST,
CheckCode::F706 => &LintSource::AST,
CheckCode::F821 => &LintSource::AST,
CheckCode::F831 => &LintSource::AST,
CheckCode::F832 => &LintSource::AST,
CheckCode::F901 => &LintSource::AST,
}
}
@@ -78,6 +87,8 @@ pub enum CheckKind {
LineTooLong,
RaiseNotImplemented,
ReturnOutsideFunction,
UndefinedName(String),
UndefinedLocal(String),
UnusedImport(String),
}
@@ -92,6 +103,8 @@ impl CheckKind {
CheckKind::LineTooLong => &CheckCode::E501,
CheckKind::RaiseNotImplemented => &CheckCode::F901,
CheckKind::ReturnOutsideFunction => &CheckCode::F706,
CheckKind::UndefinedName(_) => &CheckCode::F821,
CheckKind::UndefinedLocal(_) => &CheckCode::F832,
CheckKind::UnusedImport(_) => &CheckCode::F401,
}
}
@@ -116,6 +129,12 @@ impl CheckKind {
CheckKind::ReturnOutsideFunction => {
"a `return` statement outside of a function/method".to_string()
}
CheckKind::UndefinedName(name) => {
format!("Undefined name `{name}`")
}
CheckKind::UndefinedLocal(name) => {
format!("Local variable `{name}` referenced before assignment")
}
CheckKind::UnusedImport(name) => format!("`{name}` imported but unused"),
}
}
@@ -126,3 +145,28 @@ pub struct Check {
pub kind: CheckKind,
pub location: Location,
}
impl Check {
pub fn is_inline_ignored(&self, line: &str) -> bool {
let re = Regex::new(r"(?i)# noqa(?::\s?(?P<codes>([A-Z]+[0-9]+(?:[,\s]+)?)+))?").unwrap();
match re.captures(line) {
Some(caps) => match caps.name("codes") {
Some(codes) => {
let re = Regex::new(r"[,\s]").unwrap();
for code in re
.split(codes.as_str())
.map(|code| code.trim())
.filter(|code| !code.is_empty())
{
if code == self.kind.code().as_str() {
return true;
}
}
false
}
None => true,
},
None => false,
}
}
}

View File

@@ -1,5 +1,5 @@
use std::fs::File;
use std::io::{BufRead, BufReader, Read};
use std::io::{BufReader, Read};
use std::path::{Path, PathBuf};
use anyhow::Result;
@@ -22,16 +22,6 @@ pub fn iter_python_files(path: &PathBuf) -> impl Iterator<Item = DirEntry> {
.filter(|entry| entry.path().to_string_lossy().ends_with(".py"))
}
pub fn read_line(path: &Path, row: &usize) -> Result<String> {
let file = File::open(path)?;
let buf_reader = BufReader::new(file);
buf_reader
.lines()
.nth(*row - 1)
.unwrap()
.map_err(|e| e.into())
}
pub fn read_file(path: &Path) -> Result<String> {
let file = File::open(path)?;
let mut buf_reader = BufReader::new(file);

View File

@@ -1,3 +1,4 @@
mod builtins;
mod cache;
pub mod check_ast;
mod check_lines;

View File

@@ -36,13 +36,7 @@ pub fn check_path(path: &Path, settings: &Settings, mode: &cache::Mode) -> Resul
}
// Run the lines-based checks.
if settings
.select
.iter()
.any(|check_code| matches!(check_code.lint_source(), LintSource::Lines))
{
checks.extend(check_lines(&contents, settings));
}
check_lines(&mut checks, &contents, settings);
// Convert to messages.
let messages: Vec<Message> = checks
@@ -52,7 +46,6 @@ pub fn check_path(path: &Path, settings: &Settings, mode: &cache::Mode) -> Resul
location: check.location,
filename: path.to_string_lossy().to_string(),
})
.filter(|message| !message.is_inline_ignored())
.collect();
cache::set(path, settings, &messages, mode);
@@ -108,6 +101,11 @@ mod tests {
&cache::Mode::None,
)?;
let expected = vec![
Message {
kind: CheckKind::UnusedImport("logging.handlers".to_string()),
location: Location::new(11, 1),
filename: "./resources/test/src/F401.py".to_string(),
},
Message {
kind: CheckKind::UnusedImport("functools".to_string()),
location: Location::new(2, 1),
@@ -255,6 +253,47 @@ mod tests {
Ok(())
}
#[test]
fn f821() -> Result<()> {
let actual = check_path(
&Path::new("./resources/test/src/F821.py"),
&settings::Settings {
line_length: 88,
exclude: vec![],
select: BTreeSet::from([CheckCode::F821]),
},
&cache::Mode::None,
)?;
let expected = vec![
Message {
kind: CheckKind::UndefinedName("self".to_string()),
location: Location::new(2, 12),
filename: "./resources/test/src/F821.py".to_string(),
},
Message {
kind: CheckKind::UndefinedName("self".to_string()),
location: Location::new(6, 13),
filename: "./resources/test/src/F821.py".to_string(),
},
Message {
kind: CheckKind::UndefinedName("self".to_string()),
location: Location::new(10, 9),
filename: "./resources/test/src/F821.py".to_string(),
},
Message {
kind: CheckKind::UndefinedName("numeric_string".to_string()),
location: Location::new(21, 12),
filename: "./resources/test/src/F821.py".to_string(),
},
];
assert_eq!(actual.len(), expected.len());
for i in 0..actual.len() {
assert_eq!(actual[i], expected[i]);
}
Ok(())
}
#[test]
fn f831() -> Result<()> {
let actual = check_path(
@@ -291,6 +330,30 @@ mod tests {
Ok(())
}
#[test]
fn f832() -> Result<()> {
let actual = check_path(
&Path::new("./resources/test/src/F832.py"),
&settings::Settings {
line_length: 88,
exclude: vec![],
select: BTreeSet::from([CheckCode::F832]),
},
&cache::Mode::None,
)?;
let expected = vec![Message {
kind: CheckKind::UndefinedLocal("my_var".to_string()),
location: Location::new(6, 5),
filename: "./resources/test/src/F832.py".to_string(),
}];
assert_eq!(actual.len(), expected.len());
for i in 0..actual.len() {
assert_eq!(actual[i], expected[i]);
}
Ok(())
}
#[test]
fn f901() -> Result<()> {
let actual = check_path(

View File

@@ -56,7 +56,7 @@ fn run_once(files: &[PathBuf], settings: &Settings, cache: bool) -> Result<Vec<M
debug!("Identified files to lint in: {:?}", duration);
let start = Instant::now();
let messages: Vec<Message> = files
let mut messages: Vec<Message> = files
.par_iter()
.filter(|entry| {
!settings
@@ -72,6 +72,7 @@ fn run_once(files: &[PathBuf], settings: &Settings, cache: bool) -> Result<Vec<M
})
.flatten()
.collect();
messages.sort_unstable();
let duration = start.elapsed();
debug!("Checked files in: {:?}", duration);

View File

@@ -1,13 +1,11 @@
use std::cmp::Ordering;
use std::fmt;
use std::path::Path;
use colored::Colorize;
use regex::Regex;
use rustpython_parser::ast::Location;
use serde::{Deserialize, Serialize};
use crate::checks::CheckKind;
use crate::fs;
#[derive(Serialize, Deserialize)]
#[serde(remote = "Location")]
@@ -32,35 +30,19 @@ pub struct Message {
pub filename: String,
}
impl Message {
pub fn is_inline_ignored(&self) -> bool {
match fs::read_line(Path::new(&self.filename), &self.location.row()) {
Ok(line) => {
// https://github.com/PyCQA/flake8/blob/799c71eeb61cf26c7c176aed43e22523e2a6d991/src/flake8/defaults.py#L26
let re = Regex::new(r"(?i)# noqa(?::\s?(?P<codes>([A-Z]+[0-9]+(?:[,\s]+)?)+))?")
.unwrap();
match re.captures(&line) {
Some(caps) => match caps.name("codes") {
Some(codes) => {
let re = Regex::new(r"[,\s]").unwrap();
for code in re
.split(codes.as_str())
.map(|code| code.trim())
.filter(|code| !code.is_empty())
{
if code == self.kind.code().as_str() {
return true;
}
}
false
}
None => true,
},
None => false,
}
}
Err(_) => false,
}
impl Ord for Message {
fn cmp(&self, other: &Self) -> Ordering {
(&self.filename, self.location.row(), self.location.column()).cmp(&(
&other.filename,
other.location.row(),
self.location.column(),
))
}
}
impl PartialOrd for Message {
fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
Some(self.cmp(other))
}
}

View File

@@ -233,7 +233,9 @@ other-attribute = 1
CheckCode::F541,
CheckCode::F634,
CheckCode::F706,
CheckCode::F821,
CheckCode::F831,
CheckCode::F832,
CheckCode::F901,
])),
}

View File

@@ -49,6 +49,7 @@ impl Settings {
CheckCode::F634,
CheckCode::F706,
CheckCode::F831,
CheckCode::F832,
CheckCode::F901,
])
}),

View File

@@ -340,33 +340,33 @@ pub fn walk_expr<V: Visitor + ?Sized>(visitor: &mut V, expr: &Expr) {
}
}
ExprKind::ListComp { elt, generators } => {
visitor.visit_expr(elt);
for comprehension in generators {
visitor.visit_comprehension(comprehension)
}
visitor.visit_expr(elt);
}
ExprKind::SetComp { elt, generators } => {
visitor.visit_expr(elt);
for comprehension in generators {
visitor.visit_comprehension(comprehension)
}
visitor.visit_expr(elt);
}
ExprKind::DictComp {
key,
value,
generators,
} => {
for comprehension in generators {
visitor.visit_comprehension(comprehension)
}
visitor.visit_expr(key);
visitor.visit_expr(value);
for comprehension in generators {
visitor.visit_comprehension(comprehension)
}
}
ExprKind::GeneratorExp { elt, generators } => {
visitor.visit_expr(elt);
for comprehension in generators {
visitor.visit_comprehension(comprehension)
}
visitor.visit_expr(elt);
}
ExprKind::Await { value } => visitor.visit_expr(value),
ExprKind::Yield { value } => {