Compare commits

..

6 Commits

Author SHA1 Message Date
Charlie Marsh
17c4690b5e Bump version to v0.6.7 (#13439) 2024-09-21 13:16:36 -04:00
Charlie Marsh
f06d44e6e5 Use forget for module resolver database (#13438)
## Summary

A tiny bit faster and the `red-knot` CLI does the same thing.
2024-09-21 17:00:02 +00:00
Micha Reiser
653c09001a Use an empty vendored file system in Ruff (#13436)
## Summary

This PR changes removes the typeshed stubs from the vendored file system
shipped with ruff
and instead ships an empty "typeshed".

Making the typeshed files optional required extracting the typshed files
into a new `ruff_vendored` crate. I do like this even if all our builds
always include typeshed because it means `red_knot_python_semantic`
contains less code that needs compiling.

This also allows us to use deflate because the compression algorithm
doesn't matter for an archive containing a single, empty file.

## Test Plan

`cargo test`

I verified with ` cargo tree -f "{p} {f}" -p <package> ` that:

* red_knot_wasm: enables `deflate` compression
* red_knot: enables `zstd` compression
* `ruff`: uses stored


I'm not quiet sure how to build the binary that maturin builds but
comparing the release artifact size with `strip = true` shows a `1.5MB`
size reduction

---------

Co-authored-by: Charlie Marsh <charlie.r.marsh@gmail.com>
2024-09-21 16:31:42 +00:00
Micha Reiser
8921fbb54c vendored_typeshed_versions should use db.vendored (#13434) 2024-09-21 16:35:06 +02:00
Charlie Marsh
3018303c87 Avoid parsing with Salsa (#13437)
## Summary

For reasons I haven't investigated, this speeds up the resolver about 2x
(from 6.404s to 3.612s on an extremely large codebase).

## Test Plan

\cc @BurntSushi 

```
[andrew@duff rippling]$ time ruff analyze graph --preview > /dev/null

real    3.274
user    16.039
sys     7.609
maxmem  11631 MB
faults  0
[andrew@duff rippling]$ time ruff-patch analyze graph --preview > /dev/null

real    1.841
user    14.625
sys     3.639
maxmem  7173 MB
faults  0
[andrew@duff rippling]$ time ruff-patch2 analyze graph --preview > /dev/null

real    2.087
user    15.333
sys     4.869
maxmem  8642 MB
faults  0
```

Where that's `main`, then (`ruff-patch`) using the version with no
`File`, no `SemanticModel`, then (`ruff-patch2`) using `File`.
2024-09-21 13:52:16 +00:00
haarisr
6c303b2445 red-knot: Add not unary operator for boolean literals (#13422)
## Summary

Contributes to #12701

## Test Plan

Added test for boolean literals

Signed-off-by: haaris <haarisrahman@gmail.com>
2024-09-20 15:24:38 -07:00
615 changed files with 361 additions and 318 deletions

View File

@@ -37,13 +37,13 @@ jobs:
- name: Sync typeshed
id: sync
run: |
rm -rf ruff/crates/red_knot_python_semantic/vendor/typeshed
mkdir ruff/crates/red_knot_python_semantic/vendor/typeshed
cp typeshed/README.md ruff/crates/red_knot_python_semantic/vendor/typeshed
cp typeshed/LICENSE ruff/crates/red_knot_python_semantic/vendor/typeshed
cp -r typeshed/stdlib ruff/crates/red_knot_python_semantic/vendor/typeshed/stdlib
rm -rf ruff/crates/red_knot_python_semantic/vendor/typeshed/stdlib/@tests
git -C typeshed rev-parse HEAD > ruff/crates/red_knot_python_semantic/vendor/typeshed/source_commit.txt
rm -rf ruff/crates/ruff_vendored/vendor/typeshed
mkdir ruff/crates/ruff_vendored/vendor/typeshed
cp typeshed/README.md ruff/crates/ruff_vendored/vendor/typeshed
cp typeshed/LICENSE ruff/crates/ruff_vendored/vendor/typeshed
cp -r typeshed/stdlib ruff/crates/ruff_vendored/vendor/typeshed/stdlib
rm -rf ruff/crates/ruff_vendored/vendor/typeshed/stdlib/@tests
git -C typeshed rev-parse HEAD > ruff/crates/ruff_vendored/vendor/typeshed/source_commit.txt
- name: Commit the changes
id: commit
if: ${{ steps.sync.outcome == 'success' }}

View File

@@ -2,7 +2,7 @@ fail_fast: true
exclude: |
(?x)^(
crates/red_knot_python_semantic/vendor/.*|
crates/ruff_vendored/vendor/.*|
crates/red_knot_workspace/resources/.*|
crates/ruff_linter/resources/.*|
crates/ruff_linter/src/rules/.*/snapshots/.*|

View File

@@ -1,5 +1,26 @@
# Changelog
## 0.6.7
### Preview features
- Add Python version support to ruff analyze CLI ([#13426](https://github.com/astral-sh/ruff/pull/13426))
- Add `exclude` support to `ruff analyze` ([#13425](https://github.com/astral-sh/ruff/pull/13425))
- Fix parentheses around return type annotations ([#13381](https://github.com/astral-sh/ruff/pull/13381))
### Rule changes
- \[`pycodestyle`\] Fix: Don't autofix if the first line ends in a question mark? (D400) ([#13399](https://github.com/astral-sh/ruff/pull/13399))
### Bug fixes
- Respect `lint.exclude` in ruff check `--add-noqa` ([#13427](https://github.com/astral-sh/ruff/pull/13427))
### Performance
- Avoid tracking module resolver files in Salsa ([#13437](https://github.com/astral-sh/ruff/pull/13437))
- Use `forget` for module resolver database ([#13438](https://github.com/astral-sh/ruff/pull/13438))
## 0.6.6
### Preview features

26
Cargo.lock generated
View File

@@ -2083,9 +2083,7 @@ dependencies = [
"countme",
"hashbrown",
"insta",
"once_cell",
"ordermap",
"path-slash",
"ruff_db",
"ruff_index",
"ruff_python_ast",
@@ -2094,6 +2092,7 @@ dependencies = [
"ruff_python_stdlib",
"ruff_source_file",
"ruff_text_size",
"ruff_vendored",
"rustc-hash 2.0.0",
"salsa",
"smallvec",
@@ -2102,8 +2101,6 @@ dependencies = [
"test-case",
"thiserror",
"tracing",
"walkdir",
"zip",
]
[[package]]
@@ -2159,6 +2156,7 @@ dependencies = [
"ruff_db",
"ruff_python_ast",
"ruff_text_size",
"ruff_vendored",
"rustc-hash 2.0.0",
"salsa",
"tempfile",
@@ -2255,7 +2253,7 @@ dependencies = [
[[package]]
name = "ruff"
version = "0.6.6"
version = "0.6.7"
dependencies = [
"anyhow",
"argfile",
@@ -2450,15 +2448,18 @@ version = "0.1.0"
dependencies = [
"anyhow",
"clap",
"once_cell",
"red_knot_python_semantic",
"ruff_cache",
"ruff_db",
"ruff_linter",
"ruff_macros",
"ruff_python_ast",
"ruff_python_parser",
"salsa",
"schemars",
"serde",
"zip",
]
[[package]]
@@ -2471,7 +2472,7 @@ dependencies = [
[[package]]
name = "ruff_linter"
version = "0.6.6"
version = "0.6.7"
dependencies = [
"aho-corasick",
"annotate-snippets 0.9.2",
@@ -2789,9 +2790,20 @@ dependencies = [
"static_assertions",
]
[[package]]
name = "ruff_vendored"
version = "0.0.0"
dependencies = [
"once_cell",
"path-slash",
"ruff_db",
"walkdir",
"zip",
]
[[package]]
name = "ruff_wasm"
version = "0.6.6"
version = "0.6.7"
dependencies = [
"console_error_panic_hook",
"console_log",

View File

@@ -14,7 +14,7 @@ license = "MIT"
[workspace.dependencies]
ruff = { path = "crates/ruff" }
ruff_cache = { path = "crates/ruff_cache" }
ruff_db = { path = "crates/ruff_db" }
ruff_db = { path = "crates/ruff_db", default-features = false }
ruff_diagnostics = { path = "crates/ruff_diagnostics" }
ruff_formatter = { path = "crates/ruff_formatter" }
ruff_graph = { path = "crates/ruff_graph" }
@@ -34,11 +34,12 @@ ruff_python_trivia = { path = "crates/ruff_python_trivia" }
ruff_server = { path = "crates/ruff_server" }
ruff_source_file = { path = "crates/ruff_source_file" }
ruff_text_size = { path = "crates/ruff_text_size" }
ruff_vendored = { path = "crates/ruff_vendored" }
ruff_workspace = { path = "crates/ruff_workspace" }
red_knot_python_semantic = { path = "crates/red_knot_python_semantic" }
red_knot_server = { path = "crates/red_knot_server" }
red_knot_workspace = { path = "crates/red_knot_workspace" }
red_knot_workspace = { path = "crates/red_knot_workspace", default-features = false }
aho-corasick = { version = "1.1.3" }
annotate-snippets = { version = "0.9.2", features = ["color"] }

View File

@@ -136,8 +136,8 @@ curl -LsSf https://astral.sh/ruff/install.sh | sh
powershell -c "irm https://astral.sh/ruff/install.ps1 | iex"
# For a specific version.
curl -LsSf https://astral.sh/ruff/0.6.6/install.sh | sh
powershell -c "irm https://astral.sh/ruff/0.6.6/install.ps1 | iex"
curl -LsSf https://astral.sh/ruff/0.6.7/install.sh | sh
powershell -c "irm https://astral.sh/ruff/0.6.7/install.ps1 | iex"
```
You can also install Ruff via [Homebrew](https://formulae.brew.sh/formula/ruff), [Conda](https://anaconda.org/conda-forge/ruff),
@@ -170,7 +170,7 @@ Ruff can also be used as a [pre-commit](https://pre-commit.com/) hook via [`ruff
```yaml
- repo: https://github.com/astral-sh/ruff-pre-commit
# Ruff version.
rev: v0.6.6
rev: v0.6.7
hooks:
# Run the linter.
- id: ruff

View File

@@ -13,9 +13,8 @@ license.workspace = true
[dependencies]
red_knot_python_semantic = { workspace = true }
red_knot_workspace = { workspace = true }
red_knot_workspace = { workspace = true, features = ["zstd"] }
red_knot_server = { workspace = true }
ruff_db = { workspace = true, features = ["os", "cache"] }
anyhow = { workspace = true }

View File

@@ -24,7 +24,6 @@ bitflags = { workspace = true }
camino = { workspace = true }
compact_str = { workspace = true }
countme = { workspace = true }
once_cell = { workspace = true }
ordermap = { workspace = true }
salsa = { workspace = true }
thiserror = { workspace = true }
@@ -35,25 +34,14 @@ smallvec = { workspace = true }
static_assertions = { workspace = true }
test-case = { workspace = true }
[build-dependencies]
path-slash = { workspace = true }
walkdir = { workspace = true }
[target.'cfg(not(target_arch = "powerpc64"))'.build-dependencies]
zip = { workspace = true, features = ["deflate", "zstd"] }
[target.'cfg(target_arch = "powerpc64")'.build-dependencies]
zip = { workspace = true, features = ["deflate"] }
[dev-dependencies]
ruff_db = { workspace = true, features = ["os", "testing"] }
ruff_python_parser = { workspace = true }
ruff_vendored = { workspace = true }
anyhow = { workspace = true }
insta = { workspace = true }
tempfile = { workspace = true }
walkdir = { workspace = true }
zip = { workspace = true }
[lints]
workspace = true

View File

@@ -11,7 +11,6 @@ pub trait Db: SourceDb + Upcast<dyn SourceDb> {
pub(crate) mod tests {
use std::sync::Arc;
use crate::module_resolver::vendored_typeshed_stubs;
use ruff_db::files::{File, Files};
use ruff_db::system::{DbWithTestSystem, System, TestSystem};
use ruff_db::vendored::VendoredFileSystem;
@@ -33,7 +32,7 @@ pub(crate) mod tests {
Self {
storage: salsa::Storage::default(),
system: TestSystem::default(),
vendored: vendored_typeshed_stubs().clone(),
vendored: ruff_vendored::file_system().clone(),
events: std::sync::Arc::default(),
files: Files::default(),
}

View File

@@ -4,9 +4,7 @@ use rustc_hash::FxHasher;
pub use db::Db;
pub use module_name::ModuleName;
pub use module_resolver::{
resolve_module, system_module_search_paths, vendored_typeshed_stubs, Module,
};
pub use module_resolver::{resolve_module, system_module_search_paths, Module};
pub use program::{Program, ProgramSettings, SearchPathSettings, SitePackages};
pub use python_version::PythonVersion;
pub use semantic_model::{HasTy, SemanticModel};

View File

@@ -4,7 +4,6 @@ pub use module::Module;
pub use resolver::resolve_module;
pub(crate) use resolver::{file_to_module, SearchPaths};
use ruff_db::system::SystemPath;
pub use typeshed::vendored_typeshed_stubs;
use crate::module_resolver::resolver::search_paths;
use crate::Db;

View File

@@ -1,20 +1,21 @@
use rustc_hash::{FxBuildHasher, FxHashSet};
use std::borrow::Cow;
use std::iter::FusedIterator;
use std::ops::Deref;
use rustc_hash::{FxBuildHasher, FxHashSet};
use ruff_db::files::{File, FilePath, FileRootKind};
use ruff_db::system::{DirectoryEntry, System, SystemPath, SystemPathBuf};
use ruff_db::vendored::{VendoredFileSystem, VendoredPath};
use super::module::{Module, ModuleKind};
use super::path::{ModulePath, SearchPath, SearchPathValidationError};
use crate::db::Db;
use crate::module_name::ModuleName;
use crate::module_resolver::typeshed::{vendored_typeshed_versions, TypeshedVersions};
use crate::site_packages::VirtualEnvironment;
use crate::{Program, PythonVersion, SearchPathSettings, SitePackages};
use super::module::{Module, ModuleKind};
use super::path::{ModulePath, SearchPath, SearchPathValidationError};
/// Resolves a module name to a module.
pub fn resolve_module(db: &dyn Db, module_name: ModuleName) -> Option<Module> {
let interned_name = ModuleNameIngredient::new(db, module_name);
@@ -136,7 +137,7 @@ pub(crate) struct SearchPaths {
/// for the first `site-packages` path
site_packages: Vec<SearchPath>,
typeshed_versions: ResolvedTypeshedVersions,
typeshed_versions: TypeshedVersions,
}
impl SearchPaths {
@@ -202,11 +203,11 @@ impl SearchPaths {
let search_path = SearchPath::custom_stdlib(db, &custom_typeshed)?;
(ResolvedTypeshedVersions::Custom(parsed), search_path)
(parsed, search_path)
} else {
tracing::debug!("Using vendored stdlib");
(
ResolvedTypeshedVersions::Vendored(vendored_typeshed_versions()),
vendored_typeshed_versions(db),
SearchPath::vendored_stdlib(),
)
};
@@ -279,23 +280,6 @@ impl SearchPaths {
}
}
#[derive(Debug, PartialEq, Eq)]
enum ResolvedTypeshedVersions {
Vendored(&'static TypeshedVersions),
Custom(TypeshedVersions),
}
impl Deref for ResolvedTypeshedVersions {
type Target = TypeshedVersions;
fn deref(&self) -> &Self::Target {
match self {
ResolvedTypeshedVersions::Vendored(versions) => versions,
ResolvedTypeshedVersions::Custom(versions) => versions,
}
}
}
/// Collect all dynamic search paths. For each `site-packages` path:
/// - Collect that `site-packages` path
/// - Collect any search paths listed in `.pth` files in that `site-packages` directory

View File

@@ -4,25 +4,19 @@ use std::num::{NonZeroU16, NonZeroUsize};
use std::ops::{RangeFrom, RangeInclusive};
use std::str::FromStr;
use once_cell::sync::Lazy;
use rustc_hash::FxHashMap;
use super::vendored::vendored_typeshed_stubs;
use crate::db::Db;
use crate::module_name::ModuleName;
use crate::{Program, PythonVersion};
static VENDORED_VERSIONS: Lazy<TypeshedVersions> = Lazy::new(|| {
pub(in crate::module_resolver) fn vendored_typeshed_versions(db: &dyn Db) -> TypeshedVersions {
TypeshedVersions::from_str(
&vendored_typeshed_stubs()
&db.vendored()
.read_to_string("stdlib/VERSIONS")
.unwrap(),
.expect("The vendored typeshed stubs should contain a VERSIONS file"),
)
.unwrap()
});
pub(crate) fn vendored_typeshed_versions() -> &'static TypeshedVersions {
&VENDORED_VERSIONS
.expect("The VERSIONS file in the vendored typeshed stubs should be well-formed")
}
pub(crate) fn typeshed_versions(db: &dyn Db) -> &TypeshedVersions {
@@ -332,6 +326,8 @@ mod tests {
use insta::assert_snapshot;
use crate::db::tests::TestDb;
use super::*;
const TYPESHED_STDLIB_DIR: &str = "stdlib";
@@ -353,12 +349,9 @@ mod tests {
#[test]
fn can_parse_vendored_versions_file() {
let versions_data = include_str!(concat!(
env!("CARGO_MANIFEST_DIR"),
"/vendor/typeshed/stdlib/VERSIONS"
));
let db = TestDb::new();
let versions = TypeshedVersions::from_str(versions_data).unwrap();
let versions = vendored_typeshed_versions(&db);
assert!(versions.len() > 100);
assert!(versions.len() < 1000);
@@ -395,9 +388,10 @@ mod tests {
#[test]
fn typeshed_versions_consistent_with_vendored_stubs() {
const VERSIONS_DATA: &str = include_str!("../../../vendor/typeshed/stdlib/VERSIONS");
let vendored_typeshed_dir = Path::new("vendor/typeshed").canonicalize().unwrap();
let vendored_typeshed_versions = TypeshedVersions::from_str(VERSIONS_DATA).unwrap();
let db = TestDb::new();
let vendored_typeshed_versions = vendored_typeshed_versions(&db);
let vendored_typeshed_dir =
Path::new(env!("CARGO_MANIFEST_DIR")).join("../ruff_vendored/vendor/typeshed");
let mut empty_iterator = true;

View File

@@ -1,8 +0,0 @@
pub use self::vendored::vendored_typeshed_stubs;
pub(super) use self::versions::{
typeshed_versions, vendored_typeshed_versions, TypeshedVersions, TypeshedVersionsParseError,
TypeshedVersionsQueryResult,
};
mod vendored;
mod versions;

View File

@@ -701,12 +701,12 @@ enum CallOutcome<'db> {
impl<'db> CallOutcome<'db> {
/// Create a new `CallOutcome::Callable` with given return type.
fn callable(return_ty: Type<'db>) -> CallOutcome {
fn callable(return_ty: Type<'db>) -> CallOutcome<'db> {
CallOutcome::Callable { return_ty }
}
/// Create a new `CallOutcome::NotCallable` with given not-callable type.
fn not_callable(not_callable_ty: Type<'db>) -> CallOutcome {
fn not_callable(not_callable_ty: Type<'db>) -> CallOutcome<'db> {
CallOutcome::NotCallable { not_callable_ty }
}
@@ -719,7 +719,10 @@ impl<'db> CallOutcome<'db> {
}
/// Create a new `CallOutcome::Union` with given wrapped outcomes.
fn union(called_ty: Type<'db>, outcomes: impl Into<Box<[CallOutcome<'db>]>>) -> CallOutcome {
fn union(
called_ty: Type<'db>,
outcomes: impl Into<Box<[CallOutcome<'db>]>>,
) -> CallOutcome<'db> {
CallOutcome::Union {
called_ty,
outcomes: outcomes.into(),

View File

@@ -2211,6 +2211,7 @@ impl<'db> TypeInferenceBuilder<'db> {
match (op, self.infer_expression(operand)) {
(UnaryOp::USub, Type::IntLiteral(value)) => Type::IntLiteral(-value),
(UnaryOp::Not, Type::BooleanLiteral(value)) => Type::BooleanLiteral(!value),
_ => Type::Unknown, // TODO other unary op types
}
}
@@ -3142,6 +3143,28 @@ mod tests {
Ok(())
}
#[test]
fn not_boolean_literal() -> anyhow::Result<()> {
let mut db = setup_db();
db.write_file(
"src/a.py",
r#"
w = True
x = False
y = not w
z = not x
"#,
)?;
assert_public_ty(&db, "src/a.py", "w", "Literal[True]");
assert_public_ty(&db, "src/a.py", "x", "Literal[False]");
assert_public_ty(&db, "src/a.py", "y", "Literal[False]");
assert_public_ty(&db, "src/a.py", "z", "Literal[True]");
Ok(())
}
#[test]
fn string_type() -> anyhow::Result<()> {
let mut db = setup_db();

View File

@@ -20,9 +20,9 @@ default = ["console_error_panic_hook"]
[dependencies]
red_knot_python_semantic = { workspace = true }
red_knot_workspace = { workspace = true }
red_knot_workspace = { workspace = true, default-features = false, features = ["deflate"] }
ruff_db = { workspace = true }
ruff_db = { workspace = true, features = [] }
ruff_notebook = { workspace = true }
console_error_panic_hook = { workspace = true, optional = true }

View File

@@ -18,6 +18,7 @@ ruff_cache = { workspace = true }
ruff_db = { workspace = true, features = ["os", "cache"] }
ruff_python_ast = { workspace = true }
ruff_text_size = { workspace = true }
ruff_vendored = { workspace = true }
anyhow = { workspace = true }
crossbeam = { workspace = true }
@@ -31,5 +32,10 @@ tracing = { workspace = true }
ruff_db = { workspace = true, features = ["testing"] }
tempfile = { workspace = true }
[features]
default = ["zstd"]
zstd = ["ruff_vendored/zstd"]
deflate = ["ruff_vendored/deflate"]
[lints]
workspace = true

View File

@@ -4,7 +4,7 @@ use std::sync::Arc;
use salsa::plumbing::ZalsaDatabase;
use salsa::{Cancelled, Event};
use red_knot_python_semantic::{vendored_typeshed_stubs, Db as SemanticDb, Program};
use red_knot_python_semantic::{Db as SemanticDb, Program};
use ruff_db::files::{File, Files};
use ruff_db::system::System;
use ruff_db::vendored::VendoredFileSystem;
@@ -124,7 +124,7 @@ impl SemanticDb for RootDatabase {
#[salsa::db]
impl SourceDb for RootDatabase {
fn vendored(&self) -> &VendoredFileSystem {
vendored_typeshed_stubs()
ruff_vendored::file_system()
}
fn system(&self) -> &dyn System {
@@ -161,7 +161,7 @@ pub(crate) mod tests {
use salsa::Event;
use red_knot_python_semantic::{vendored_typeshed_stubs, Db as SemanticDb};
use red_knot_python_semantic::Db as SemanticDb;
use ruff_db::files::Files;
use ruff_db::system::{DbWithTestSystem, System, TestSystem};
use ruff_db::vendored::VendoredFileSystem;
@@ -183,7 +183,7 @@ pub(crate) mod tests {
Self {
storage: salsa::Storage::default(),
system: TestSystem::default(),
vendored: vendored_typeshed_stubs().clone(),
vendored: ruff_vendored::file_system().clone(),
files: Files::default(),
events: Arc::default(),
}

View File

@@ -1,6 +1,6 @@
[package]
name = "ruff"
version = "0.6.6"
version = "0.6.7"
publish = true
authors = { workspace = true }
edition = { workspace = true }
@@ -14,7 +14,7 @@ default-run = "ruff"
[dependencies]
ruff_cache = { workspace = true }
ruff_db = { workspace = true }
ruff_db = { workspace = true, default-features = false, features = ["os"] }
ruff_diagnostics = { workspace = true }
ruff_graph = { workspace = true, features = ["serde", "clap"] }
ruff_linter = { workspace = true, features = ["clap"] }

View File

@@ -67,102 +67,105 @@ pub(crate) fn analyze_graph(
.into(),
)?;
// Create a cache for resolved globs.
let glob_resolver = Arc::new(Mutex::new(GlobResolver::default()));
let imports = {
// Create a cache for resolved globs.
let glob_resolver = Arc::new(Mutex::new(GlobResolver::default()));
// Collect and resolve the imports for each file.
let result = Arc::new(Mutex::new(Vec::new()));
let inner_result = Arc::clone(&result);
// Collect and resolve the imports for each file.
let result = Arc::new(Mutex::new(Vec::new()));
let inner_result = Arc::clone(&result);
let db = db.snapshot();
rayon::scope(move |scope| {
for resolved_file in paths {
let Ok(resolved_file) = resolved_file else {
continue;
};
rayon::scope(move |scope| {
for resolved_file in paths {
let Ok(resolved_file) = resolved_file else {
continue;
};
let path = resolved_file.path();
let package = path
.parent()
.and_then(|parent| package_roots.get(parent))
.and_then(Clone::clone);
let path = resolved_file.path();
let package = path
.parent()
.and_then(|parent| package_roots.get(parent))
.and_then(Clone::clone);
// Resolve the per-file settings.
let settings = resolver.resolve(path);
let string_imports = settings.analyze.detect_string_imports;
let include_dependencies = settings.analyze.include_dependencies.get(path).cloned();
// Resolve the per-file settings.
let settings = resolver.resolve(path);
let string_imports = settings.analyze.detect_string_imports;
let include_dependencies = settings.analyze.include_dependencies.get(path).cloned();
// Skip excluded files.
if (settings.file_resolver.force_exclude || !resolved_file.is_root())
&& match_exclusion(
resolved_file.path(),
resolved_file.file_name(),
&settings.analyze.exclude,
)
{
continue;
}
// Ignore non-Python files.
let source_type = match settings.analyze.extension.get(path) {
None => match SourceType::from(&path) {
SourceType::Python(source_type) => source_type,
SourceType::Toml(_) => {
debug!("Ignoring TOML file: {}", path.display());
continue;
}
},
Some(language) => PySourceType::from(language),
};
if matches!(source_type, PySourceType::Ipynb) {
debug!("Ignoring Jupyter notebook: {}", path.display());
continue;
}
// Convert to system paths.
let Ok(package) = package.map(SystemPathBuf::from_path_buf).transpose() else {
warn!("Failed to convert package to system path");
continue;
};
let Ok(path) = SystemPathBuf::from_path_buf(resolved_file.into_path()) else {
warn!("Failed to convert path to system path");
continue;
};
let db = db.snapshot();
let glob_resolver = glob_resolver.clone();
let root = root.clone();
let result = inner_result.clone();
scope.spawn(move |_| {
// Identify any imports via static analysis.
let mut imports =
ModuleImports::detect(&db, &path, package.as_deref(), string_imports)
.unwrap_or_else(|err| {
warn!("Failed to generate import map for {path}: {err}");
ModuleImports::default()
});
debug!("Discovered {} imports for {}", imports.len(), path);
// Append any imports that were statically defined in the configuration.
if let Some((root, globs)) = include_dependencies {
let mut glob_resolver = glob_resolver.lock().unwrap();
imports.extend(glob_resolver.resolve(root, globs));
// Skip excluded files.
if (settings.file_resolver.force_exclude || !resolved_file.is_root())
&& match_exclusion(
resolved_file.path(),
resolved_file.file_name(),
&settings.analyze.exclude,
)
{
continue;
}
// Convert the path (and imports) to be relative to the working directory.
let path = path
.strip_prefix(&root)
.map(SystemPath::to_path_buf)
.unwrap_or(path);
let imports = imports.relative_to(&root);
// Ignore non-Python files.
let source_type = match settings.analyze.extension.get(path) {
None => match SourceType::from(&path) {
SourceType::Python(source_type) => source_type,
SourceType::Toml(_) => {
debug!("Ignoring TOML file: {}", path.display());
continue;
}
},
Some(language) => PySourceType::from(language),
};
if matches!(source_type, PySourceType::Ipynb) {
debug!("Ignoring Jupyter notebook: {}", path.display());
continue;
}
result.lock().unwrap().push((path, imports));
});
}
});
// Convert to system paths.
let Ok(package) = package.map(SystemPathBuf::from_path_buf).transpose() else {
warn!("Failed to convert package to system path");
continue;
};
let Ok(path) = SystemPathBuf::from_path_buf(resolved_file.into_path()) else {
warn!("Failed to convert path to system path");
continue;
};
// Collect the results.
let imports = Arc::into_inner(result).unwrap().into_inner()?;
let db = db.snapshot();
let glob_resolver = glob_resolver.clone();
let root = root.clone();
let result = inner_result.clone();
scope.spawn(move |_| {
// Identify any imports via static analysis.
let mut imports =
ModuleImports::detect(&db, &path, package.as_deref(), string_imports)
.unwrap_or_else(|err| {
warn!("Failed to generate import map for {path}: {err}");
ModuleImports::default()
});
debug!("Discovered {} imports for {}", imports.len(), path);
// Append any imports that were statically defined in the configuration.
if let Some((root, globs)) = include_dependencies {
let mut glob_resolver = glob_resolver.lock().unwrap();
imports.extend(glob_resolver.resolve(root, globs));
}
// Convert the path (and imports) to be relative to the working directory.
let path = path
.strip_prefix(&root)
.map(SystemPath::to_path_buf)
.unwrap_or(path);
let imports = imports.relative_to(&root);
result.lock().unwrap().push((path, imports));
});
}
});
// Collect the results.
Arc::into_inner(result).unwrap().into_inner()?
};
// Generate the import map.
let import_map = match args.direction {
@@ -173,6 +176,8 @@ pub(crate) fn analyze_graph(
// Print to JSON.
println!("{}", serde_json::to_string_pretty(&import_map)?);
std::mem::forget(db);
Ok(ExitStatus::Success)
}

View File

@@ -33,14 +33,9 @@ tracing = { workspace = true }
tracing-subscriber = { workspace = true, optional = true }
tracing-tree = { workspace = true, optional = true }
rustc-hash = { workspace = true }
zip = { workspace = true }
[target.'cfg(not(any(target_arch = "wasm32", target_arch = "powerpc64")))'.dependencies]
zip = { workspace = true, features = ["zstd"] }
[target.'cfg(any(target_arch = "wasm32", target_arch = "powerpc64"))'.dependencies]
zip = { workspace = true, features = ["deflate"] }
[target.'cfg(target_arch = "wasm32")'.dependencies]
[target.'cfg(target_arch="wasm32")'.dependencies]
web-time = { version = "1.1.0" }
[dev-dependencies]
@@ -48,6 +43,7 @@ insta = { workspace = true }
tempfile = { workspace = true }
[features]
default = ["os"]
cache = ["ruff_cache"]
os = ["ignore"]
serde = ["dep:serde", "camino/serde1"]

View File

@@ -503,7 +503,8 @@ mod tests {
use crate::files::{system_path_to_file, vendored_path_to_file, FileError};
use crate::system::DbWithTestSystem;
use crate::tests::TestDb;
use crate::vendored::tests::VendoredFileSystemBuilder;
use crate::vendored::VendoredFileSystemBuilder;
use zip::CompressionMethod;
#[test]
fn system_existing_file() -> crate::system::Result<()> {
@@ -548,7 +549,7 @@ mod tests {
fn stubbed_vendored_file() -> crate::system::Result<()> {
let mut db = TestDb::new();
let mut vendored_builder = VendoredFileSystemBuilder::new();
let mut vendored_builder = VendoredFileSystemBuilder::new(CompressionMethod::Stored);
vendored_builder
.add_file("test.pyi", "def foo() -> str")
.unwrap();

View File

@@ -79,8 +79,9 @@ mod tests {
use crate::parsed::parsed_module;
use crate::system::{DbWithTestSystem, SystemPath, SystemVirtualPath};
use crate::tests::TestDb;
use crate::vendored::{tests::VendoredFileSystemBuilder, VendoredPath};
use crate::vendored::{VendoredFileSystemBuilder, VendoredPath};
use crate::Db;
use zip::CompressionMethod;
#[test]
fn python_file() -> crate::system::Result<()> {
@@ -150,7 +151,7 @@ mod tests {
fn vendored_file() {
let mut db = TestDb::new();
let mut vendored_builder = VendoredFileSystemBuilder::new();
let mut vendored_builder = VendoredFileSystemBuilder::new(CompressionMethod::Stored);
vendored_builder
.add_file(
"path.pyi",

View File

@@ -1,12 +1,13 @@
use std::borrow::Cow;
use std::collections::BTreeMap;
use std::fmt::{self, Debug};
use std::io::{self, Read};
use std::io::{self, Read, Write};
use std::sync::{Arc, Mutex, MutexGuard};
use zip::{read::ZipFile, ZipArchive, ZipWriter};
use crate::file_revision::FileRevision;
use zip::result::ZipResult;
use zip::write::FileOptions;
use zip::{read::ZipFile, CompressionMethod, ZipArchive, ZipWriter};
pub use self::path::{VendoredPath, VendoredPathBuf};
@@ -177,7 +178,6 @@ struct ZipFileDebugInfo {
crc32_hash: u32,
compressed_size: u64,
uncompressed_size: u64,
compression_method: zip::CompressionMethod,
kind: FileType,
}
@@ -187,7 +187,6 @@ impl<'a> From<ZipFile<'a>> for ZipFileDebugInfo {
crc32_hash: value.crc32(),
compressed_size: value.compressed_size(),
uncompressed_size: value.size(),
compression_method: value.compression(),
kind: if value.is_dir() {
FileType::Directory
} else {
@@ -341,69 +340,61 @@ impl<'a> From<&'a VendoredPath> for NormalizedVendoredPath<'a> {
}
}
pub struct VendoredFileSystemBuilder {
writer: ZipWriter<io::Cursor<Vec<u8>>>,
compression_method: CompressionMethod,
}
impl VendoredFileSystemBuilder {
pub fn new(compression_method: CompressionMethod) -> Self {
let buffer = io::Cursor::new(Vec::new());
Self {
writer: ZipWriter::new(buffer),
compression_method,
}
}
pub fn add_file(
&mut self,
path: impl AsRef<VendoredPath>,
content: &str,
) -> std::io::Result<()> {
self.writer
.start_file(path.as_ref().as_str(), self.options())?;
self.writer.write_all(content.as_bytes())
}
pub fn add_directory(&mut self, path: impl AsRef<VendoredPath>) -> ZipResult<()> {
self.writer
.add_directory(path.as_ref().as_str(), self.options())
}
pub fn finish(mut self) -> Result<VendoredFileSystem> {
let buffer = self.writer.finish()?;
VendoredFileSystem::new(buffer.into_inner())
}
fn options(&self) -> FileOptions {
FileOptions::default()
.compression_method(self.compression_method)
.unix_permissions(0o644)
}
}
#[cfg(test)]
pub(crate) mod tests {
use std::io::Write;
use insta::assert_snapshot;
use zip::result::ZipResult;
use zip::write::FileOptions;
use zip::{CompressionMethod, ZipWriter};
use super::*;
const FUNCTOOLS_CONTENTS: &str = "def update_wrapper(): ...";
const ASYNCIO_TASKS_CONTENTS: &str = "class Task: ...";
pub struct VendoredFileSystemBuilder {
writer: ZipWriter<io::Cursor<Vec<u8>>>,
}
impl Default for VendoredFileSystemBuilder {
fn default() -> Self {
Self::new()
}
}
impl VendoredFileSystemBuilder {
pub fn new() -> Self {
let buffer = io::Cursor::new(Vec::new());
Self {
writer: ZipWriter::new(buffer),
}
}
pub fn add_file(
&mut self,
path: impl AsRef<VendoredPath>,
content: &str,
) -> std::io::Result<()> {
self.writer
.start_file(path.as_ref().as_str(), Self::options())?;
self.writer.write_all(content.as_bytes())
}
pub fn add_directory(&mut self, path: impl AsRef<VendoredPath>) -> ZipResult<()> {
self.writer
.add_directory(path.as_ref().as_str(), Self::options())
}
pub fn finish(mut self) -> Result<VendoredFileSystem> {
let buffer = self.writer.finish()?;
VendoredFileSystem::new(buffer.into_inner())
}
fn options() -> FileOptions {
FileOptions::default()
.compression_method(CompressionMethod::Zstd)
.unix_permissions(0o644)
}
}
fn mock_typeshed() -> VendoredFileSystem {
let mut builder = VendoredFileSystemBuilder::new();
let mut builder = VendoredFileSystemBuilder::new(CompressionMethod::Stored);
builder.add_directory("stdlib/").unwrap();
builder
@@ -441,28 +432,24 @@ pub(crate) mod tests {
crc32_hash: 0,
compressed_size: 0,
uncompressed_size: 0,
compression_method: Stored,
kind: Directory,
},
"stdlib/asyncio/": ZipFileDebugInfo {
crc32_hash: 0,
compressed_size: 0,
uncompressed_size: 0,
compression_method: Stored,
kind: Directory,
},
"stdlib/asyncio/tasks.pyi": ZipFileDebugInfo {
crc32_hash: 2826547428,
compressed_size: 24,
compressed_size: 15,
uncompressed_size: 15,
compression_method: Zstd,
kind: File,
},
"stdlib/functools.pyi": ZipFileDebugInfo {
crc32_hash: 1099005079,
compressed_size: 34,
compressed_size: 25,
uncompressed_size: 25,
compression_method: Zstd,
kind: File,
},
},

View File

@@ -16,12 +16,15 @@ ruff_db = { workspace = true, features = ["os", "serde"] }
ruff_linter = { workspace = true }
ruff_macros = { workspace = true }
ruff_python_ast = { workspace = true }
ruff_python_parser = { workspace = true }
anyhow = { workspace = true }
clap = { workspace = true, optional = true }
once_cell = { workspace = true }
salsa = { workspace = true }
schemars = { workspace = true, optional = true }
serde = { workspace = true, optional = true }
zip = { workspace = true, features = [] }
[lints]
workspace = true

View File

@@ -1,6 +1,8 @@
use red_knot_python_semantic::ModuleName;
use ruff_python_ast::visitor::source_order::{walk_body, walk_expr, walk_stmt, SourceOrderVisitor};
use ruff_python_ast::{self as ast, Expr, ModModule, Stmt};
use ruff_python_ast::visitor::source_order::{
walk_expr, walk_module, walk_stmt, SourceOrderVisitor,
};
use ruff_python_ast::{self as ast, Expr, Mod, Stmt};
/// Collect all imports for a given Python file.
#[derive(Default, Debug)]
@@ -23,8 +25,8 @@ impl<'a> Collector<'a> {
}
#[must_use]
pub(crate) fn collect(mut self, module: &ModModule) -> Vec<CollectedImport> {
walk_body(&mut self, &module.body);
pub(crate) fn collect(mut self, module: &Mod) -> Vec<CollectedImport> {
walk_module(&mut self, module);
self.imports
}
}

View File

@@ -1,17 +1,25 @@
use anyhow::Result;
use zip::CompressionMethod;
use red_knot_python_semantic::{Db, Program, ProgramSettings, PythonVersion, SearchPathSettings};
use ruff_db::files::{File, Files};
use ruff_db::system::{OsSystem, System, SystemPathBuf};
use ruff_db::vendored::VendoredFileSystem;
use ruff_db::vendored::{VendoredFileSystem, VendoredFileSystemBuilder};
use ruff_db::{Db as SourceDb, Upcast};
static EMPTY_VENDORED: once_cell::sync::Lazy<VendoredFileSystem> =
once_cell::sync::Lazy::new(|| {
let mut builder = VendoredFileSystemBuilder::new(CompressionMethod::Stored);
builder.add_file("stdlib/VERSIONS", "\n").unwrap();
builder.finish().unwrap()
});
#[salsa::db]
#[derive(Default)]
pub struct ModuleDb {
storage: salsa::Storage<Self>,
files: Files,
system: OsSystem,
vendored: VendoredFileSystem,
}
impl ModuleDb {
@@ -26,12 +34,10 @@ impl ModuleDb {
.next()
.ok_or_else(|| anyhow::anyhow!("No source roots provided"))?;
let mut search_paths = SearchPathSettings::new(src_root.to_path_buf());
let mut search_paths = SearchPathSettings::new(src_root);
// Add the remaining source roots as extra paths.
for src_root in src_roots {
search_paths.extra_paths.push(src_root.to_path_buf());
}
search_paths.extra_paths.extend(src_roots);
search_paths
};
@@ -54,7 +60,6 @@ impl ModuleDb {
Self {
storage: self.storage.clone(),
system: self.system.clone(),
vendored: self.vendored.clone(),
files: self.files.snapshot(),
}
}
@@ -72,7 +77,7 @@ impl Upcast<dyn SourceDb> for ModuleDb {
#[salsa::db]
impl SourceDb for ModuleDb {
fn vendored(&self) -> &VendoredFileSystem {
&self.vendored
&EMPTY_VENDORED
}
fn system(&self) -> &dyn System {

View File

@@ -3,11 +3,9 @@ pub use crate::db::ModuleDb;
use crate::resolver::Resolver;
pub use crate::settings::{AnalyzeSettings, Direction};
use anyhow::Result;
use red_knot_python_semantic::SemanticModel;
use ruff_db::files::system_path_to_file;
use ruff_db::parsed::parsed_module;
use ruff_db::system::{SystemPath, SystemPathBuf};
use ruff_python_ast::helpers::to_module_path;
use ruff_python_parser::{parse, Mode};
use serde::{Deserialize, Serialize};
use std::collections::{BTreeMap, BTreeSet};
@@ -29,11 +27,11 @@ impl ModuleImports {
string_imports: bool,
) -> Result<Self> {
// Read and parse the source code.
let file = system_path_to_file(db, path)?;
let parsed = parsed_module(db, file);
let source = std::fs::read_to_string(path)?;
let parsed = parse(&source, Mode::Module)?;
let module_path =
package.and_then(|package| to_module_path(package.as_std_path(), path.as_std_path()));
let model = SemanticModel::new(db, file);
// Collect the imports.
let imports =
@@ -42,7 +40,7 @@ impl ModuleImports {
// Resolve the imports.
let mut resolved_imports = ModuleImports::default();
for import in imports {
let Some(resolved) = Resolver::new(&model).resolve(import) else {
let Some(resolved) = Resolver::new(db).resolve(import) else {
continue;
};
let Some(path) = resolved.as_system_path() else {

View File

@@ -1,37 +1,36 @@
use red_knot_python_semantic::SemanticModel;
use red_knot_python_semantic::resolve_module;
use ruff_db::files::FilePath;
use crate::collector::CollectedImport;
use crate::ModuleDb;
/// Collect all imports for a given Python file.
pub(crate) struct Resolver<'a> {
semantic: &'a SemanticModel<'a>,
db: &'a ModuleDb,
}
impl<'a> Resolver<'a> {
/// Initialize a [`Resolver`] with a given [`SemanticModel`].
pub(crate) fn new(semantic: &'a SemanticModel<'a>) -> Self {
Self { semantic }
/// Initialize a [`Resolver`] with a given [`ModuleDb`].
pub(crate) fn new(db: &'a ModuleDb) -> Self {
Self { db }
}
/// Resolve the [`CollectedImport`] into a [`FilePath`].
pub(crate) fn resolve(&self, import: CollectedImport) -> Option<&'a FilePath> {
match import {
CollectedImport::Import(import) => self
.semantic
.resolve_module(import)
.map(|module| module.file().path(self.semantic.db())),
CollectedImport::Import(import) => {
resolve_module(self.db, import).map(|module| module.file().path(self.db))
}
CollectedImport::ImportFrom(import) => {
// Attempt to resolve the member (e.g., given `from foo import bar`, look for `foo.bar`).
let parent = import.parent();
self.semantic
.resolve_module(import)
.map(|module| module.file().path(self.semantic.db()))
resolve_module(self.db, import)
.map(|module| module.file().path(self.db))
.or_else(|| {
// Attempt to resolve the module (e.g., given `from foo import bar`, look for `foo`).
self.semantic
.resolve_module(parent?)
.map(|module| module.file().path(self.semantic.db()))
resolve_module(self.db, parent?).map(|module| module.file().path(self.db))
})
}
}

View File

@@ -1,6 +1,6 @@
[package]
name = "ruff_linter"
version = "0.6.6"
version = "0.6.7"
publish = false
authors = { workspace = true }
edition = { workspace = true }

View File

@@ -0,0 +1,32 @@
[package]
name = "ruff_vendored"
version = "0.0.0"
publish = false
authors = { workspace = true }
edition = { workspace = true }
rust-version = { workspace = true }
homepage = { workspace = true }
documentation = { workspace = true }
repository = { workspace = true }
license = { workspace = true }
[dependencies]
ruff_db = { workspace = true }
once_cell = { workspace = true }
zip = { workspace = true }
[build-dependencies]
path-slash = { workspace = true }
walkdir = { workspace = true }
zip = { workspace = true, features = ["zstd", "deflate"] }
[dev-dependencies]
walkdir = { workspace = true }
[features]
zstd = ["zip/zstd"]
deflate = ["zip/deflate"]
[lints]
workspace = true

View File

@@ -30,17 +30,12 @@ fn zip_dir(directory_path: &str, writer: File) -> ZipResult<File> {
// We can't use `#[cfg(...)]` here because the target-arch in a build script is the
// architecture of the system running the build script and not the architecture of the build-target.
// That's why we use the `TARGET` environment variable here.
#[cfg(target_arch = "powerpc64")]
let method = CompressionMethod::Deflated;
#[cfg(not(target_arch = "powerpc64"))]
let method = {
let target = std::env::var("TARGET").unwrap();
if target.contains("wasm32") || target.contains("powerpc64") {
CompressionMethod::Deflated
} else {
CompressionMethod::Zstd
}
let method = if cfg!(feature = "zstd") {
CompressionMethod::Zstd
} else if cfg!(feature = "deflate") {
CompressionMethod::Deflated
} else {
CompressionMethod::Stored
};
let options = FileOptions::default()

View File

@@ -6,7 +6,7 @@ use ruff_db::vendored::VendoredFileSystem;
// Luckily this crate will fail to build if this file isn't available at build time.
static TYPESHED_ZIP_BYTES: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/zipped_typeshed.zip"));
pub fn vendored_typeshed_stubs() -> &'static VendoredFileSystem {
pub fn file_system() -> &'static VendoredFileSystem {
static VENDORED_TYPESHED_STUBS: Lazy<VendoredFileSystem> =
Lazy::new(|| VendoredFileSystem::new_static(TYPESHED_ZIP_BYTES).unwrap());
&VENDORED_TYPESHED_STUBS
@@ -42,7 +42,7 @@ mod tests {
#[test]
fn typeshed_vfs_consistent_with_vendored_stubs() {
let vendored_typeshed_dir = Path::new("vendor/typeshed").canonicalize().unwrap();
let vendored_typeshed_stubs = vendored_typeshed_stubs();
let vendored_typeshed_stubs = file_system();
let mut empty_iterator = true;
for entry in walkdir::WalkDir::new(&vendored_typeshed_dir).min_depth(1) {

View File

@@ -1,4 +1,4 @@
# PEP 249 Database API 2.0 Types
# PEP 249 Database API 2.0 Types
# https://www.python.org/dev/peps/pep-0249/
from collections.abc import Mapping, Sequence

Some files were not shown because too many files have changed in this diff Show More