Compare commits
2 Commits
red-knot-f
...
dhruv/curs
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
ea48d1068d | ||
|
|
afe4cdb691 |
28
Cargo.lock
generated
28
Cargo.lock
generated
@@ -501,19 +501,6 @@ dependencies = [
|
||||
"itertools 0.10.5",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam"
|
||||
version = "0.8.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1137cd7e7fc0fb5d3c5a8678be38ec56e819125d8d7907411fe24ccb943faca8"
|
||||
dependencies = [
|
||||
"crossbeam-channel",
|
||||
"crossbeam-deque",
|
||||
"crossbeam-epoch",
|
||||
"crossbeam-queue",
|
||||
"crossbeam-utils",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-channel"
|
||||
version = "0.5.12"
|
||||
@@ -542,15 +529,6 @@ dependencies = [
|
||||
"crossbeam-utils",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-queue"
|
||||
version = "0.3.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "df0346b5d5e76ac2fe4e327c5fd1118d6be7c51dfb18f9b7922923f287471e35"
|
||||
dependencies = [
|
||||
"crossbeam-utils",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-utils"
|
||||
version = "0.8.19"
|
||||
@@ -1826,7 +1804,7 @@ version = "0.1.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"bitflags 2.5.0",
|
||||
"crossbeam",
|
||||
"crossbeam-channel",
|
||||
"ctrlc",
|
||||
"dashmap",
|
||||
"hashbrown 0.14.5",
|
||||
@@ -1835,11 +1813,9 @@ dependencies = [
|
||||
"notify",
|
||||
"parking_lot",
|
||||
"rayon",
|
||||
"ruff_formatter",
|
||||
"ruff_index",
|
||||
"ruff_notebook",
|
||||
"ruff_python_ast",
|
||||
"ruff_python_formatter",
|
||||
"ruff_python_parser",
|
||||
"ruff_python_trivia",
|
||||
"ruff_text_size",
|
||||
@@ -2365,7 +2341,7 @@ name = "ruff_server"
|
||||
version = "0.2.2"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"crossbeam",
|
||||
"crossbeam-channel",
|
||||
"insta",
|
||||
"jod-thread",
|
||||
"libc",
|
||||
|
||||
@@ -30,7 +30,7 @@ console_error_panic_hook = { version = "0.1.7" }
|
||||
console_log = { version = "1.0.0" }
|
||||
countme = { version = "3.0.1" }
|
||||
criterion = { version = "0.5.1", default-features = false }
|
||||
crossbeam = { version = "0.8.4" }
|
||||
crossbeam-channel = { version = "0.5.12" }
|
||||
dashmap = { version = "5.5.3" }
|
||||
dirs = { version = "5.0.0" }
|
||||
drop_bomb = { version = "0.1.5" }
|
||||
|
||||
@@ -12,19 +12,17 @@ license.workspace = true
|
||||
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
||||
|
||||
[dependencies]
|
||||
ruff_formatter = { path = "../ruff_formatter" }
|
||||
ruff_index = { path = "../ruff_index" }
|
||||
ruff_notebook = { path = "../ruff_notebook" }
|
||||
ruff_python_ast = { path = "../ruff_python_ast" }
|
||||
ruff_python_formatter = { path = "../ruff_python_formatter" }
|
||||
ruff_python_parser = { path = "../ruff_python_parser" }
|
||||
ruff_python_ast = { path = "../ruff_python_ast" }
|
||||
ruff_python_trivia = { path = "../ruff_python_trivia" }
|
||||
ruff_text_size = { path = "../ruff_text_size" }
|
||||
ruff_index = { path = "../ruff_index" }
|
||||
ruff_notebook = { path = "../ruff_notebook" }
|
||||
|
||||
anyhow = { workspace = true }
|
||||
bitflags = { workspace = true }
|
||||
ctrlc = "3.4.4"
|
||||
crossbeam = { workspace = true }
|
||||
crossbeam-channel = { workspace = true }
|
||||
dashmap = { workspace = true }
|
||||
hashbrown = { workspace = true }
|
||||
indexmap = { workspace = true }
|
||||
|
||||
@@ -2,7 +2,6 @@ use std::fmt::Formatter;
|
||||
use std::hash::Hash;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
|
||||
use crate::db::QueryResult;
|
||||
use dashmap::mapref::entry::Entry;
|
||||
|
||||
use crate::FxDashMap;
|
||||
@@ -28,11 +27,11 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
pub fn get<F>(&self, key: &K, compute: F) -> QueryResult<V>
|
||||
pub fn get<F>(&self, key: &K, compute: F) -> V
|
||||
where
|
||||
F: FnOnce(&K) -> QueryResult<V>,
|
||||
F: FnOnce(&K) -> V,
|
||||
{
|
||||
Ok(match self.map.entry(key.clone()) {
|
||||
match self.map.entry(key.clone()) {
|
||||
Entry::Occupied(cached) => {
|
||||
self.statistics.hit();
|
||||
|
||||
@@ -41,11 +40,11 @@ where
|
||||
Entry::Vacant(vacant) => {
|
||||
self.statistics.miss();
|
||||
|
||||
let value = compute(key)?;
|
||||
let value = compute(key);
|
||||
vacant.insert(value.clone());
|
||||
value
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
pub fn set(&mut self, key: K, value: V) {
|
||||
@@ -118,29 +117,23 @@ pub type CacheStatistics = DebugStatistics;
|
||||
#[cfg(not(debug_assertions))]
|
||||
pub type CacheStatistics = ReleaseStatistics;
|
||||
|
||||
pub trait StatisticsRecorder {
|
||||
fn hit(&self);
|
||||
fn miss(&self);
|
||||
fn to_statistics(&self) -> Option<Statistics>;
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
pub struct DebugStatistics {
|
||||
hits: AtomicUsize,
|
||||
misses: AtomicUsize,
|
||||
}
|
||||
|
||||
impl StatisticsRecorder for DebugStatistics {
|
||||
impl DebugStatistics {
|
||||
// TODO figure out appropriate Ordering
|
||||
fn hit(&self) {
|
||||
pub fn hit(&self) {
|
||||
self.hits.fetch_add(1, Ordering::SeqCst);
|
||||
}
|
||||
|
||||
fn miss(&self) {
|
||||
pub fn miss(&self) {
|
||||
self.misses.fetch_add(1, Ordering::SeqCst);
|
||||
}
|
||||
|
||||
fn to_statistics(&self) -> Option<Statistics> {
|
||||
pub fn to_statistics(&self) -> Option<Statistics> {
|
||||
let hits = self.hits.load(Ordering::SeqCst);
|
||||
let misses = self.misses.load(Ordering::SeqCst);
|
||||
|
||||
@@ -151,15 +144,15 @@ impl StatisticsRecorder for DebugStatistics {
|
||||
#[derive(Debug, Default)]
|
||||
pub struct ReleaseStatistics;
|
||||
|
||||
impl StatisticsRecorder for ReleaseStatistics {
|
||||
impl ReleaseStatistics {
|
||||
#[inline]
|
||||
fn hit(&self) {}
|
||||
pub const fn hit(&self) {}
|
||||
|
||||
#[inline]
|
||||
fn miss(&self) {}
|
||||
pub const fn miss(&self) {}
|
||||
|
||||
#[inline]
|
||||
fn to_statistics(&self) -> Option<Statistics> {
|
||||
pub const fn to_statistics(&self) -> Option<Statistics> {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,25 +1,35 @@
|
||||
use std::sync::atomic::AtomicBool;
|
||||
use std::sync::Arc;
|
||||
use std::sync::{Arc, Condvar, Mutex};
|
||||
|
||||
#[derive(Debug, Clone, Default)]
|
||||
#[derive(Debug, Default)]
|
||||
pub struct CancellationTokenSource {
|
||||
signal: Arc<AtomicBool>,
|
||||
signal: Arc<(Mutex<bool>, Condvar)>,
|
||||
}
|
||||
|
||||
impl CancellationTokenSource {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
signal: Arc::new(AtomicBool::new(false)),
|
||||
signal: Arc::new((Mutex::new(false), Condvar::default())),
|
||||
}
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "trace", skip_all)]
|
||||
pub fn cancel(&self) {
|
||||
self.signal.store(true, std::sync::atomic::Ordering::SeqCst);
|
||||
let (cancelled, condvar) = &*self.signal;
|
||||
|
||||
let mut cancelled = cancelled.lock().unwrap();
|
||||
|
||||
if *cancelled {
|
||||
return;
|
||||
}
|
||||
|
||||
*cancelled = true;
|
||||
condvar.notify_all();
|
||||
}
|
||||
|
||||
pub fn is_cancelled(&self) -> bool {
|
||||
self.signal.load(std::sync::atomic::Ordering::SeqCst)
|
||||
let (cancelled, _) = &*self.signal;
|
||||
|
||||
*cancelled.lock().unwrap()
|
||||
}
|
||||
|
||||
pub fn token(&self) -> CancellationToken {
|
||||
@@ -31,12 +41,26 @@ impl CancellationTokenSource {
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct CancellationToken {
|
||||
signal: Arc<AtomicBool>,
|
||||
signal: Arc<(Mutex<bool>, Condvar)>,
|
||||
}
|
||||
|
||||
impl CancellationToken {
|
||||
/// Returns `true` if cancellation has been requested.
|
||||
pub fn is_cancelled(&self) -> bool {
|
||||
self.signal.load(std::sync::atomic::Ordering::SeqCst)
|
||||
let (cancelled, _) = &*self.signal;
|
||||
|
||||
*cancelled.lock().unwrap()
|
||||
}
|
||||
|
||||
pub fn wait(&self) {
|
||||
let (bool, condvar) = &*self.signal;
|
||||
|
||||
let lock = condvar
|
||||
.wait_while(bool.lock().unwrap(), |bool| !*bool)
|
||||
.unwrap();
|
||||
|
||||
debug_assert!(*lock);
|
||||
|
||||
drop(lock);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,11 +1,6 @@
|
||||
use std::path::Path;
|
||||
use std::sync::Arc;
|
||||
|
||||
pub use jars::{HasJar, HasJars};
|
||||
pub use query::{QueryError, QueryResult};
|
||||
pub use runtime::DbRuntime;
|
||||
pub use storage::JarsStorage;
|
||||
|
||||
use crate::files::FileId;
|
||||
use crate::lint::{Diagnostics, LintSemanticStorage, LintSyntaxStorage};
|
||||
use crate::module::{Module, ModuleData, ModuleName, ModuleResolver, ModuleSearchPath};
|
||||
@@ -14,111 +9,32 @@ use crate::source::{Source, SourceStorage};
|
||||
use crate::symbols::{SymbolId, SymbolTable, SymbolTablesStorage};
|
||||
use crate::types::{Type, TypeStore};
|
||||
|
||||
mod jars;
|
||||
mod query;
|
||||
mod runtime;
|
||||
mod storage;
|
||||
|
||||
pub trait Database {
|
||||
/// Returns a reference to the runtime of the current worker.
|
||||
fn runtime(&self) -> &DbRuntime;
|
||||
|
||||
/// Returns a mutable reference to the runtime. Only one worker can hold a mutable reference to the runtime.
|
||||
fn runtime_mut(&mut self) -> &mut DbRuntime;
|
||||
|
||||
/// Returns `Ok` if the queries have not been cancelled and `Err(QueryError::Cancelled)` otherwise.
|
||||
fn cancelled(&self) -> QueryResult<()> {
|
||||
self.runtime().cancelled()
|
||||
}
|
||||
|
||||
/// Returns `true` if the queries have been cancelled.
|
||||
fn is_cancelled(&self) -> bool {
|
||||
self.runtime().is_cancelled()
|
||||
}
|
||||
}
|
||||
|
||||
/// Database that supports running queries from multiple threads.
|
||||
pub trait ParallelDatabase: Database + Send {
|
||||
/// Creates a snapshot of the database state that can be used to query the database in another thread.
|
||||
///
|
||||
/// The snapshot is a read-only view of the database but query results are shared between threads.
|
||||
/// All queries will be automatically cancelled when applying any mutations (calling [`HasJars::jars_mut`])
|
||||
/// to the database (not the snapshot, because they're readonly).
|
||||
///
|
||||
/// ## Creating a snapshot
|
||||
///
|
||||
/// Creating a snapshot of the database's jars is cheap but creating a snapshot of
|
||||
/// other state stored on the database might require deep-cloning data. That's why you should
|
||||
/// avoid creating snapshots in a hot function (e.g. don't create a snapshot for each file, instead
|
||||
/// create a snapshot when scheduling the check of an entire program).
|
||||
///
|
||||
/// ## Salsa compatibility
|
||||
/// Salsa prohibits creating a snapshot while running a local query (it's fine if other workers run a query) [[source](https://github.com/salsa-rs/salsa/issues/80)].
|
||||
/// We should avoid creating snapshots while running a query because we might want to adopt Salsa in the future (if we can figure out persistent caching).
|
||||
/// Unfortunately, the infrastructure doesn't provide an automated way of knowing when a query is run, that's
|
||||
/// why we have to "enforce" this constraint manually.
|
||||
fn snapshot(&self) -> Snapshot<Self>;
|
||||
}
|
||||
|
||||
/// Readonly snapshot of a database.
|
||||
///
|
||||
/// ## Dead locks
|
||||
/// A snapshot should always be dropped as soon as it is no longer necessary to run queries.
|
||||
/// Storing the snapshot without running a query or periodically checking if cancellation was requested
|
||||
/// can lead to deadlocks because mutating the [`Database`] requires cancels all pending queries
|
||||
/// and waiting for all [`Snapshot`]s to be dropped.
|
||||
#[derive(Debug)]
|
||||
pub struct Snapshot<DB: ?Sized>
|
||||
where
|
||||
DB: ParallelDatabase,
|
||||
{
|
||||
db: DB,
|
||||
}
|
||||
|
||||
impl<DB> Snapshot<DB>
|
||||
where
|
||||
DB: ParallelDatabase,
|
||||
{
|
||||
pub fn new(db: DB) -> Self {
|
||||
Snapshot { db }
|
||||
}
|
||||
}
|
||||
|
||||
impl<DB> std::ops::Deref for Snapshot<DB>
|
||||
where
|
||||
DB: ParallelDatabase,
|
||||
{
|
||||
type Target = DB;
|
||||
|
||||
fn deref(&self) -> &DB {
|
||||
&self.db
|
||||
}
|
||||
}
|
||||
|
||||
// Red knot specific databases code.
|
||||
|
||||
pub trait SourceDb: Database {
|
||||
pub trait SourceDb {
|
||||
// queries
|
||||
fn file_id(&self, path: &std::path::Path) -> FileId;
|
||||
|
||||
fn file_path(&self, file_id: FileId) -> Arc<std::path::Path>;
|
||||
|
||||
fn source(&self, file_id: FileId) -> QueryResult<Source>;
|
||||
fn source(&self, file_id: FileId) -> Source;
|
||||
|
||||
fn parse(&self, file_id: FileId) -> QueryResult<Parsed>;
|
||||
fn parse(&self, file_id: FileId) -> Parsed;
|
||||
|
||||
fn lint_syntax(&self, file_id: FileId) -> Diagnostics;
|
||||
}
|
||||
|
||||
pub trait SemanticDb: SourceDb {
|
||||
// queries
|
||||
fn resolve_module(&self, name: ModuleName) -> QueryResult<Option<Module>>;
|
||||
fn resolve_module(&self, name: ModuleName) -> Option<Module>;
|
||||
|
||||
fn file_to_module(&self, file_id: FileId) -> QueryResult<Option<Module>>;
|
||||
fn file_to_module(&self, file_id: FileId) -> Option<Module>;
|
||||
|
||||
fn path_to_module(&self, path: &Path) -> QueryResult<Option<Module>>;
|
||||
fn path_to_module(&self, path: &Path) -> Option<Module>;
|
||||
|
||||
fn symbol_table(&self, file_id: FileId) -> QueryResult<Arc<SymbolTable>>;
|
||||
fn symbol_table(&self, file_id: FileId) -> Arc<SymbolTable>;
|
||||
|
||||
fn infer_symbol_type(&self, file_id: FileId, symbol_id: SymbolId) -> QueryResult<Type>;
|
||||
fn infer_symbol_type(&self, file_id: FileId, symbol_id: SymbolId) -> Type;
|
||||
|
||||
fn lint_semantic(&self, file_id: FileId) -> Diagnostics;
|
||||
|
||||
// mutations
|
||||
|
||||
@@ -127,18 +43,13 @@ pub trait SemanticDb: SourceDb {
|
||||
fn set_module_search_paths(&mut self, paths: Vec<ModuleSearchPath>);
|
||||
}
|
||||
|
||||
pub trait LintDb: SemanticDb {
|
||||
fn lint_syntax(&self, file_id: FileId) -> QueryResult<Diagnostics>;
|
||||
|
||||
fn lint_semantic(&self, file_id: FileId) -> QueryResult<Diagnostics>;
|
||||
}
|
||||
|
||||
pub trait Db: LintDb {}
|
||||
pub trait Db: SemanticDb {}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
pub struct SourceJar {
|
||||
pub sources: SourceStorage,
|
||||
pub parsed: ParsedStorage,
|
||||
pub lint_syntax: LintSyntaxStorage,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
@@ -146,12 +57,27 @@ pub struct SemanticJar {
|
||||
pub module_resolver: ModuleResolver,
|
||||
pub symbol_tables: SymbolTablesStorage,
|
||||
pub type_store: TypeStore,
|
||||
pub lint_semantic: LintSemanticStorage,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
pub struct LintJar {
|
||||
pub lint_syntax: LintSyntaxStorage,
|
||||
pub lint_semantic: LintSemanticStorage,
|
||||
/// Gives access to a specific jar in the database.
|
||||
///
|
||||
/// Nope, the terminology isn't borrowed from Java but from Salsa <https://salsa-rs.github.io/salsa/>,
|
||||
/// which is an analogy to storing the salsa in different jars.
|
||||
///
|
||||
/// The basic idea is that each crate can define its own jar and the jars can be combined to a single
|
||||
/// database in the top level crate. Each crate also defines its own `Database` trait. The combination of
|
||||
/// `Database` trait and the jar allows to write queries in isolation without having to know how they get composed at the upper levels.
|
||||
///
|
||||
/// Salsa further defines a `HasIngredient` trait which slices the jar to a specific storage (e.g. a specific cache).
|
||||
/// We don't need this just jet because we write our queries by hand. We may want a similar trait if we decide
|
||||
/// to use a macro to generate the queries.
|
||||
pub trait HasJar<T> {
|
||||
/// Gives a read-only reference to the jar.
|
||||
fn jar(&self) -> &T;
|
||||
|
||||
/// Gives a mutable reference to the jar.
|
||||
fn jar_mut(&mut self) -> &mut T;
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -159,10 +85,7 @@ pub(crate) mod tests {
|
||||
use std::path::Path;
|
||||
use std::sync::Arc;
|
||||
|
||||
use crate::db::{
|
||||
Database, DbRuntime, HasJar, HasJars, JarsStorage, LintDb, LintJar, QueryResult, SourceDb,
|
||||
SourceJar,
|
||||
};
|
||||
use crate::db::{HasJar, SourceDb, SourceJar};
|
||||
use crate::files::{FileId, Files};
|
||||
use crate::lint::{lint_semantic, lint_syntax, Diagnostics};
|
||||
use crate::module::{
|
||||
@@ -181,36 +104,27 @@ pub(crate) mod tests {
|
||||
#[derive(Debug, Default)]
|
||||
pub(crate) struct TestDb {
|
||||
files: Files,
|
||||
jars: JarsStorage<Self>,
|
||||
source: SourceJar,
|
||||
semantic: SemanticJar,
|
||||
}
|
||||
|
||||
impl HasJar<SourceJar> for TestDb {
|
||||
fn jar(&self) -> QueryResult<&SourceJar> {
|
||||
Ok(&self.jars()?.0)
|
||||
fn jar(&self) -> &SourceJar {
|
||||
&self.source
|
||||
}
|
||||
|
||||
fn jar_mut(&mut self) -> &mut SourceJar {
|
||||
&mut self.jars_mut().0
|
||||
&mut self.source
|
||||
}
|
||||
}
|
||||
|
||||
impl HasJar<SemanticJar> for TestDb {
|
||||
fn jar(&self) -> QueryResult<&SemanticJar> {
|
||||
Ok(&self.jars()?.1)
|
||||
fn jar(&self) -> &SemanticJar {
|
||||
&self.semantic
|
||||
}
|
||||
|
||||
fn jar_mut(&mut self) -> &mut SemanticJar {
|
||||
&mut self.jars_mut().1
|
||||
}
|
||||
}
|
||||
|
||||
impl HasJar<LintJar> for TestDb {
|
||||
fn jar(&self) -> QueryResult<&LintJar> {
|
||||
Ok(&self.jars()?.2)
|
||||
}
|
||||
|
||||
fn jar_mut(&mut self) -> &mut LintJar {
|
||||
&mut self.jars_mut().2
|
||||
&mut self.semantic
|
||||
}
|
||||
}
|
||||
|
||||
@@ -223,36 +137,44 @@ pub(crate) mod tests {
|
||||
self.files.path(file_id)
|
||||
}
|
||||
|
||||
fn source(&self, file_id: FileId) -> QueryResult<Source> {
|
||||
fn source(&self, file_id: FileId) -> Source {
|
||||
source_text(self, file_id)
|
||||
}
|
||||
|
||||
fn parse(&self, file_id: FileId) -> QueryResult<Parsed> {
|
||||
fn parse(&self, file_id: FileId) -> Parsed {
|
||||
parse(self, file_id)
|
||||
}
|
||||
|
||||
fn lint_syntax(&self, file_id: FileId) -> Diagnostics {
|
||||
lint_syntax(self, file_id)
|
||||
}
|
||||
}
|
||||
|
||||
impl SemanticDb for TestDb {
|
||||
fn resolve_module(&self, name: ModuleName) -> QueryResult<Option<Module>> {
|
||||
fn resolve_module(&self, name: ModuleName) -> Option<Module> {
|
||||
resolve_module(self, name)
|
||||
}
|
||||
|
||||
fn file_to_module(&self, file_id: FileId) -> QueryResult<Option<Module>> {
|
||||
fn file_to_module(&self, file_id: FileId) -> Option<Module> {
|
||||
file_to_module(self, file_id)
|
||||
}
|
||||
|
||||
fn path_to_module(&self, path: &Path) -> QueryResult<Option<Module>> {
|
||||
fn path_to_module(&self, path: &Path) -> Option<Module> {
|
||||
path_to_module(self, path)
|
||||
}
|
||||
|
||||
fn symbol_table(&self, file_id: FileId) -> QueryResult<Arc<SymbolTable>> {
|
||||
fn symbol_table(&self, file_id: FileId) -> Arc<SymbolTable> {
|
||||
symbol_table(self, file_id)
|
||||
}
|
||||
|
||||
fn infer_symbol_type(&self, file_id: FileId, symbol_id: SymbolId) -> QueryResult<Type> {
|
||||
fn infer_symbol_type(&self, file_id: FileId, symbol_id: SymbolId) -> Type {
|
||||
infer_symbol_type(self, file_id, symbol_id)
|
||||
}
|
||||
|
||||
fn lint_semantic(&self, file_id: FileId) -> Diagnostics {
|
||||
lint_semantic(self, file_id)
|
||||
}
|
||||
|
||||
fn add_module(&mut self, path: &Path) -> Option<(Module, Vec<Arc<ModuleData>>)> {
|
||||
add_module(self, path)
|
||||
}
|
||||
@@ -261,36 +183,4 @@ pub(crate) mod tests {
|
||||
set_module_search_paths(self, paths);
|
||||
}
|
||||
}
|
||||
|
||||
impl LintDb for TestDb {
|
||||
fn lint_syntax(&self, file_id: FileId) -> QueryResult<Diagnostics> {
|
||||
lint_syntax(self, file_id)
|
||||
}
|
||||
|
||||
fn lint_semantic(&self, file_id: FileId) -> QueryResult<Diagnostics> {
|
||||
lint_semantic(self, file_id)
|
||||
}
|
||||
}
|
||||
|
||||
impl HasJars for TestDb {
|
||||
type Jars = (SourceJar, SemanticJar, LintJar);
|
||||
|
||||
fn jars(&self) -> QueryResult<&Self::Jars> {
|
||||
self.jars.jars()
|
||||
}
|
||||
|
||||
fn jars_mut(&mut self) -> &mut Self::Jars {
|
||||
self.jars.jars_mut()
|
||||
}
|
||||
}
|
||||
|
||||
impl Database for TestDb {
|
||||
fn runtime(&self) -> &DbRuntime {
|
||||
self.jars.runtime()
|
||||
}
|
||||
|
||||
fn runtime_mut(&mut self) -> &mut DbRuntime {
|
||||
self.jars.runtime_mut()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,37 +0,0 @@
|
||||
use crate::db::query::QueryResult;
|
||||
|
||||
/// Gives access to a specific jar in the database.
|
||||
///
|
||||
/// Nope, the terminology isn't borrowed from Java but from Salsa <https://salsa-rs.github.io/salsa/>,
|
||||
/// which is an analogy to storing the salsa in different jars.
|
||||
///
|
||||
/// The basic idea is that each crate can define its own jar and the jars can be combined to a single
|
||||
/// database in the top level crate. Each crate also defines its own `Database` trait. The combination of
|
||||
/// `Database` trait and the jar allows to write queries in isolation without having to know how they get composed at the upper levels.
|
||||
///
|
||||
/// Salsa further defines a `HasIngredient` trait which slices the jar to a specific storage (e.g. a specific cache).
|
||||
/// We don't need this just jet because we write our queries by hand. We may want a similar trait if we decide
|
||||
/// to use a macro to generate the queries.
|
||||
pub trait HasJar<T> {
|
||||
/// Gives a read-only reference to the jar.
|
||||
fn jar(&self) -> QueryResult<&T>;
|
||||
|
||||
/// Gives a mutable reference to the jar.
|
||||
fn jar_mut(&mut self) -> &mut T;
|
||||
}
|
||||
|
||||
/// Gives access to the jars in a database.
|
||||
pub trait HasJars {
|
||||
/// A type storing the jars.
|
||||
///
|
||||
/// Most commonly, this is a tuple where each jar is a tuple element.
|
||||
type Jars: Default;
|
||||
|
||||
/// Gives access to the underlying jars but tests if the queries have been cancelled.
|
||||
///
|
||||
/// Returns `Err(QueryError::Cancelled)` if the queries have been cancelled.
|
||||
fn jars(&self) -> QueryResult<&Self::Jars>;
|
||||
|
||||
/// Gives mutable access to the underlying jars.
|
||||
fn jars_mut(&mut self) -> &mut Self::Jars;
|
||||
}
|
||||
@@ -1,20 +0,0 @@
|
||||
use std::fmt::{Display, Formatter};
|
||||
|
||||
/// Reason why a db query operation failed.
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub enum QueryError {
|
||||
/// The query was cancelled because the DB was mutated or the query was cancelled by the host (e.g. on a file change or when pressing CTRL+C).
|
||||
Cancelled,
|
||||
}
|
||||
|
||||
impl Display for QueryError {
|
||||
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
|
||||
match self {
|
||||
QueryError::Cancelled => f.write_str("query was cancelled"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for QueryError {}
|
||||
|
||||
pub type QueryResult<T> = Result<T, QueryError>;
|
||||
@@ -1,41 +0,0 @@
|
||||
use crate::cancellation::CancellationTokenSource;
|
||||
use crate::db::{QueryError, QueryResult};
|
||||
|
||||
/// Holds the jar agnostic state of the database.
|
||||
#[derive(Debug, Default)]
|
||||
pub struct DbRuntime {
|
||||
/// The cancellation token source used to signal other works that the queries should be aborted and
|
||||
/// exit at the next possible point.
|
||||
cancellation_token: CancellationTokenSource,
|
||||
}
|
||||
|
||||
impl DbRuntime {
|
||||
pub(super) fn snapshot(&self) -> Self {
|
||||
Self {
|
||||
cancellation_token: self.cancellation_token.clone(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Cancels the pending queries of other workers. The current worker cannot have any pending
|
||||
/// queries because we're holding a mutable reference to the runtime.
|
||||
pub(super) fn cancel_other_workers(&mut self) {
|
||||
self.cancellation_token.cancel();
|
||||
// Set a new cancellation token so that we're in a non-cancelled state again when running the next
|
||||
// query.
|
||||
self.cancellation_token = CancellationTokenSource::default();
|
||||
}
|
||||
|
||||
/// Returns `Ok` if the queries have not been cancelled and `Err(QueryError::Cancelled)` otherwise.
|
||||
pub(super) fn cancelled(&self) -> QueryResult<()> {
|
||||
if self.cancellation_token.is_cancelled() {
|
||||
Err(QueryError::Cancelled)
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns `true` if the queries have been cancelled.
|
||||
pub(super) fn is_cancelled(&self) -> bool {
|
||||
self.cancellation_token.is_cancelled()
|
||||
}
|
||||
}
|
||||
@@ -1,117 +0,0 @@
|
||||
use std::fmt::Formatter;
|
||||
use std::sync::Arc;
|
||||
|
||||
use crossbeam::sync::WaitGroup;
|
||||
|
||||
use crate::db::query::QueryResult;
|
||||
use crate::db::runtime::DbRuntime;
|
||||
use crate::db::{HasJars, ParallelDatabase};
|
||||
|
||||
/// Stores the jars of a database and the state for each worker.
|
||||
///
|
||||
/// Today, all state is shared across all workers, but it may be desired to store data per worker in the future.
|
||||
pub struct JarsStorage<T>
|
||||
where
|
||||
T: HasJars + Sized,
|
||||
{
|
||||
// It's important that `jars_wait_group` is declared after `jars` to ensure that `jars` is dropped first.
|
||||
// See https://doc.rust-lang.org/reference/destructors.html
|
||||
/// Stores the jars of the database.
|
||||
jars: Arc<T::Jars>,
|
||||
|
||||
/// Used to count the references to `jars`. Allows implementing `jars_mut` without requiring to clone `jars`.
|
||||
jars_wait_group: WaitGroup,
|
||||
|
||||
/// The data agnostic state.
|
||||
runtime: DbRuntime,
|
||||
}
|
||||
|
||||
impl<Db> JarsStorage<Db>
|
||||
where
|
||||
Db: HasJars,
|
||||
{
|
||||
pub(super) fn new() -> Self {
|
||||
Self {
|
||||
jars: Arc::new(Db::Jars::default()),
|
||||
jars_wait_group: WaitGroup::default(),
|
||||
runtime: DbRuntime::default(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Creates a snapshot of the jars.
|
||||
///
|
||||
/// Creating the snapshot is cheap because it doesn't clone the jars, it only increments a ref counter.
|
||||
#[must_use]
|
||||
pub fn snapshot(&self) -> JarsStorage<Db>
|
||||
where
|
||||
Db: ParallelDatabase,
|
||||
{
|
||||
Self {
|
||||
jars: self.jars.clone(),
|
||||
jars_wait_group: self.jars_wait_group.clone(),
|
||||
runtime: self.runtime.snapshot(),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn jars(&self) -> QueryResult<&Db::Jars> {
|
||||
self.runtime.cancelled()?;
|
||||
Ok(&self.jars)
|
||||
}
|
||||
|
||||
/// Returns a mutable reference to the jars without cloning their content.
|
||||
///
|
||||
/// The method cancels any pending queries of other works and waits for them to complete so that
|
||||
/// this instance is the only instance holding a reference to the jars.
|
||||
pub(crate) fn jars_mut(&mut self) -> &mut Db::Jars {
|
||||
// We have a mutable ref here, so no more workers can be spawned between calling this function and taking the mut ref below.
|
||||
self.cancel_other_workers();
|
||||
|
||||
// Now all other references to `self.jars` should have been released. We can now safely return a mutable reference
|
||||
// to the Arc's content.
|
||||
let jars =
|
||||
Arc::get_mut(&mut self.jars).expect("All references to jars should have been released");
|
||||
|
||||
jars
|
||||
}
|
||||
|
||||
pub(crate) fn runtime(&self) -> &DbRuntime {
|
||||
&self.runtime
|
||||
}
|
||||
|
||||
pub(crate) fn runtime_mut(&mut self) -> &mut DbRuntime {
|
||||
// Note: This method may need to use a similar trick to `jars_mut` if `DbRuntime` is ever to store data that is shared between workers.
|
||||
&mut self.runtime
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "trace", skip(self))]
|
||||
fn cancel_other_workers(&mut self) {
|
||||
self.runtime.cancel_other_workers();
|
||||
|
||||
// Wait for all other works to complete.
|
||||
let existing_wait = std::mem::take(&mut self.jars_wait_group);
|
||||
existing_wait.wait();
|
||||
}
|
||||
}
|
||||
|
||||
impl<Db> Default for JarsStorage<Db>
|
||||
where
|
||||
Db: HasJars,
|
||||
{
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
impl<T> std::fmt::Debug for JarsStorage<T>
|
||||
where
|
||||
T: HasJars,
|
||||
<T as HasJars>::Jars: std::fmt::Debug,
|
||||
{
|
||||
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
|
||||
f.debug_struct("SharedStorage")
|
||||
.field("jars", &self.jars)
|
||||
.field("jars_wait_group", &self.jars_wait_group)
|
||||
.field("runtime", &self.runtime)
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
@@ -1,135 +0,0 @@
|
||||
use std::ops::{Deref, DerefMut};
|
||||
|
||||
use ruff_formatter::PrintedRange;
|
||||
use ruff_python_formatter::{FormatModuleError, PyFormatOptions};
|
||||
use ruff_text_size::TextRange;
|
||||
|
||||
use crate::cache::KeyValueCache;
|
||||
use crate::db::{HasJar, QueryError, SourceDb};
|
||||
use crate::files::FileId;
|
||||
use crate::lint::Diagnostics;
|
||||
use crate::FxDashSet;
|
||||
|
||||
pub(crate) trait FormatDb: SourceDb {
|
||||
/// Formats a file and returns its formatted content or an indicator that it is unchanged.
|
||||
fn format_file(&self, file_id: FileId) -> Result<FormattedFile, FormatError>;
|
||||
|
||||
/// Formats a range in a file.
|
||||
fn format_file_range(
|
||||
&self,
|
||||
file_id: FileId,
|
||||
range: TextRange,
|
||||
) -> Result<PrintedRange, FormatError>;
|
||||
|
||||
fn check_file_formatted(&self, file_id: FileId) -> Result<Diagnostics, FormatError>;
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "trace", skip(db))]
|
||||
pub(crate) fn format_file<Db>(db: &Db, file_id: FileId) -> Result<FormattedFile, FormatError>
|
||||
where
|
||||
Db: FormatDb + HasJar<FormatJar>,
|
||||
{
|
||||
let formatted = &db.jar()?.formatted;
|
||||
|
||||
if formatted.contains(&file_id) {
|
||||
return Ok(FormattedFile::Unchanged);
|
||||
}
|
||||
|
||||
let source = db.source(file_id)?;
|
||||
|
||||
// TODO use the `format_module` method here to re-use the AST.
|
||||
let printed =
|
||||
ruff_python_formatter::format_module_source(source.text(), PyFormatOptions::default())?;
|
||||
|
||||
Ok(if printed.as_code() == source.text() {
|
||||
formatted.insert(file_id);
|
||||
FormattedFile::Unchanged
|
||||
} else {
|
||||
FormattedFile::Formatted(printed.into_code())
|
||||
})
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "trace", skip(db))]
|
||||
pub(crate) fn format_file_range<Db: FormatDb + HasJar<FormatJar>>(
|
||||
db: &Db,
|
||||
file_id: FileId,
|
||||
range: TextRange,
|
||||
) -> Result<PrintedRange, FormatError> {
|
||||
let formatted = &db.jar()?.formatted;
|
||||
let source = db.source(file_id)?;
|
||||
|
||||
if formatted.contains(&file_id) {
|
||||
return Ok(PrintedRange::new(source.text()[range].into(), range));
|
||||
}
|
||||
|
||||
// TODO use the `format_module` method here to re-use the AST.
|
||||
|
||||
let result =
|
||||
ruff_python_formatter::format_range(source.text(), range, PyFormatOptions::default())?;
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
/// Checks if the file is correctly formatted. It creates a diagnostic for formatting issues.
|
||||
#[tracing::instrument(level = "trace", skip(db))]
|
||||
pub(crate) fn check_formatted<Db>(db: &Db, file_id: FileId) -> Result<Diagnostics, FormatError>
|
||||
where
|
||||
Db: FormatDb + HasJar<FormatJar>,
|
||||
{
|
||||
Ok(if db.format_file(file_id)?.is_unchanged() {
|
||||
Diagnostics::Empty
|
||||
} else {
|
||||
Diagnostics::from(vec!["File is not formatted".to_string()])
|
||||
})
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub(crate) enum FormatError {
|
||||
Format(FormatModuleError),
|
||||
Query(QueryError),
|
||||
}
|
||||
|
||||
impl From<FormatModuleError> for FormatError {
|
||||
fn from(value: FormatModuleError) -> Self {
|
||||
Self::Format(value)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<QueryError> for FormatError {
|
||||
fn from(value: QueryError) -> Self {
|
||||
Self::Query(value)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Eq, PartialEq, Debug)]
|
||||
pub(crate) enum FormattedFile {
|
||||
Formatted(String),
|
||||
Unchanged,
|
||||
}
|
||||
|
||||
impl FormattedFile {
|
||||
pub(crate) const fn is_unchanged(&self) -> bool {
|
||||
matches!(self, FormattedFile::Unchanged)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
pub struct FormatJar {
|
||||
pub formatted: FxDashSet<FileId>,
|
||||
}
|
||||
|
||||
#[derive(Default, Debug)]
|
||||
pub(crate) struct FormattedStorage(KeyValueCache<FileId, ()>);
|
||||
|
||||
impl Deref for FormattedStorage {
|
||||
type Target = KeyValueCache<FileId, ()>;
|
||||
|
||||
fn deref(&self) -> &Self::Target {
|
||||
&self.0
|
||||
}
|
||||
}
|
||||
|
||||
impl DerefMut for FormattedStorage {
|
||||
fn deref_mut(&mut self) -> &mut Self::Target {
|
||||
&mut self.0
|
||||
}
|
||||
}
|
||||
@@ -12,7 +12,6 @@ pub mod cache;
|
||||
pub mod cancellation;
|
||||
pub mod db;
|
||||
pub mod files;
|
||||
mod format;
|
||||
pub mod hir;
|
||||
pub mod lint;
|
||||
pub mod module;
|
||||
@@ -28,7 +27,7 @@ pub(crate) type FxDashMap<K, V> = dashmap::DashMap<K, V, BuildHasherDefault<FxHa
|
||||
pub(crate) type FxDashSet<V> = dashmap::DashSet<V, BuildHasherDefault<FxHasher>>;
|
||||
pub(crate) type FxIndexSet<V> = indexmap::set::IndexSet<V, BuildHasherDefault<FxHasher>>;
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
#[derive(Debug)]
|
||||
pub struct Workspace {
|
||||
/// TODO this should be a resolved path. We should probably use a newtype wrapper that guarantees that
|
||||
/// PATH is a UTF-8 path and is normalized.
|
||||
|
||||
@@ -1,13 +1,12 @@
|
||||
use std::cell::RefCell;
|
||||
use std::ops::{Deref, DerefMut};
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use ruff_python_ast::visitor::Visitor;
|
||||
use ruff_python_ast::{ModModule, StringLiteral};
|
||||
|
||||
use crate::cache::KeyValueCache;
|
||||
use crate::db::{HasJar, LintDb, LintJar, QueryResult, SemanticDb};
|
||||
use crate::db::{HasJar, SemanticDb, SemanticJar, SourceDb, SourceJar};
|
||||
use crate::files::FileId;
|
||||
use crate::parse::Parsed;
|
||||
use crate::source::Source;
|
||||
@@ -15,28 +14,19 @@ use crate::symbols::{Definition, SymbolId, SymbolTable};
|
||||
use crate::types::Type;
|
||||
|
||||
#[tracing::instrument(level = "debug", skip(db))]
|
||||
pub(crate) fn lint_syntax<Db>(db: &Db, file_id: FileId) -> QueryResult<Diagnostics>
|
||||
pub(crate) fn lint_syntax<Db>(db: &Db, file_id: FileId) -> Diagnostics
|
||||
where
|
||||
Db: LintDb + HasJar<LintJar>,
|
||||
Db: SourceDb + HasJar<SourceJar>,
|
||||
{
|
||||
let storage = &db.jar()?.lint_syntax;
|
||||
|
||||
#[allow(clippy::print_stdout)]
|
||||
if std::env::var("RED_KNOT_SLOW_LINT").is_ok() {
|
||||
for i in 0..10 {
|
||||
db.cancelled()?;
|
||||
println!("RED_KNOT_SLOW_LINT is set, sleeping for {i}/10 seconds");
|
||||
std::thread::sleep(Duration::from_secs(1));
|
||||
}
|
||||
}
|
||||
let storage = &db.jar().lint_syntax;
|
||||
|
||||
storage.get(&file_id, |file_id| {
|
||||
let mut diagnostics = Vec::new();
|
||||
|
||||
let source = db.source(*file_id)?;
|
||||
let source = db.source(*file_id);
|
||||
lint_lines(source.text(), &mut diagnostics);
|
||||
|
||||
let parsed = db.parse(*file_id)?;
|
||||
let parsed = db.parse(*file_id);
|
||||
|
||||
if parsed.errors().is_empty() {
|
||||
let ast = parsed.ast();
|
||||
@@ -51,7 +41,7 @@ where
|
||||
diagnostics.extend(parsed.errors().iter().map(std::string::ToString::to_string));
|
||||
}
|
||||
|
||||
Ok(Diagnostics::from(diagnostics))
|
||||
Diagnostics::from(diagnostics)
|
||||
})
|
||||
}
|
||||
|
||||
@@ -73,16 +63,16 @@ fn lint_lines(source: &str, diagnostics: &mut Vec<String>) {
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "debug", skip(db))]
|
||||
pub(crate) fn lint_semantic<Db>(db: &Db, file_id: FileId) -> QueryResult<Diagnostics>
|
||||
pub(crate) fn lint_semantic<Db>(db: &Db, file_id: FileId) -> Diagnostics
|
||||
where
|
||||
Db: LintDb + HasJar<LintJar>,
|
||||
Db: SemanticDb + HasJar<SemanticJar>,
|
||||
{
|
||||
let storage = &db.jar()?.lint_semantic;
|
||||
let storage = &db.jar().lint_semantic;
|
||||
|
||||
storage.get(&file_id, |file_id| {
|
||||
let source = db.source(*file_id)?;
|
||||
let parsed = db.parse(*file_id)?;
|
||||
let symbols = db.symbol_table(*file_id)?;
|
||||
let source = db.source(*file_id);
|
||||
let parsed = db.parse(*file_id);
|
||||
let symbols = db.symbol_table(*file_id);
|
||||
|
||||
let context = SemanticLintContext {
|
||||
file_id: *file_id,
|
||||
@@ -93,25 +83,25 @@ where
|
||||
diagnostics: RefCell::new(Vec::new()),
|
||||
};
|
||||
|
||||
lint_unresolved_imports(&context)?;
|
||||
lint_unresolved_imports(&context);
|
||||
|
||||
Ok(Diagnostics::from(context.diagnostics.take()))
|
||||
Diagnostics::from(context.diagnostics.take())
|
||||
})
|
||||
}
|
||||
|
||||
fn lint_unresolved_imports(context: &SemanticLintContext) -> QueryResult<()> {
|
||||
fn lint_unresolved_imports(context: &SemanticLintContext) {
|
||||
// TODO: Consider iterating over the dependencies (imports) only instead of all definitions.
|
||||
for (symbol, definition) in context.symbols().all_definitions() {
|
||||
match definition {
|
||||
Definition::Import(import) => {
|
||||
let ty = context.infer_symbol_type(symbol)?;
|
||||
let ty = context.eval_symbol(symbol);
|
||||
|
||||
if ty.is_unknown() {
|
||||
context.push_diagnostic(format!("Unresolved module {}", import.module));
|
||||
}
|
||||
}
|
||||
Definition::ImportFrom(import) => {
|
||||
let ty = context.infer_symbol_type(symbol)?;
|
||||
let ty = context.eval_symbol(symbol);
|
||||
|
||||
if ty.is_unknown() {
|
||||
let module_name = import.module().map(Deref::deref).unwrap_or_default();
|
||||
@@ -136,8 +126,6 @@ fn lint_unresolved_imports(context: &SemanticLintContext) -> QueryResult<()> {
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub struct SemanticLintContext<'a> {
|
||||
@@ -166,7 +154,7 @@ impl<'a> SemanticLintContext<'a> {
|
||||
&self.symbols
|
||||
}
|
||||
|
||||
pub fn infer_symbol_type(&self, symbol_id: SymbolId) -> QueryResult<Type> {
|
||||
pub fn eval_symbol(&self, symbol_id: SymbolId) -> Type {
|
||||
self.db.infer_symbol_type(self.file_id, symbol_id)
|
||||
}
|
||||
|
||||
|
||||
@@ -4,7 +4,6 @@ use std::collections::hash_map::Entry;
|
||||
use std::path::Path;
|
||||
use std::sync::Mutex;
|
||||
|
||||
use crossbeam::channel as crossbeam_channel;
|
||||
use rustc_hash::FxHashMap;
|
||||
use tracing::subscriber::Interest;
|
||||
use tracing::{Level, Metadata};
|
||||
@@ -13,12 +12,11 @@ use tracing_subscriber::layer::{Context, Filter, SubscriberExt};
|
||||
use tracing_subscriber::{Layer, Registry};
|
||||
use tracing_tree::time::Uptime;
|
||||
|
||||
use red_knot::db::{
|
||||
Database, HasJar, ParallelDatabase, QueryError, SemanticDb, SourceDb, SourceJar,
|
||||
};
|
||||
use red_knot::cancellation::CancellationTokenSource;
|
||||
use red_knot::db::{HasJar, SourceDb, SourceJar};
|
||||
use red_knot::files::FileId;
|
||||
use red_knot::module::{ModuleSearchPath, ModuleSearchPathKind};
|
||||
use red_knot::program::check::ExecutionMode;
|
||||
use red_knot::program::check::{CheckError, RayonCheckScheduler};
|
||||
use red_knot::program::{FileChange, FileChangeKind, Program};
|
||||
use red_knot::watch::FileWatcher;
|
||||
use red_knot::Workspace;
|
||||
@@ -53,8 +51,7 @@ fn main() -> anyhow::Result<()> {
|
||||
workspace.root().to_path_buf(),
|
||||
ModuleSearchPathKind::FirstParty,
|
||||
);
|
||||
let mut program = Program::new(workspace);
|
||||
program.set_module_search_paths(vec![workspace_search_path]);
|
||||
let mut program = Program::new(workspace, vec![workspace_search_path]);
|
||||
|
||||
let entry_id = program.file_id(entry_point);
|
||||
program.workspace_mut().open_file(entry_id);
|
||||
@@ -85,7 +82,7 @@ fn main() -> anyhow::Result<()> {
|
||||
|
||||
main_loop.run(&mut program);
|
||||
|
||||
let source_jar: &SourceJar = program.jar().unwrap();
|
||||
let source_jar: &SourceJar = program.jar();
|
||||
|
||||
dbg!(source_jar.parsed.statistics());
|
||||
dbg!(source_jar.sources.statistics());
|
||||
@@ -104,9 +101,10 @@ impl MainLoop {
|
||||
let (main_loop_sender, main_loop_receiver) = crossbeam_channel::bounded(1);
|
||||
|
||||
let mut orchestrator = Orchestrator {
|
||||
pending_analysis: None,
|
||||
receiver: orchestrator_receiver,
|
||||
sender: main_loop_sender.clone(),
|
||||
revision: 0,
|
||||
aggregated_changes: AggregatedChanges::default(),
|
||||
};
|
||||
|
||||
std::thread::spawn(move || {
|
||||
@@ -139,32 +137,34 @@ impl MainLoop {
|
||||
tracing::trace!("Main Loop: Tick");
|
||||
|
||||
match message {
|
||||
MainLoopMessage::CheckProgram { revision } => {
|
||||
{
|
||||
let program = program.snapshot();
|
||||
let sender = self.orchestrator_sender.clone();
|
||||
MainLoopMessage::CheckProgram => {
|
||||
// Remove mutability from program.
|
||||
let program = &*program;
|
||||
let run_cancellation_token_source = CancellationTokenSource::new();
|
||||
let run_cancellation_token = run_cancellation_token_source.token();
|
||||
let sender = &self.orchestrator_sender;
|
||||
|
||||
// Spawn a new task that checks the program. This needs to be done in a separate thread
|
||||
// to prevent blocking the main loop here.
|
||||
rayon::spawn(move || match program.check(ExecutionMode::ThreadPool) {
|
||||
Ok(result) => {
|
||||
sender
|
||||
.send(OrchestratorMessage::CheckProgramCompleted {
|
||||
diagnostics: result,
|
||||
revision,
|
||||
})
|
||||
.unwrap();
|
||||
}
|
||||
Err(QueryError::Cancelled) => {}
|
||||
});
|
||||
}
|
||||
sender
|
||||
.send(OrchestratorMessage::CheckProgramStarted {
|
||||
cancellation_token: run_cancellation_token_source,
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
if !program.is_cancelled() {
|
||||
let _ = program.format();
|
||||
}
|
||||
rayon::in_place_scope(|scope| {
|
||||
let scheduler = RayonCheckScheduler::new(program, scope);
|
||||
|
||||
let result = program.check(&scheduler, run_cancellation_token);
|
||||
match result {
|
||||
Ok(result) => sender
|
||||
.send(OrchestratorMessage::CheckProgramCompleted(result))
|
||||
.unwrap(),
|
||||
Err(CheckError::Cancelled) => sender
|
||||
.send(OrchestratorMessage::CheckProgramCancelled)
|
||||
.unwrap(),
|
||||
}
|
||||
});
|
||||
}
|
||||
MainLoopMessage::ApplyChanges(changes) => {
|
||||
// Automatically cancels any pending queries and waits for them to complete.
|
||||
program.apply_changes(changes.iter());
|
||||
}
|
||||
MainLoopMessage::CheckCompleted(diagnostics) => {
|
||||
@@ -211,11 +211,13 @@ impl MainLoopCancellationToken {
|
||||
}
|
||||
|
||||
struct Orchestrator {
|
||||
aggregated_changes: AggregatedChanges,
|
||||
pending_analysis: Option<PendingAnalysisState>,
|
||||
|
||||
/// Sends messages to the main loop.
|
||||
sender: crossbeam_channel::Sender<MainLoopMessage>,
|
||||
/// Receives messages from the main loop.
|
||||
receiver: crossbeam_channel::Receiver<OrchestratorMessage>,
|
||||
revision: usize,
|
||||
}
|
||||
|
||||
impl Orchestrator {
|
||||
@@ -223,33 +225,51 @@ impl Orchestrator {
|
||||
while let Ok(message) = self.receiver.recv() {
|
||||
match message {
|
||||
OrchestratorMessage::Run => {
|
||||
self.pending_analysis = None;
|
||||
self.sender.send(MainLoopMessage::CheckProgram).unwrap();
|
||||
}
|
||||
|
||||
OrchestratorMessage::CheckProgramStarted { cancellation_token } => {
|
||||
debug_assert!(self.pending_analysis.is_none());
|
||||
|
||||
self.pending_analysis = Some(PendingAnalysisState { cancellation_token });
|
||||
}
|
||||
|
||||
OrchestratorMessage::CheckProgramCompleted(diagnostics) => {
|
||||
self.pending_analysis
|
||||
.take()
|
||||
.expect("Expected a pending analysis.");
|
||||
|
||||
self.sender
|
||||
.send(MainLoopMessage::CheckProgram {
|
||||
revision: self.revision,
|
||||
})
|
||||
.send(MainLoopMessage::CheckCompleted(diagnostics))
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
OrchestratorMessage::CheckProgramCompleted {
|
||||
diagnostics,
|
||||
revision,
|
||||
} => {
|
||||
// Only take the diagnostics if they are for the latest revision.
|
||||
if self.revision == revision {
|
||||
self.sender
|
||||
.send(MainLoopMessage::CheckCompleted(diagnostics))
|
||||
.unwrap();
|
||||
} else {
|
||||
tracing::debug!("Discarding diagnostics for outdated revision {revision} (current: {}).", self.revision);
|
||||
}
|
||||
OrchestratorMessage::CheckProgramCancelled => {
|
||||
self.pending_analysis
|
||||
.take()
|
||||
.expect("Expected a pending analysis.");
|
||||
|
||||
self.debounce_changes();
|
||||
}
|
||||
|
||||
OrchestratorMessage::FileChanges(changes) => {
|
||||
// Request cancellation, but wait until all analysis tasks have completed to
|
||||
// avoid stale messages in the next main loop.
|
||||
let pending = if let Some(pending_state) = self.pending_analysis.as_ref() {
|
||||
pending_state.cancellation_token.cancel();
|
||||
true
|
||||
} else {
|
||||
false
|
||||
};
|
||||
|
||||
self.revision += 1;
|
||||
self.debounce_changes(changes);
|
||||
self.aggregated_changes.extend(changes);
|
||||
|
||||
// If there are no pending analysis tasks, apply the file changes. Otherwise
|
||||
// keep running until all file checks have completed.
|
||||
if !pending {
|
||||
self.debounce_changes();
|
||||
}
|
||||
}
|
||||
OrchestratorMessage::Shutdown => {
|
||||
return self.shutdown();
|
||||
@@ -258,9 +278,8 @@ impl Orchestrator {
|
||||
}
|
||||
}
|
||||
|
||||
fn debounce_changes(&self, changes: Vec<FileChange>) {
|
||||
let mut aggregated_changes = AggregatedChanges::default();
|
||||
aggregated_changes.extend(changes);
|
||||
fn debounce_changes(&mut self) {
|
||||
debug_assert!(self.pending_analysis.is_none());
|
||||
|
||||
loop {
|
||||
// Consume possibly incoming file change messages before running a new analysis, but don't wait for more than 100ms.
|
||||
@@ -271,12 +290,10 @@ impl Orchestrator {
|
||||
return self.shutdown();
|
||||
}
|
||||
Ok(OrchestratorMessage::FileChanges(file_changes)) => {
|
||||
aggregated_changes.extend(file_changes);
|
||||
self.aggregated_changes.extend(file_changes);
|
||||
}
|
||||
|
||||
Ok(OrchestratorMessage::CheckProgramCompleted { .. })=> {
|
||||
// disregard any outdated completion message.
|
||||
}
|
||||
Ok(OrchestratorMessage::CheckProgramStarted {..}| OrchestratorMessage::CheckProgramCompleted(_) | OrchestratorMessage::CheckProgramCancelled) => unreachable!("No program check should be running while debouncing changes."),
|
||||
Ok(OrchestratorMessage::Run) => unreachable!("The orchestrator is already running."),
|
||||
|
||||
Err(_) => {
|
||||
@@ -285,10 +302,10 @@ impl Orchestrator {
|
||||
}
|
||||
}
|
||||
},
|
||||
default(std::time::Duration::from_millis(10)) => {
|
||||
// No more file changes after 10 ms, send the changes and schedule a new analysis
|
||||
self.sender.send(MainLoopMessage::ApplyChanges(aggregated_changes)).unwrap();
|
||||
self.sender.send(MainLoopMessage::CheckProgram { revision: self.revision}).unwrap();
|
||||
default(std::time::Duration::from_millis(100)) => {
|
||||
// No more file changes after 100 ms, send the changes and schedule a new analysis
|
||||
self.sender.send(MainLoopMessage::ApplyChanges(std::mem::take(&mut self.aggregated_changes))).unwrap();
|
||||
self.sender.send(MainLoopMessage::CheckProgram).unwrap();
|
||||
return;
|
||||
}
|
||||
}
|
||||
@@ -301,10 +318,15 @@ impl Orchestrator {
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct PendingAnalysisState {
|
||||
cancellation_token: CancellationTokenSource,
|
||||
}
|
||||
|
||||
/// Message sent from the orchestrator to the main loop.
|
||||
#[derive(Debug)]
|
||||
enum MainLoopMessage {
|
||||
CheckProgram { revision: usize },
|
||||
CheckProgram,
|
||||
CheckCompleted(Vec<String>),
|
||||
ApplyChanges(AggregatedChanges),
|
||||
Exit,
|
||||
@@ -315,10 +337,11 @@ enum OrchestratorMessage {
|
||||
Run,
|
||||
Shutdown,
|
||||
|
||||
CheckProgramCompleted {
|
||||
diagnostics: Vec<String>,
|
||||
revision: usize,
|
||||
CheckProgramStarted {
|
||||
cancellation_token: CancellationTokenSource,
|
||||
},
|
||||
CheckProgramCompleted(Vec<String>),
|
||||
CheckProgramCancelled,
|
||||
|
||||
FileChanges(Vec<FileChange>),
|
||||
}
|
||||
|
||||
@@ -7,7 +7,7 @@ use std::sync::Arc;
|
||||
use dashmap::mapref::entry::Entry;
|
||||
use smol_str::SmolStr;
|
||||
|
||||
use crate::db::{HasJar, QueryResult, SemanticDb, SemanticJar};
|
||||
use crate::db::{HasJar, SemanticDb, SemanticJar};
|
||||
use crate::files::FileId;
|
||||
use crate::symbols::Dependency;
|
||||
use crate::FxDashMap;
|
||||
@@ -17,48 +17,44 @@ use crate::FxDashMap;
|
||||
pub struct Module(u32);
|
||||
|
||||
impl Module {
|
||||
pub fn name<Db>(&self, db: &Db) -> QueryResult<ModuleName>
|
||||
pub fn name<Db>(&self, db: &Db) -> ModuleName
|
||||
where
|
||||
Db: HasJar<SemanticJar>,
|
||||
{
|
||||
let modules = &db.jar()?.module_resolver;
|
||||
let modules = &db.jar().module_resolver;
|
||||
|
||||
Ok(modules.modules.get(self).unwrap().name.clone())
|
||||
modules.modules.get(self).unwrap().name.clone()
|
||||
}
|
||||
|
||||
pub fn path<Db>(&self, db: &Db) -> QueryResult<ModulePath>
|
||||
pub fn path<Db>(&self, db: &Db) -> ModulePath
|
||||
where
|
||||
Db: HasJar<SemanticJar>,
|
||||
{
|
||||
let modules = &db.jar()?.module_resolver;
|
||||
let modules = &db.jar().module_resolver;
|
||||
|
||||
Ok(modules.modules.get(self).unwrap().path.clone())
|
||||
modules.modules.get(self).unwrap().path.clone()
|
||||
}
|
||||
|
||||
pub fn kind<Db>(&self, db: &Db) -> QueryResult<ModuleKind>
|
||||
pub fn kind<Db>(&self, db: &Db) -> ModuleKind
|
||||
where
|
||||
Db: HasJar<SemanticJar>,
|
||||
{
|
||||
let modules = &db.jar()?.module_resolver;
|
||||
let modules = &db.jar().module_resolver;
|
||||
|
||||
Ok(modules.modules.get(self).unwrap().kind)
|
||||
modules.modules.get(self).unwrap().kind
|
||||
}
|
||||
|
||||
pub fn resolve_dependency<Db>(
|
||||
&self,
|
||||
db: &Db,
|
||||
dependency: &Dependency,
|
||||
) -> QueryResult<Option<ModuleName>>
|
||||
pub fn resolve_dependency<Db>(&self, db: &Db, dependency: &Dependency) -> Option<ModuleName>
|
||||
where
|
||||
Db: HasJar<SemanticJar>,
|
||||
{
|
||||
let (level, module) = match dependency {
|
||||
Dependency::Module(module) => return Ok(Some(module.clone())),
|
||||
Dependency::Module(module) => return Some(module.clone()),
|
||||
Dependency::Relative { level, module } => (*level, module.as_deref()),
|
||||
};
|
||||
|
||||
let name = self.name(db)?;
|
||||
let kind = self.kind(db)?;
|
||||
let name = self.name(db);
|
||||
let kind = self.kind(db);
|
||||
|
||||
let mut components = name.components().peekable();
|
||||
|
||||
@@ -71,9 +67,7 @@ impl Module {
|
||||
|
||||
// Skip over the relative parts.
|
||||
for _ in start..level.get() {
|
||||
if components.next_back().is_none() {
|
||||
return Ok(None);
|
||||
}
|
||||
components.next_back()?;
|
||||
}
|
||||
|
||||
let mut name = String::new();
|
||||
@@ -86,11 +80,11 @@ impl Module {
|
||||
name.push_str(part);
|
||||
}
|
||||
|
||||
Ok(if name.is_empty() {
|
||||
if name.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(ModuleName(SmolStr::new(name)))
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -244,25 +238,20 @@ pub struct ModuleData {
|
||||
/// TODO: This would not work with Salsa because `ModuleName` isn't an ingredient and, therefore, cannot be used as part of a query.
|
||||
/// For this to work with salsa, it would be necessary to intern all `ModuleName`s.
|
||||
#[tracing::instrument(level = "debug", skip(db))]
|
||||
pub fn resolve_module<Db>(db: &Db, name: ModuleName) -> QueryResult<Option<Module>>
|
||||
pub fn resolve_module<Db>(db: &Db, name: ModuleName) -> Option<Module>
|
||||
where
|
||||
Db: SemanticDb + HasJar<SemanticJar>,
|
||||
{
|
||||
let jar = db.jar();
|
||||
let modules = &jar?.module_resolver;
|
||||
let modules = &jar.module_resolver;
|
||||
|
||||
let entry = modules.by_name.entry(name.clone());
|
||||
|
||||
match entry {
|
||||
Entry::Occupied(entry) => Ok(Some(*entry.get())),
|
||||
Entry::Occupied(entry) => Some(*entry.get()),
|
||||
Entry::Vacant(entry) => {
|
||||
let Some((root_path, absolute_path, kind)) = resolve_name(&name, &modules.search_paths)
|
||||
else {
|
||||
return Ok(None);
|
||||
};
|
||||
let Ok(normalized) = absolute_path.canonicalize() else {
|
||||
return Ok(None);
|
||||
};
|
||||
let (root_path, absolute_path, kind) = resolve_name(&name, &modules.search_paths)?;
|
||||
let normalized = absolute_path.canonicalize().ok()?;
|
||||
|
||||
let file_id = db.file_id(&normalized);
|
||||
let path = ModulePath::new(root_path.clone(), file_id);
|
||||
@@ -284,76 +273,59 @@ where
|
||||
// ```
|
||||
// Here, both `foo` and `bar` resolve to the same module but through different paths.
|
||||
// That's why we need to insert the absolute path and not the normalized path here.
|
||||
let absolute_id = if absolute_path == normalized {
|
||||
file_id
|
||||
} else {
|
||||
db.file_id(&absolute_path)
|
||||
};
|
||||
|
||||
modules.by_file.insert(absolute_id, id);
|
||||
modules.by_path.insert(absolute_path, id);
|
||||
|
||||
entry.insert_entry(id);
|
||||
|
||||
Ok(Some(id))
|
||||
Some(id)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Resolves the module id for the given path.
|
||||
///
|
||||
/// Returns `None` if the path is not a module in `sys.path`.
|
||||
#[tracing::instrument(level = "debug", skip(db))]
|
||||
pub fn path_to_module<Db>(db: &Db, path: &Path) -> QueryResult<Option<Module>>
|
||||
where
|
||||
Db: SemanticDb + HasJar<SemanticJar>,
|
||||
{
|
||||
let file = db.file_id(path);
|
||||
file_to_module(db, file)
|
||||
}
|
||||
|
||||
/// Resolves the module id for the file with the given id.
|
||||
///
|
||||
/// Returns `None` if the file is not a module in `sys.path`.
|
||||
#[tracing::instrument(level = "debug", skip(db))]
|
||||
pub fn file_to_module<Db>(db: &Db, file: FileId) -> QueryResult<Option<Module>>
|
||||
pub fn file_to_module<Db>(db: &Db, file: FileId) -> Option<Module>
|
||||
where
|
||||
Db: SemanticDb + HasJar<SemanticJar>,
|
||||
{
|
||||
let jar = db.jar()?;
|
||||
let modules = &jar.module_resolver;
|
||||
|
||||
if let Some(existing) = modules.by_file.get(&file) {
|
||||
return Ok(Some(*existing));
|
||||
}
|
||||
|
||||
let path = db.file_path(file);
|
||||
path_to_module(db, &path)
|
||||
}
|
||||
|
||||
/// Resolves the module id for the given path.
|
||||
///
|
||||
/// Returns `None` if the path is not a module in `sys.path`.
|
||||
#[tracing::instrument(level = "debug", skip(db))]
|
||||
pub fn path_to_module<Db>(db: &Db, path: &Path) -> Option<Module>
|
||||
where
|
||||
Db: SemanticDb + HasJar<SemanticJar>,
|
||||
{
|
||||
let jar = db.jar();
|
||||
let modules = &jar.module_resolver;
|
||||
debug_assert!(path.is_absolute());
|
||||
|
||||
let Some((root_path, relative_path)) = modules.search_paths.iter().find_map(|root| {
|
||||
if let Some(existing) = modules.by_path.get(path) {
|
||||
return Some(*existing);
|
||||
}
|
||||
|
||||
let (root_path, relative_path) = modules.search_paths.iter().find_map(|root| {
|
||||
let relative_path = path.strip_prefix(root.path()).ok()?;
|
||||
Some((root.clone(), relative_path))
|
||||
}) else {
|
||||
return Ok(None);
|
||||
};
|
||||
})?;
|
||||
|
||||
let Some(module_name) = ModuleName::from_relative_path(relative_path) else {
|
||||
return Ok(None);
|
||||
};
|
||||
let module_name = ModuleName::from_relative_path(relative_path)?;
|
||||
|
||||
// Resolve the module name to see if Python would resolve the name to the same path.
|
||||
// If it doesn't, then that means that multiple modules have the same in different
|
||||
// root paths, but that the module corresponding to the past path is in a lower priority path,
|
||||
// in which case we ignore it.
|
||||
let Some(module_id) = resolve_module(db, module_name)? else {
|
||||
return Ok(None);
|
||||
};
|
||||
let module_path = module_id.path(db)?;
|
||||
let module_id = resolve_module(db, module_name)?;
|
||||
let module_path = module_id.path(db);
|
||||
|
||||
if module_path.root() == &root_path {
|
||||
let Ok(normalized) = path.canonicalize() else {
|
||||
return Ok(None);
|
||||
};
|
||||
let normalized = path.canonicalize().ok()?;
|
||||
let interned_normalized = db.file_id(&normalized);
|
||||
|
||||
if interned_normalized != module_path.file() {
|
||||
@@ -364,15 +336,15 @@ where
|
||||
// ```
|
||||
// The module name of `src/foo.py` is `foo`, but the module loaded by Python is `src/foo/__init__.py`.
|
||||
// That means we need to ignore `src/foo.py` even though it resolves to the same module name.
|
||||
return Ok(None);
|
||||
return None;
|
||||
}
|
||||
|
||||
// Path has been inserted by `resolved`
|
||||
Ok(Some(module_id))
|
||||
Some(module_id)
|
||||
} else {
|
||||
// This path is for a module with the same name but in a module search path with a lower priority.
|
||||
// Ignore it.
|
||||
Ok(None)
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
@@ -406,7 +378,7 @@ where
|
||||
// TODO This needs tests
|
||||
|
||||
// Note: Intentionally by-pass caching here. Module should not be in the cache yet.
|
||||
let module = path_to_module(db, path).ok()??;
|
||||
let module = path_to_module(db, path)?;
|
||||
|
||||
// The code below is to handle the addition of `__init__.py` files.
|
||||
// When an `__init__.py` file is added, we need to remove all modules that are part of the same package.
|
||||
@@ -420,7 +392,7 @@ where
|
||||
return Some((module, Vec::new()));
|
||||
}
|
||||
|
||||
let Some(parent_name) = module.name(db).ok()?.parent() else {
|
||||
let Some(parent_name) = module.name(db).parent() else {
|
||||
return Some((module, Vec::new()));
|
||||
};
|
||||
|
||||
@@ -429,7 +401,7 @@ where
|
||||
let jar = db.jar_mut();
|
||||
let modules = &mut jar.module_resolver;
|
||||
|
||||
modules.by_file.retain(|_, id| {
|
||||
modules.by_path.retain(|_, id| {
|
||||
if modules
|
||||
.modules
|
||||
.get(id)
|
||||
@@ -468,7 +440,7 @@ pub struct ModuleResolver {
|
||||
|
||||
/// Lookup from absolute path to module.
|
||||
/// The same module might be reachable from different paths when symlinks are involved.
|
||||
by_file: FxDashMap<FileId, Module>,
|
||||
by_path: FxDashMap<PathBuf, Module>,
|
||||
next_module_id: AtomicU32,
|
||||
}
|
||||
|
||||
@@ -478,14 +450,14 @@ impl ModuleResolver {
|
||||
search_paths,
|
||||
modules: FxDashMap::default(),
|
||||
by_name: FxDashMap::default(),
|
||||
by_file: FxDashMap::default(),
|
||||
by_path: FxDashMap::default(),
|
||||
next_module_id: AtomicU32::new(0),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn remove_module(&mut self, file_id: FileId) {
|
||||
pub fn remove_module(&mut self, path: &Path) {
|
||||
// No locking is required because we're holding a mutable reference to `self`.
|
||||
let Some((_, id)) = self.by_file.remove(&file_id) else {
|
||||
let Some((_, id)) = self.by_path.remove(path) else {
|
||||
return;
|
||||
};
|
||||
|
||||
@@ -498,7 +470,7 @@ impl ModuleResolver {
|
||||
self.by_name.remove(&module.name).unwrap();
|
||||
|
||||
// It's possible that multiple paths map to the same id. Search all other paths referencing the same module id.
|
||||
self.by_file.retain(|_, current_id| *current_id != id);
|
||||
self.by_path.retain(|_, current_id| *current_id != id);
|
||||
|
||||
module
|
||||
}
|
||||
@@ -719,7 +691,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn first_party_module() -> anyhow::Result<()> {
|
||||
fn first_party_module() -> std::io::Result<()> {
|
||||
let TestCase {
|
||||
db,
|
||||
src,
|
||||
@@ -730,22 +702,22 @@ mod tests {
|
||||
let foo_path = src.path().join("foo.py");
|
||||
std::fs::write(&foo_path, "print('Hello, world!')")?;
|
||||
|
||||
let foo_module = db.resolve_module(ModuleName::new("foo"))?.unwrap();
|
||||
let foo_module = db.resolve_module(ModuleName::new("foo")).unwrap();
|
||||
|
||||
assert_eq!(Some(foo_module), db.resolve_module(ModuleName::new("foo"))?);
|
||||
assert_eq!(Some(foo_module), db.resolve_module(ModuleName::new("foo")));
|
||||
|
||||
assert_eq!(ModuleName::new("foo"), foo_module.name(&db)?);
|
||||
assert_eq!(&src, foo_module.path(&db)?.root());
|
||||
assert_eq!(ModuleKind::Module, foo_module.kind(&db)?);
|
||||
assert_eq!(&foo_path, &*db.file_path(foo_module.path(&db)?.file()));
|
||||
assert_eq!(ModuleName::new("foo"), foo_module.name(&db));
|
||||
assert_eq!(&src, foo_module.path(&db).root());
|
||||
assert_eq!(ModuleKind::Module, foo_module.kind(&db));
|
||||
assert_eq!(&foo_path, &*db.file_path(foo_module.path(&db).file()));
|
||||
|
||||
assert_eq!(Some(foo_module), db.path_to_module(&foo_path)?);
|
||||
assert_eq!(Some(foo_module), db.path_to_module(&foo_path));
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_package() -> anyhow::Result<()> {
|
||||
fn resolve_package() -> std::io::Result<()> {
|
||||
let TestCase {
|
||||
src,
|
||||
db,
|
||||
@@ -758,22 +730,22 @@ mod tests {
|
||||
std::fs::create_dir(&foo_dir)?;
|
||||
std::fs::write(&foo_path, "print('Hello, world!')")?;
|
||||
|
||||
let foo_module = db.resolve_module(ModuleName::new("foo"))?.unwrap();
|
||||
let foo_module = db.resolve_module(ModuleName::new("foo")).unwrap();
|
||||
|
||||
assert_eq!(ModuleName::new("foo"), foo_module.name(&db)?);
|
||||
assert_eq!(&src, foo_module.path(&db)?.root());
|
||||
assert_eq!(&foo_path, &*db.file_path(foo_module.path(&db)?.file()));
|
||||
assert_eq!(ModuleName::new("foo"), foo_module.name(&db));
|
||||
assert_eq!(&src, foo_module.path(&db).root());
|
||||
assert_eq!(&foo_path, &*db.file_path(foo_module.path(&db).file()));
|
||||
|
||||
assert_eq!(Some(foo_module), db.path_to_module(&foo_path)?);
|
||||
assert_eq!(Some(foo_module), db.path_to_module(&foo_path));
|
||||
|
||||
// Resolving by directory doesn't resolve to the init file.
|
||||
assert_eq!(None, db.path_to_module(&foo_dir)?);
|
||||
assert_eq!(None, db.path_to_module(&foo_dir));
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn package_priority_over_module() -> anyhow::Result<()> {
|
||||
fn package_priority_over_module() -> std::io::Result<()> {
|
||||
let TestCase {
|
||||
db,
|
||||
temp_dir: _temp_dir,
|
||||
@@ -789,20 +761,20 @@ mod tests {
|
||||
let foo_py = src.path().join("foo.py");
|
||||
std::fs::write(&foo_py, "print('Hello, world!')")?;
|
||||
|
||||
let foo_module = db.resolve_module(ModuleName::new("foo"))?.unwrap();
|
||||
let foo_module = db.resolve_module(ModuleName::new("foo")).unwrap();
|
||||
|
||||
assert_eq!(&src, foo_module.path(&db)?.root());
|
||||
assert_eq!(&foo_init, &*db.file_path(foo_module.path(&db)?.file()));
|
||||
assert_eq!(ModuleKind::Package, foo_module.kind(&db)?);
|
||||
assert_eq!(&src, foo_module.path(&db).root());
|
||||
assert_eq!(&foo_init, &*db.file_path(foo_module.path(&db).file()));
|
||||
assert_eq!(ModuleKind::Package, foo_module.kind(&db));
|
||||
|
||||
assert_eq!(Some(foo_module), db.path_to_module(&foo_init)?);
|
||||
assert_eq!(None, db.path_to_module(&foo_py)?);
|
||||
assert_eq!(Some(foo_module), db.path_to_module(&foo_init));
|
||||
assert_eq!(None, db.path_to_module(&foo_py));
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn typing_stub_over_module() -> anyhow::Result<()> {
|
||||
fn typing_stub_over_module() -> std::io::Result<()> {
|
||||
let TestCase {
|
||||
db,
|
||||
src,
|
||||
@@ -815,19 +787,19 @@ mod tests {
|
||||
std::fs::write(&foo_stub, "x: int")?;
|
||||
std::fs::write(&foo_py, "print('Hello, world!')")?;
|
||||
|
||||
let foo = db.resolve_module(ModuleName::new("foo"))?.unwrap();
|
||||
let foo = db.resolve_module(ModuleName::new("foo")).unwrap();
|
||||
|
||||
assert_eq!(&src, foo.path(&db)?.root());
|
||||
assert_eq!(&foo_stub, &*db.file_path(foo.path(&db)?.file()));
|
||||
assert_eq!(&src, foo.path(&db).root());
|
||||
assert_eq!(&foo_stub, &*db.file_path(foo.path(&db).file()));
|
||||
|
||||
assert_eq!(Some(foo), db.path_to_module(&foo_stub)?);
|
||||
assert_eq!(None, db.path_to_module(&foo_py)?);
|
||||
assert_eq!(Some(foo), db.path_to_module(&foo_stub));
|
||||
assert_eq!(None, db.path_to_module(&foo_py));
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sub_packages() -> anyhow::Result<()> {
|
||||
fn sub_packages() -> std::io::Result<()> {
|
||||
let TestCase {
|
||||
db,
|
||||
src,
|
||||
@@ -844,18 +816,18 @@ mod tests {
|
||||
std::fs::write(bar.join("__init__.py"), "")?;
|
||||
std::fs::write(&baz, "print('Hello, world!')")?;
|
||||
|
||||
let baz_module = db.resolve_module(ModuleName::new("foo.bar.baz"))?.unwrap();
|
||||
let baz_module = db.resolve_module(ModuleName::new("foo.bar.baz")).unwrap();
|
||||
|
||||
assert_eq!(&src, baz_module.path(&db)?.root());
|
||||
assert_eq!(&baz, &*db.file_path(baz_module.path(&db)?.file()));
|
||||
assert_eq!(&src, baz_module.path(&db).root());
|
||||
assert_eq!(&baz, &*db.file_path(baz_module.path(&db).file()));
|
||||
|
||||
assert_eq!(Some(baz_module), db.path_to_module(&baz)?);
|
||||
assert_eq!(Some(baz_module), db.path_to_module(&baz));
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn namespace_package() -> anyhow::Result<()> {
|
||||
fn namespace_package() -> std::io::Result<()> {
|
||||
let TestCase {
|
||||
db,
|
||||
temp_dir: _,
|
||||
@@ -891,21 +863,21 @@ mod tests {
|
||||
std::fs::write(&two, "print('Hello, world!')")?;
|
||||
|
||||
let one_module = db
|
||||
.resolve_module(ModuleName::new("parent.child.one"))?
|
||||
.resolve_module(ModuleName::new("parent.child.one"))
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(Some(one_module), db.path_to_module(&one)?);
|
||||
assert_eq!(Some(one_module), db.path_to_module(&one));
|
||||
|
||||
let two_module = db
|
||||
.resolve_module(ModuleName::new("parent.child.two"))?
|
||||
.resolve_module(ModuleName::new("parent.child.two"))
|
||||
.unwrap();
|
||||
assert_eq!(Some(two_module), db.path_to_module(&two)?);
|
||||
assert_eq!(Some(two_module), db.path_to_module(&two));
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn regular_package_in_namespace_package() -> anyhow::Result<()> {
|
||||
fn regular_package_in_namespace_package() -> std::io::Result<()> {
|
||||
let TestCase {
|
||||
db,
|
||||
temp_dir: _,
|
||||
@@ -942,20 +914,17 @@ mod tests {
|
||||
std::fs::write(two, "print('Hello, world!')")?;
|
||||
|
||||
let one_module = db
|
||||
.resolve_module(ModuleName::new("parent.child.one"))?
|
||||
.resolve_module(ModuleName::new("parent.child.one"))
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(Some(one_module), db.path_to_module(&one)?);
|
||||
assert_eq!(Some(one_module), db.path_to_module(&one));
|
||||
|
||||
assert_eq!(
|
||||
None,
|
||||
db.resolve_module(ModuleName::new("parent.child.two"))?
|
||||
);
|
||||
assert_eq!(None, db.resolve_module(ModuleName::new("parent.child.two")));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn module_search_path_priority() -> anyhow::Result<()> {
|
||||
fn module_search_path_priority() -> std::io::Result<()> {
|
||||
let TestCase {
|
||||
db,
|
||||
src,
|
||||
@@ -969,20 +938,20 @@ mod tests {
|
||||
std::fs::write(&foo_src, "")?;
|
||||
std::fs::write(&foo_site_packages, "")?;
|
||||
|
||||
let foo_module = db.resolve_module(ModuleName::new("foo"))?.unwrap();
|
||||
let foo_module = db.resolve_module(ModuleName::new("foo")).unwrap();
|
||||
|
||||
assert_eq!(&src, foo_module.path(&db)?.root());
|
||||
assert_eq!(&foo_src, &*db.file_path(foo_module.path(&db)?.file()));
|
||||
assert_eq!(&src, foo_module.path(&db).root());
|
||||
assert_eq!(&foo_src, &*db.file_path(foo_module.path(&db).file()));
|
||||
|
||||
assert_eq!(Some(foo_module), db.path_to_module(&foo_src)?);
|
||||
assert_eq!(None, db.path_to_module(&foo_site_packages)?);
|
||||
assert_eq!(Some(foo_module), db.path_to_module(&foo_src));
|
||||
assert_eq!(None, db.path_to_module(&foo_site_packages));
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[cfg(target_family = "unix")]
|
||||
fn symlink() -> anyhow::Result<()> {
|
||||
fn symlink() -> std::io::Result<()> {
|
||||
let TestCase {
|
||||
db,
|
||||
src,
|
||||
@@ -996,28 +965,28 @@ mod tests {
|
||||
std::fs::write(&foo, "")?;
|
||||
std::os::unix::fs::symlink(&foo, &bar)?;
|
||||
|
||||
let foo_module = db.resolve_module(ModuleName::new("foo"))?.unwrap();
|
||||
let bar_module = db.resolve_module(ModuleName::new("bar"))?.unwrap();
|
||||
let foo_module = db.resolve_module(ModuleName::new("foo")).unwrap();
|
||||
let bar_module = db.resolve_module(ModuleName::new("bar")).unwrap();
|
||||
|
||||
assert_ne!(foo_module, bar_module);
|
||||
|
||||
assert_eq!(&src, foo_module.path(&db)?.root());
|
||||
assert_eq!(&foo, &*db.file_path(foo_module.path(&db)?.file()));
|
||||
assert_eq!(&src, foo_module.path(&db).root());
|
||||
assert_eq!(&foo, &*db.file_path(foo_module.path(&db).file()));
|
||||
|
||||
// Bar has a different name but it should point to the same file.
|
||||
|
||||
assert_eq!(&src, bar_module.path(&db)?.root());
|
||||
assert_eq!(foo_module.path(&db)?.file(), bar_module.path(&db)?.file());
|
||||
assert_eq!(&foo, &*db.file_path(bar_module.path(&db)?.file()));
|
||||
assert_eq!(&src, bar_module.path(&db).root());
|
||||
assert_eq!(foo_module.path(&db).file(), bar_module.path(&db).file());
|
||||
assert_eq!(&foo, &*db.file_path(bar_module.path(&db).file()));
|
||||
|
||||
assert_eq!(Some(foo_module), db.path_to_module(&foo)?);
|
||||
assert_eq!(Some(bar_module), db.path_to_module(&bar)?);
|
||||
assert_eq!(Some(foo_module), db.path_to_module(&foo));
|
||||
assert_eq!(Some(bar_module), db.path_to_module(&bar));
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_dependency() -> anyhow::Result<()> {
|
||||
fn resolve_dependency() -> std::io::Result<()> {
|
||||
let TestCase {
|
||||
src,
|
||||
db,
|
||||
@@ -1033,8 +1002,8 @@ mod tests {
|
||||
std::fs::write(foo_path, "from .bar import test")?;
|
||||
std::fs::write(bar_path, "test = 'Hello world'")?;
|
||||
|
||||
let foo_module = db.resolve_module(ModuleName::new("foo"))?.unwrap();
|
||||
let bar_module = db.resolve_module(ModuleName::new("foo.bar"))?.unwrap();
|
||||
let foo_module = db.resolve_module(ModuleName::new("foo")).unwrap();
|
||||
let bar_module = db.resolve_module(ModuleName::new("foo.bar")).unwrap();
|
||||
|
||||
// `from . import bar` in `foo/__init__.py` resolves to `foo`
|
||||
assert_eq!(
|
||||
@@ -1045,13 +1014,13 @@ mod tests {
|
||||
level: NonZeroU32::new(1).unwrap(),
|
||||
module: None,
|
||||
}
|
||||
)?
|
||||
)
|
||||
);
|
||||
|
||||
// `from baz import bar` in `foo/__init__.py` should resolve to `baz.py`
|
||||
assert_eq!(
|
||||
Some(ModuleName::new("baz")),
|
||||
foo_module.resolve_dependency(&db, &Dependency::Module(ModuleName::new("baz")))?
|
||||
foo_module.resolve_dependency(&db, &Dependency::Module(ModuleName::new("baz")))
|
||||
);
|
||||
|
||||
// from .bar import test in `foo/__init__.py` should resolve to `foo/bar.py`
|
||||
@@ -1063,7 +1032,7 @@ mod tests {
|
||||
level: NonZeroU32::new(1).unwrap(),
|
||||
module: Some(ModuleName::new("bar"))
|
||||
}
|
||||
)?
|
||||
)
|
||||
);
|
||||
|
||||
// from .. import test in `foo/__init__.py` resolves to `` which is not a module
|
||||
@@ -1075,7 +1044,7 @@ mod tests {
|
||||
level: NonZeroU32::new(2).unwrap(),
|
||||
module: None
|
||||
}
|
||||
)?
|
||||
)
|
||||
);
|
||||
|
||||
// `from . import test` in `foo/bar.py` resolves to `foo`
|
||||
@@ -1087,13 +1056,13 @@ mod tests {
|
||||
level: NonZeroU32::new(1).unwrap(),
|
||||
module: None
|
||||
}
|
||||
)?
|
||||
)
|
||||
);
|
||||
|
||||
// `from baz import test` in `foo/bar.py` resolves to `baz`
|
||||
assert_eq!(
|
||||
Some(ModuleName::new("baz")),
|
||||
bar_module.resolve_dependency(&db, &Dependency::Module(ModuleName::new("baz")))?
|
||||
bar_module.resolve_dependency(&db, &Dependency::Module(ModuleName::new("baz")))
|
||||
);
|
||||
|
||||
// `from .baz import test` in `foo/bar.py` resolves to `foo.baz`.
|
||||
@@ -1105,7 +1074,7 @@ mod tests {
|
||||
level: NonZeroU32::new(1).unwrap(),
|
||||
module: Some(ModuleName::new("baz"))
|
||||
}
|
||||
)?
|
||||
)
|
||||
);
|
||||
|
||||
Ok(())
|
||||
|
||||
@@ -6,7 +6,7 @@ use ruff_python_parser::{Mode, ParseError};
|
||||
use ruff_text_size::{Ranged, TextRange};
|
||||
|
||||
use crate::cache::KeyValueCache;
|
||||
use crate::db::{HasJar, QueryResult, SourceDb, SourceJar};
|
||||
use crate::db::{HasJar, SourceDb, SourceJar};
|
||||
use crate::files::FileId;
|
||||
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
@@ -64,16 +64,16 @@ impl Parsed {
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "debug", skip(db))]
|
||||
pub(crate) fn parse<Db>(db: &Db, file_id: FileId) -> QueryResult<Parsed>
|
||||
pub(crate) fn parse<Db>(db: &Db, file_id: FileId) -> Parsed
|
||||
where
|
||||
Db: SourceDb + HasJar<SourceJar>,
|
||||
{
|
||||
let parsed = db.jar()?;
|
||||
let parsed = db.jar();
|
||||
|
||||
parsed.parsed.get(&file_id, |file_id| {
|
||||
let source = db.source(*file_id)?;
|
||||
let source = db.source(*file_id);
|
||||
|
||||
Ok(Parsed::from_text(source.text()))
|
||||
Parsed::from_text(source.text())
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -1,9 +1,11 @@
|
||||
use rayon::{current_num_threads, yield_local};
|
||||
use std::num::NonZeroUsize;
|
||||
|
||||
use rayon::max_num_threads;
|
||||
use rustc_hash::FxHashSet;
|
||||
|
||||
use crate::db::{Database, LintDb, QueryError, QueryResult, SemanticDb};
|
||||
use crate::cancellation::CancellationToken;
|
||||
use crate::db::{SemanticDb, SourceDb};
|
||||
use crate::files::FileId;
|
||||
use crate::format::{FormatDb, FormatError};
|
||||
use crate::lint::Diagnostics;
|
||||
use crate::program::Program;
|
||||
use crate::symbols::Dependency;
|
||||
@@ -11,28 +13,42 @@ use crate::symbols::Dependency;
|
||||
impl Program {
|
||||
/// Checks all open files in the workspace and its dependencies.
|
||||
#[tracing::instrument(level = "debug", skip_all)]
|
||||
pub fn check(&self, mode: ExecutionMode) -> QueryResult<Vec<String>> {
|
||||
self.cancelled()?;
|
||||
pub fn check(
|
||||
&self,
|
||||
scheduler: &dyn CheckScheduler,
|
||||
cancellation_token: CancellationToken,
|
||||
) -> Result<Vec<String>, CheckError> {
|
||||
let check_loop = CheckFilesLoop::new(scheduler, cancellation_token);
|
||||
|
||||
let mut context = CheckContext::new(self);
|
||||
check_loop.run(self.workspace().open_files.iter().copied())
|
||||
}
|
||||
|
||||
match mode {
|
||||
ExecutionMode::SingleThreaded => SingleThreadedExecutor.run(&mut context)?,
|
||||
ExecutionMode::ThreadPool => ThreadPoolExecutor.run(&mut context)?,
|
||||
};
|
||||
/// Checks a single file and its dependencies.
|
||||
#[tracing::instrument(level = "debug", skip(self, scheduler, cancellation_token))]
|
||||
pub fn check_file(
|
||||
&self,
|
||||
file: FileId,
|
||||
scheduler: &dyn CheckScheduler,
|
||||
cancellation_token: CancellationToken,
|
||||
) -> Result<Vec<String>, CheckError> {
|
||||
let check_loop = CheckFilesLoop::new(scheduler, cancellation_token);
|
||||
|
||||
Ok(context.finish())
|
||||
check_loop.run([file].into_iter())
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "debug", skip(self, context))]
|
||||
fn check_file(&self, file: FileId, context: &CheckFileContext) -> QueryResult<Diagnostics> {
|
||||
self.cancelled()?;
|
||||
fn do_check_file(
|
||||
&self,
|
||||
file: FileId,
|
||||
context: &CheckContext,
|
||||
) -> Result<Diagnostics, CheckError> {
|
||||
context.cancelled_ok()?;
|
||||
|
||||
let symbol_table = self.symbol_table(file)?;
|
||||
let symbol_table = self.symbol_table(file);
|
||||
let dependencies = symbol_table.dependencies();
|
||||
|
||||
if !dependencies.is_empty() {
|
||||
let module = self.file_to_module(file)?;
|
||||
let module = self.file_to_module(file);
|
||||
|
||||
// TODO scheduling all dependencies here is wasteful if we don't infer any types on them
|
||||
// but I think that's unlikely, so it is okay?
|
||||
@@ -41,19 +57,18 @@ impl Program {
|
||||
for dependency in dependencies {
|
||||
let dependency_name = match dependency {
|
||||
Dependency::Module(name) => Some(name.clone()),
|
||||
Dependency::Relative { .. } => match &module {
|
||||
Some(module) => module.resolve_dependency(self, dependency)?,
|
||||
None => None,
|
||||
},
|
||||
Dependency::Relative { .. } => module
|
||||
.as_ref()
|
||||
.and_then(|module| module.resolve_dependency(self, dependency)),
|
||||
};
|
||||
|
||||
if let Some(dependency_name) = dependency_name {
|
||||
// TODO We may want to have a different check functions for non-first-party
|
||||
// files because we only need to index them and not check them.
|
||||
// Supporting non-first-party code also requires supporting typing stubs.
|
||||
if let Some(dependency) = self.resolve_module(dependency_name)? {
|
||||
if dependency.path(self)?.root().kind().is_first_party() {
|
||||
context.schedule_dependency(dependency.path(self)?.file());
|
||||
if let Some(dependency) = self.resolve_module(dependency_name) {
|
||||
if dependency.path(self).root().kind().is_first_party() {
|
||||
context.schedule_check_file(dependency.path(self).file());
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -63,363 +78,238 @@ impl Program {
|
||||
let mut diagnostics = Vec::new();
|
||||
|
||||
if self.workspace().is_file_open(file) {
|
||||
diagnostics.extend_from_slice(&self.lint_syntax(file)?);
|
||||
diagnostics.extend_from_slice(&self.lint_semantic(file)?);
|
||||
|
||||
match self.check_file_formatted(file) {
|
||||
Ok(format_diagnostics) => {
|
||||
diagnostics.extend_from_slice(&format_diagnostics);
|
||||
}
|
||||
Err(FormatError::Query(err)) => {
|
||||
return Err(err);
|
||||
}
|
||||
Err(FormatError::Format(error)) => {
|
||||
diagnostics.push(format!("Error formatting file: {error}"));
|
||||
}
|
||||
}
|
||||
diagnostics.extend_from_slice(&self.lint_syntax(file));
|
||||
diagnostics.extend_from_slice(&self.lint_semantic(file));
|
||||
}
|
||||
|
||||
Ok(Diagnostics::from(diagnostics))
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Copy, Clone, Debug, PartialEq, Eq)]
|
||||
pub enum ExecutionMode {
|
||||
SingleThreaded,
|
||||
ThreadPool,
|
||||
}
|
||||
|
||||
/// Context that stores state information about the entire check operation.
|
||||
struct CheckContext<'a> {
|
||||
/// IDs of the files that have been queued for checking.
|
||||
/// Schedules checks for files.
|
||||
pub trait CheckScheduler {
|
||||
/// Schedules a check for a file.
|
||||
///
|
||||
/// Used to avoid queuing the same file twice.
|
||||
scheduled_files: FxHashSet<FileId>,
|
||||
/// The check can either be run immediately on the current thread or the check can be queued
|
||||
/// in a thread pool and ran asynchronously.
|
||||
///
|
||||
/// The order in which scheduled checks are executed is not guaranteed.
|
||||
///
|
||||
/// The implementation should call [`CheckFileTask::run`] to execute the check.
|
||||
fn check_file(&self, file_task: CheckFileTask);
|
||||
|
||||
/// Reference to the program that is checked.
|
||||
program: &'a Program,
|
||||
|
||||
/// The aggregated diagnostics
|
||||
diagnostics: Vec<String>,
|
||||
/// The maximum number of checks that can be run concurrently.
|
||||
///
|
||||
/// Returns `None` if the checks run on the current thread (no concurrency).
|
||||
fn max_concurrency(&self) -> Option<NonZeroUsize>;
|
||||
}
|
||||
|
||||
impl<'a> CheckContext<'a> {
|
||||
fn new(program: &'a Program) -> Self {
|
||||
Self {
|
||||
scheduled_files: FxHashSet::default(),
|
||||
program,
|
||||
diagnostics: Vec::new(),
|
||||
}
|
||||
}
|
||||
/// Scheduler that runs checks on a rayon thread pool.
|
||||
pub struct RayonCheckScheduler<'program, 'scope_ref, 'scope> {
|
||||
program: &'program Program,
|
||||
scope: &'scope_ref rayon::Scope<'scope>,
|
||||
}
|
||||
|
||||
/// Returns the tasks to check all open files in the workspace.
|
||||
fn check_open_files(&mut self) -> Vec<CheckOpenFileTask> {
|
||||
self.scheduled_files
|
||||
.extend(self.program.workspace().open_files());
|
||||
|
||||
self.program
|
||||
.workspace()
|
||||
.open_files()
|
||||
.map(|file_id| CheckOpenFileTask { file_id })
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Returns the task to check a dependency.
|
||||
fn check_dependency(&mut self, file_id: FileId) -> Option<CheckDependencyTask> {
|
||||
if self.scheduled_files.insert(file_id) {
|
||||
Some(CheckDependencyTask { file_id })
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
/// Pushes the result for a single file check operation
|
||||
fn push_diagnostics(&mut self, diagnostics: &Diagnostics) {
|
||||
self.diagnostics.extend_from_slice(diagnostics);
|
||||
}
|
||||
|
||||
/// Returns a reference to the program that is being checked.
|
||||
fn program(&self) -> &'a Program {
|
||||
self.program
|
||||
}
|
||||
|
||||
/// Creates a task context that is used to check a single file.
|
||||
fn task_context<'b, S>(&self, dependency_scheduler: &'b S) -> CheckTaskContext<'a, 'b, S>
|
||||
where
|
||||
S: ScheduleDependency,
|
||||
{
|
||||
CheckTaskContext {
|
||||
program: self.program,
|
||||
dependency_scheduler,
|
||||
}
|
||||
}
|
||||
|
||||
fn finish(self) -> Vec<String> {
|
||||
self.diagnostics
|
||||
impl<'program, 'scope_ref, 'scope> RayonCheckScheduler<'program, 'scope_ref, 'scope> {
|
||||
pub fn new(program: &'program Program, scope: &'scope_ref rayon::Scope<'scope>) -> Self {
|
||||
Self { program, scope }
|
||||
}
|
||||
}
|
||||
|
||||
/// Trait that abstracts away how a dependency of a file gets scheduled for checking.
|
||||
trait ScheduleDependency {
|
||||
/// Schedules the file with the given ID for checking.
|
||||
fn schedule(&self, file_id: FileId);
|
||||
}
|
||||
|
||||
impl<T> ScheduleDependency for T
|
||||
impl<'program, 'scope_ref, 'scope> CheckScheduler
|
||||
for RayonCheckScheduler<'program, 'scope_ref, 'scope>
|
||||
where
|
||||
T: Fn(FileId),
|
||||
'program: 'scope,
|
||||
{
|
||||
fn schedule(&self, file_id: FileId) {
|
||||
let f = self;
|
||||
f(file_id);
|
||||
fn check_file(&self, check_file_task: CheckFileTask) {
|
||||
let child_span =
|
||||
tracing::trace_span!("check_file", file_id = check_file_task.file_id.as_u32());
|
||||
let program = self.program;
|
||||
|
||||
self.scope
|
||||
.spawn(move |_| child_span.in_scope(|| check_file_task.run(program)));
|
||||
}
|
||||
|
||||
fn max_concurrency(&self) -> Option<NonZeroUsize> {
|
||||
Some(NonZeroUsize::new(max_num_threads()).unwrap_or(NonZeroUsize::MIN))
|
||||
}
|
||||
}
|
||||
|
||||
/// Context that is used to run a single file check task.
|
||||
///
|
||||
/// The task is generic over `S` because it is passed across thread boundaries and
|
||||
/// we don't want to add the requirement that [`ScheduleDependency`] must be [`Send`].
|
||||
struct CheckTaskContext<'a, 'scheduler, S>
|
||||
where
|
||||
S: ScheduleDependency,
|
||||
{
|
||||
dependency_scheduler: &'scheduler S,
|
||||
/// Scheduler that runs all checks on the current thread.
|
||||
pub struct SameThreadCheckScheduler<'a> {
|
||||
program: &'a Program,
|
||||
}
|
||||
|
||||
impl<'a, 'scheduler, S> CheckTaskContext<'a, 'scheduler, S>
|
||||
where
|
||||
S: ScheduleDependency,
|
||||
{
|
||||
fn as_file_context(&self) -> CheckFileContext<'scheduler> {
|
||||
CheckFileContext {
|
||||
dependency_scheduler: self.dependency_scheduler,
|
||||
}
|
||||
impl<'a> SameThreadCheckScheduler<'a> {
|
||||
pub fn new(program: &'a Program) -> Self {
|
||||
Self { program }
|
||||
}
|
||||
}
|
||||
|
||||
/// Context passed when checking a single file.
|
||||
///
|
||||
/// This is a trimmed down version of [`CheckTaskContext`] with the type parameter `S` erased
|
||||
/// to avoid monomorphization of [`Program:check_file`].
|
||||
struct CheckFileContext<'a> {
|
||||
dependency_scheduler: &'a dyn ScheduleDependency,
|
||||
impl CheckScheduler for SameThreadCheckScheduler<'_> {
|
||||
fn check_file(&self, task: CheckFileTask) {
|
||||
task.run(self.program);
|
||||
}
|
||||
|
||||
fn max_concurrency(&self) -> Option<NonZeroUsize> {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a> CheckFileContext<'a> {
|
||||
fn schedule_dependency(&self, file_id: FileId) {
|
||||
self.dependency_scheduler.schedule(file_id);
|
||||
}
|
||||
#[derive(Debug, Clone)]
|
||||
pub enum CheckError {
|
||||
Cancelled,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
enum CheckFileTask {
|
||||
OpenFile(CheckOpenFileTask),
|
||||
Dependency(CheckDependencyTask),
|
||||
pub struct CheckFileTask {
|
||||
file_id: FileId,
|
||||
context: CheckContext,
|
||||
}
|
||||
|
||||
impl CheckFileTask {
|
||||
/// Runs the task and returns the results for checking this file.
|
||||
fn run<S>(&self, context: &CheckTaskContext<S>) -> QueryResult<Diagnostics>
|
||||
where
|
||||
S: ScheduleDependency,
|
||||
{
|
||||
match self {
|
||||
Self::OpenFile(task) => task.run(context),
|
||||
Self::Dependency(task) => task.run(context),
|
||||
}
|
||||
}
|
||||
|
||||
fn file_id(&self) -> FileId {
|
||||
match self {
|
||||
CheckFileTask::OpenFile(task) => task.file_id,
|
||||
CheckFileTask::Dependency(task) => task.file_id,
|
||||
/// Runs the check and communicates the result to the orchestrator.
|
||||
pub fn run(self, program: &Program) {
|
||||
match program.do_check_file(self.file_id, &self.context) {
|
||||
Ok(diagnostics) => self
|
||||
.context
|
||||
.sender
|
||||
.send(CheckFileMessage::Completed(diagnostics))
|
||||
.unwrap(),
|
||||
Err(CheckError::Cancelled) => self
|
||||
.context
|
||||
.sender
|
||||
.send(CheckFileMessage::Cancelled)
|
||||
.unwrap(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Task to check an open file.
|
||||
|
||||
#[derive(Debug)]
|
||||
struct CheckOpenFileTask {
|
||||
file_id: FileId,
|
||||
#[derive(Clone, Debug)]
|
||||
struct CheckContext {
|
||||
cancellation_token: CancellationToken,
|
||||
sender: crossbeam_channel::Sender<CheckFileMessage>,
|
||||
}
|
||||
|
||||
impl CheckOpenFileTask {
|
||||
fn run<S>(&self, context: &CheckTaskContext<S>) -> QueryResult<Diagnostics>
|
||||
where
|
||||
S: ScheduleDependency,
|
||||
{
|
||||
context
|
||||
.program
|
||||
.check_file(self.file_id, &context.as_file_context())
|
||||
impl CheckContext {
|
||||
fn new(
|
||||
cancellation_token: CancellationToken,
|
||||
sender: crossbeam_channel::Sender<CheckFileMessage>,
|
||||
) -> Self {
|
||||
Self {
|
||||
cancellation_token,
|
||||
sender,
|
||||
}
|
||||
}
|
||||
|
||||
/// Queues a new file for checking using the [`CheckScheduler`].
|
||||
#[allow(unused)]
|
||||
fn schedule_check_file(&self, file_id: FileId) {
|
||||
self.sender.send(CheckFileMessage::Queue(file_id)).unwrap();
|
||||
}
|
||||
|
||||
/// Returns `true` if the check has been cancelled.
|
||||
fn is_cancelled(&self) -> bool {
|
||||
self.cancellation_token.is_cancelled()
|
||||
}
|
||||
|
||||
fn cancelled_ok(&self) -> Result<(), CheckError> {
|
||||
if self.is_cancelled() {
|
||||
Err(CheckError::Cancelled)
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Task to check a dependency file.
|
||||
#[derive(Debug)]
|
||||
struct CheckDependencyTask {
|
||||
file_id: FileId,
|
||||
struct CheckFilesLoop<'a> {
|
||||
scheduler: &'a dyn CheckScheduler,
|
||||
cancellation_token: CancellationToken,
|
||||
pending: usize,
|
||||
queued_files: FxHashSet<FileId>,
|
||||
}
|
||||
|
||||
impl CheckDependencyTask {
|
||||
fn run<S>(&self, context: &CheckTaskContext<S>) -> QueryResult<Diagnostics>
|
||||
where
|
||||
S: ScheduleDependency,
|
||||
{
|
||||
context
|
||||
.program
|
||||
.check_file(self.file_id, &context.as_file_context())
|
||||
impl<'a> CheckFilesLoop<'a> {
|
||||
fn new(scheduler: &'a dyn CheckScheduler, cancellation_token: CancellationToken) -> Self {
|
||||
Self {
|
||||
scheduler,
|
||||
cancellation_token,
|
||||
|
||||
queued_files: FxHashSet::default(),
|
||||
pending: 0,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Executor that schedules the checking of individual program files.
|
||||
trait CheckExecutor {
|
||||
fn run(self, context: &mut CheckContext) -> QueryResult<()>;
|
||||
}
|
||||
fn run(mut self, files: impl Iterator<Item = FileId>) -> Result<Vec<String>, CheckError> {
|
||||
let (sender, receiver) = if let Some(max_concurrency) = self.scheduler.max_concurrency() {
|
||||
crossbeam_channel::bounded(max_concurrency.get())
|
||||
} else {
|
||||
// The checks run on the current thread. That means it is necessary to store all messages
|
||||
// or we risk deadlocking when the main loop never gets a chance to read the messages.
|
||||
crossbeam_channel::unbounded()
|
||||
};
|
||||
|
||||
/// Executor that runs all check operations on the current thread.
|
||||
///
|
||||
/// The executor does not schedule dependencies for checking.
|
||||
/// The main motivation for scheduling dependencies
|
||||
/// in a multithreaded environment is to parse and index the dependencies concurrently.
|
||||
/// However, that doesn't make sense in a single threaded environment, because the dependencies then compute
|
||||
/// with checking the open files. Checking dependencies in a single threaded environment is more likely
|
||||
/// to hurt performance because we end up analyzing files in their entirety, even if we only need to type check parts of them.
|
||||
#[derive(Debug, Default)]
|
||||
struct SingleThreadedExecutor;
|
||||
let context = CheckContext::new(self.cancellation_token.clone(), sender.clone());
|
||||
|
||||
impl CheckExecutor for SingleThreadedExecutor {
|
||||
fn run(self, context: &mut CheckContext) -> QueryResult<()> {
|
||||
let mut queue = context.check_open_files();
|
||||
for file in files {
|
||||
self.queue_file(file, context.clone())?;
|
||||
}
|
||||
|
||||
let noop_schedule_dependency = |_| {};
|
||||
self.run_impl(receiver, &context)
|
||||
}
|
||||
|
||||
while let Some(file) = queue.pop() {
|
||||
context.program().cancelled()?;
|
||||
fn run_impl(
|
||||
mut self,
|
||||
receiver: crossbeam_channel::Receiver<CheckFileMessage>,
|
||||
context: &CheckContext,
|
||||
) -> Result<Vec<String>, CheckError> {
|
||||
if self.cancellation_token.is_cancelled() {
|
||||
return Err(CheckError::Cancelled);
|
||||
}
|
||||
|
||||
let task_context = context.task_context(&noop_schedule_dependency);
|
||||
context.push_diagnostics(&file.run(&task_context)?);
|
||||
let mut result = Vec::default();
|
||||
|
||||
for message in receiver {
|
||||
match message {
|
||||
CheckFileMessage::Completed(diagnostics) => {
|
||||
result.extend_from_slice(&diagnostics);
|
||||
|
||||
self.pending -= 1;
|
||||
|
||||
if self.pending == 0 {
|
||||
break;
|
||||
}
|
||||
}
|
||||
CheckFileMessage::Queue(id) => {
|
||||
self.queue_file(id, context.clone())?;
|
||||
}
|
||||
CheckFileMessage::Cancelled => {
|
||||
return Err(CheckError::Cancelled);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
fn queue_file(&mut self, file_id: FileId, context: CheckContext) -> Result<(), CheckError> {
|
||||
if context.is_cancelled() {
|
||||
return Err(CheckError::Cancelled);
|
||||
}
|
||||
|
||||
if self.queued_files.insert(file_id) {
|
||||
self.pending += 1;
|
||||
|
||||
self.scheduler
|
||||
.check_file(CheckFileTask { file_id, context });
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Executor that runs the check operations on a thread pool.
|
||||
///
|
||||
/// The executor runs each check operation as its own task using a thread pool.
|
||||
///
|
||||
/// Other than [`SingleThreadedExecutor`], this executor schedules dependencies for checking. It
|
||||
/// even schedules dependencies for checking when the thread pool size is 1 for a better debugging experience.
|
||||
#[derive(Debug, Default)]
|
||||
struct ThreadPoolExecutor;
|
||||
|
||||
impl CheckExecutor for ThreadPoolExecutor {
|
||||
fn run(self, context: &mut CheckContext) -> QueryResult<()> {
|
||||
let num_threads = current_num_threads();
|
||||
let single_threaded = num_threads == 1;
|
||||
let span = tracing::trace_span!("ThreadPoolExecutor::run", num_threads);
|
||||
let _ = span.enter();
|
||||
|
||||
let mut queue: Vec<_> = context
|
||||
.check_open_files()
|
||||
.into_iter()
|
||||
.map(CheckFileTask::OpenFile)
|
||||
.collect();
|
||||
|
||||
let (sender, receiver) = if single_threaded {
|
||||
// Use an unbounded queue for single threaded execution to prevent deadlocks
|
||||
// when a single file schedules multiple dependencies.
|
||||
crossbeam::channel::unbounded()
|
||||
} else {
|
||||
// Use a bounded queue to apply backpressure when the orchestration thread isn't able to keep
|
||||
// up processing messages from the worker threads.
|
||||
crossbeam::channel::bounded(num_threads)
|
||||
};
|
||||
|
||||
let schedule_sender = sender.clone();
|
||||
let schedule_dependency = move |file_id| {
|
||||
schedule_sender
|
||||
.send(ThreadPoolMessage::ScheduleDependency(file_id))
|
||||
.unwrap();
|
||||
};
|
||||
|
||||
let result = rayon::in_place_scope(|scope| {
|
||||
let mut pending = 0usize;
|
||||
|
||||
loop {
|
||||
context.program().cancelled()?;
|
||||
|
||||
// 1. Try to get a queued message to ensure that we have always remaining space in the channel to prevent blocking the worker threads.
|
||||
// 2. Try to process a queued file
|
||||
// 3. If there's no queued file wait for the next incoming message.
|
||||
// 4. Exit if there are no more messages and no senders.
|
||||
let message = if let Ok(message) = receiver.try_recv() {
|
||||
message
|
||||
} else if let Some(task) = queue.pop() {
|
||||
pending += 1;
|
||||
|
||||
let task_context = context.task_context(&schedule_dependency);
|
||||
let sender = sender.clone();
|
||||
let task_span = tracing::trace_span!(
|
||||
parent: &span,
|
||||
"CheckFileTask::run",
|
||||
file_id = task.file_id().as_u32(),
|
||||
);
|
||||
|
||||
scope.spawn(move |_| {
|
||||
task_span.in_scope(|| match task.run(&task_context) {
|
||||
Ok(result) => {
|
||||
sender.send(ThreadPoolMessage::Completed(result)).unwrap();
|
||||
}
|
||||
Err(err) => sender.send(ThreadPoolMessage::Errored(err)).unwrap(),
|
||||
});
|
||||
});
|
||||
|
||||
// If this is a single threaded rayon thread pool, yield the current thread
|
||||
// or we never start processing the work items.
|
||||
if single_threaded {
|
||||
yield_local();
|
||||
}
|
||||
|
||||
continue;
|
||||
} else if let Ok(message) = receiver.recv() {
|
||||
message
|
||||
} else {
|
||||
break;
|
||||
};
|
||||
|
||||
match message {
|
||||
ThreadPoolMessage::ScheduleDependency(dependency) => {
|
||||
if let Some(task) = context.check_dependency(dependency) {
|
||||
queue.push(CheckFileTask::Dependency(task));
|
||||
}
|
||||
}
|
||||
ThreadPoolMessage::Completed(diagnostics) => {
|
||||
context.push_diagnostics(&diagnostics);
|
||||
pending -= 1;
|
||||
|
||||
if pending == 0 && queue.is_empty() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
ThreadPoolMessage::Errored(err) => {
|
||||
return Err(err);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
});
|
||||
|
||||
result
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
enum ThreadPoolMessage {
|
||||
ScheduleDependency(FileId),
|
||||
enum CheckFileMessage {
|
||||
Completed(Diagnostics),
|
||||
Errored(QueryError),
|
||||
Queue(FileId),
|
||||
Cancelled,
|
||||
}
|
||||
|
||||
@@ -1,44 +0,0 @@
|
||||
use crate::db::{QueryResult, SourceDb};
|
||||
use crate::format::{FormatDb, FormatError, FormattedFile};
|
||||
use crate::program::Program;
|
||||
|
||||
impl Program {
|
||||
#[tracing::instrument(level = "trace", skip(self))]
|
||||
pub fn format(&mut self) -> QueryResult<()> {
|
||||
// Formats all open files
|
||||
|
||||
// TODO make `Executor` from `check` reusable.
|
||||
for file in self.workspace.open_files() {
|
||||
match self.format_file(file) {
|
||||
Ok(FormattedFile::Formatted(content)) => {
|
||||
let path = self.file_path(file);
|
||||
|
||||
// TODO: This is problematic because it immediately re-triggers the file watcher.
|
||||
// A possible solution is to track the self "inflicted" changes inside of programs
|
||||
// by tracking the file revision right after the write. It could then use the revision
|
||||
// to determine which changes are safe to ignore (and in which context).
|
||||
// An other alternative is to not write as part of the `format` command and instead
|
||||
// return a Vec with the format results and leave the writing to the caller.
|
||||
// I think that's undesired because a) we still need a way to tell the formatter
|
||||
// that it won't be necessary to format the content again and
|
||||
// b) it would reduce concurrency because the writing would need to wait for the file
|
||||
// formatting to be complete, unless we use some form of communication channel.
|
||||
std::fs::write(path, content).expect("Unable to write file");
|
||||
}
|
||||
Ok(FormattedFile::Unchanged) => {
|
||||
// No op
|
||||
}
|
||||
Err(FormatError::Query(error)) => {
|
||||
return Err(error);
|
||||
}
|
||||
Err(FormatError::Format(error)) => {
|
||||
// TODO proper error handling. We should either propagate this error or
|
||||
// emit a diagnostic (probably this).
|
||||
tracing::warn!("Failed to format file: {}", error);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -1,42 +1,45 @@
|
||||
use ruff_formatter::PrintedRange;
|
||||
use ruff_text_size::TextRange;
|
||||
pub mod check;
|
||||
|
||||
use std::path::Path;
|
||||
use std::sync::Arc;
|
||||
|
||||
use crate::db::{
|
||||
Database, Db, DbRuntime, HasJar, HasJars, JarsStorage, LintDb, LintJar, ParallelDatabase,
|
||||
QueryResult, SemanticDb, SemanticJar, Snapshot, SourceDb, SourceJar,
|
||||
};
|
||||
use crate::db::{Db, HasJar, SemanticDb, SemanticJar, SourceDb, SourceJar};
|
||||
use crate::files::{FileId, Files};
|
||||
use crate::format::{
|
||||
check_formatted, format_file, format_file_range, FormatDb, FormatError, FormatJar,
|
||||
FormattedFile,
|
||||
use crate::lint::{
|
||||
lint_semantic, lint_syntax, Diagnostics, LintSemanticStorage, LintSyntaxStorage,
|
||||
};
|
||||
use crate::lint::{lint_semantic, lint_syntax, Diagnostics};
|
||||
use crate::module::{
|
||||
add_module, file_to_module, path_to_module, resolve_module, set_module_search_paths, Module,
|
||||
ModuleData, ModuleName, ModuleSearchPath,
|
||||
ModuleData, ModuleName, ModuleResolver, ModuleSearchPath,
|
||||
};
|
||||
use crate::parse::{parse, Parsed};
|
||||
use crate::source::{source_text, Source};
|
||||
use crate::symbols::{symbol_table, SymbolId, SymbolTable};
|
||||
use crate::types::{infer_symbol_type, Type};
|
||||
use crate::parse::{parse, Parsed, ParsedStorage};
|
||||
use crate::source::{source_text, Source, SourceStorage};
|
||||
use crate::symbols::{symbol_table, SymbolId, SymbolTable, SymbolTablesStorage};
|
||||
use crate::types::{infer_symbol_type, Type, TypeStore};
|
||||
use crate::Workspace;
|
||||
|
||||
pub mod check;
|
||||
mod format;
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct Program {
|
||||
jars: JarsStorage<Program>,
|
||||
files: Files,
|
||||
source: SourceJar,
|
||||
semantic: SemanticJar,
|
||||
workspace: Workspace,
|
||||
}
|
||||
|
||||
impl Program {
|
||||
pub fn new(workspace: Workspace) -> Self {
|
||||
pub fn new(workspace: Workspace, module_search_paths: Vec<ModuleSearchPath>) -> Self {
|
||||
Self {
|
||||
jars: JarsStorage::default(),
|
||||
source: SourceJar {
|
||||
sources: SourceStorage::default(),
|
||||
parsed: ParsedStorage::default(),
|
||||
lint_syntax: LintSyntaxStorage::default(),
|
||||
},
|
||||
semantic: SemanticJar {
|
||||
module_resolver: ModuleResolver::new(module_search_paths),
|
||||
symbol_tables: SymbolTablesStorage::default(),
|
||||
type_store: TypeStore::default(),
|
||||
lint_semantic: LintSemanticStorage::default(),
|
||||
},
|
||||
files: Files::default(),
|
||||
workspace,
|
||||
}
|
||||
@@ -46,17 +49,17 @@ impl Program {
|
||||
where
|
||||
I: IntoIterator<Item = FileChange>,
|
||||
{
|
||||
let (source, semantic, lint, format) = self.jars_mut();
|
||||
for change in changes {
|
||||
semantic.module_resolver.remove_module(change.id);
|
||||
semantic.symbol_tables.remove(&change.id);
|
||||
source.sources.remove(&change.id);
|
||||
source.parsed.remove(&change.id);
|
||||
self.semantic
|
||||
.module_resolver
|
||||
.remove_module(&self.file_path(change.id));
|
||||
self.semantic.symbol_tables.remove(&change.id);
|
||||
self.source.sources.remove(&change.id);
|
||||
self.source.parsed.remove(&change.id);
|
||||
self.source.lint_syntax.remove(&change.id);
|
||||
// TODO: remove all dependent modules as well
|
||||
semantic.type_store.remove_module(change.id);
|
||||
lint.lint_syntax.remove(&change.id);
|
||||
lint.lint_semantic.remove(&change.id);
|
||||
format.formatted.remove(&change.id);
|
||||
self.semantic.type_store.remove_module(change.id);
|
||||
self.semantic.lint_semantic.remove(&change.id);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -82,36 +85,44 @@ impl SourceDb for Program {
|
||||
self.files.path(file_id)
|
||||
}
|
||||
|
||||
fn source(&self, file_id: FileId) -> QueryResult<Source> {
|
||||
fn source(&self, file_id: FileId) -> Source {
|
||||
source_text(self, file_id)
|
||||
}
|
||||
|
||||
fn parse(&self, file_id: FileId) -> QueryResult<Parsed> {
|
||||
fn parse(&self, file_id: FileId) -> Parsed {
|
||||
parse(self, file_id)
|
||||
}
|
||||
|
||||
fn lint_syntax(&self, file_id: FileId) -> Diagnostics {
|
||||
lint_syntax(self, file_id)
|
||||
}
|
||||
}
|
||||
|
||||
impl SemanticDb for Program {
|
||||
fn resolve_module(&self, name: ModuleName) -> QueryResult<Option<Module>> {
|
||||
fn resolve_module(&self, name: ModuleName) -> Option<Module> {
|
||||
resolve_module(self, name)
|
||||
}
|
||||
|
||||
fn file_to_module(&self, file_id: FileId) -> QueryResult<Option<Module>> {
|
||||
fn file_to_module(&self, file_id: FileId) -> Option<Module> {
|
||||
file_to_module(self, file_id)
|
||||
}
|
||||
|
||||
fn path_to_module(&self, path: &Path) -> QueryResult<Option<Module>> {
|
||||
fn path_to_module(&self, path: &Path) -> Option<Module> {
|
||||
path_to_module(self, path)
|
||||
}
|
||||
|
||||
fn symbol_table(&self, file_id: FileId) -> QueryResult<Arc<SymbolTable>> {
|
||||
fn symbol_table(&self, file_id: FileId) -> Arc<SymbolTable> {
|
||||
symbol_table(self, file_id)
|
||||
}
|
||||
|
||||
fn infer_symbol_type(&self, file_id: FileId, symbol_id: SymbolId) -> QueryResult<Type> {
|
||||
fn infer_symbol_type(&self, file_id: FileId, symbol_id: SymbolId) -> Type {
|
||||
infer_symbol_type(self, file_id, symbol_id)
|
||||
}
|
||||
|
||||
fn lint_semantic(&self, file_id: FileId) -> Diagnostics {
|
||||
lint_semantic(self, file_id)
|
||||
}
|
||||
|
||||
// Mutations
|
||||
fn add_module(&mut self, path: &Path) -> Option<(Module, Vec<Arc<ModuleData>>)> {
|
||||
add_module(self, path)
|
||||
@@ -122,105 +133,25 @@ impl SemanticDb for Program {
|
||||
}
|
||||
}
|
||||
|
||||
impl LintDb for Program {
|
||||
fn lint_syntax(&self, file_id: FileId) -> QueryResult<Diagnostics> {
|
||||
lint_syntax(self, file_id)
|
||||
}
|
||||
|
||||
fn lint_semantic(&self, file_id: FileId) -> QueryResult<Diagnostics> {
|
||||
lint_semantic(self, file_id)
|
||||
}
|
||||
}
|
||||
|
||||
impl FormatDb for Program {
|
||||
fn format_file(&self, file_id: FileId) -> Result<FormattedFile, FormatError> {
|
||||
format_file(self, file_id)
|
||||
}
|
||||
|
||||
fn format_file_range(
|
||||
&self,
|
||||
file_id: FileId,
|
||||
range: TextRange,
|
||||
) -> Result<PrintedRange, FormatError> {
|
||||
format_file_range(self, file_id, range)
|
||||
}
|
||||
|
||||
fn check_file_formatted(&self, file_id: FileId) -> Result<Diagnostics, FormatError> {
|
||||
check_formatted(self, file_id)
|
||||
}
|
||||
}
|
||||
|
||||
impl Db for Program {}
|
||||
|
||||
impl Database for Program {
|
||||
fn runtime(&self) -> &DbRuntime {
|
||||
self.jars.runtime()
|
||||
}
|
||||
|
||||
fn runtime_mut(&mut self) -> &mut DbRuntime {
|
||||
self.jars.runtime_mut()
|
||||
}
|
||||
}
|
||||
|
||||
impl ParallelDatabase for Program {
|
||||
fn snapshot(&self) -> Snapshot<Self> {
|
||||
Snapshot::new(Self {
|
||||
jars: self.jars.snapshot(),
|
||||
files: self.files.clone(),
|
||||
workspace: self.workspace.clone(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl HasJars for Program {
|
||||
type Jars = (SourceJar, SemanticJar, LintJar, FormatJar);
|
||||
|
||||
fn jars(&self) -> QueryResult<&Self::Jars> {
|
||||
self.jars.jars()
|
||||
}
|
||||
|
||||
fn jars_mut(&mut self) -> &mut Self::Jars {
|
||||
self.jars.jars_mut()
|
||||
}
|
||||
}
|
||||
|
||||
impl HasJar<SourceJar> for Program {
|
||||
fn jar(&self) -> QueryResult<&SourceJar> {
|
||||
Ok(&self.jars()?.0)
|
||||
fn jar(&self) -> &SourceJar {
|
||||
&self.source
|
||||
}
|
||||
|
||||
fn jar_mut(&mut self) -> &mut SourceJar {
|
||||
&mut self.jars_mut().0
|
||||
&mut self.source
|
||||
}
|
||||
}
|
||||
|
||||
impl HasJar<SemanticJar> for Program {
|
||||
fn jar(&self) -> QueryResult<&SemanticJar> {
|
||||
Ok(&self.jars()?.1)
|
||||
fn jar(&self) -> &SemanticJar {
|
||||
&self.semantic
|
||||
}
|
||||
|
||||
fn jar_mut(&mut self) -> &mut SemanticJar {
|
||||
&mut self.jars_mut().1
|
||||
}
|
||||
}
|
||||
|
||||
impl HasJar<LintJar> for Program {
|
||||
fn jar(&self) -> QueryResult<&LintJar> {
|
||||
Ok(&self.jars()?.2)
|
||||
}
|
||||
|
||||
fn jar_mut(&mut self) -> &mut LintJar {
|
||||
&mut self.jars_mut().2
|
||||
}
|
||||
}
|
||||
|
||||
impl HasJar<FormatJar> for Program {
|
||||
fn jar(&self) -> QueryResult<&FormatJar> {
|
||||
Ok(&self.jars()?.3)
|
||||
}
|
||||
|
||||
fn jar_mut(&mut self) -> &mut FormatJar {
|
||||
&mut self.jars_mut().3
|
||||
&mut self.semantic
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
use crate::cache::KeyValueCache;
|
||||
use crate::db::{HasJar, QueryResult, SourceDb, SourceJar};
|
||||
use crate::db::{HasJar, SourceDb, SourceJar};
|
||||
use ruff_notebook::Notebook;
|
||||
use ruff_python_ast::PySourceType;
|
||||
use std::ops::{Deref, DerefMut};
|
||||
@@ -8,11 +8,11 @@ use std::sync::Arc;
|
||||
use crate::files::FileId;
|
||||
|
||||
#[tracing::instrument(level = "debug", skip(db))]
|
||||
pub(crate) fn source_text<Db>(db: &Db, file_id: FileId) -> QueryResult<Source>
|
||||
pub(crate) fn source_text<Db>(db: &Db, file_id: FileId) -> Source
|
||||
where
|
||||
Db: SourceDb + HasJar<SourceJar>,
|
||||
{
|
||||
let sources = &db.jar()?.sources;
|
||||
let sources = &db.jar().sources;
|
||||
|
||||
sources.get(&file_id, |file_id| {
|
||||
let path = db.file_path(*file_id);
|
||||
@@ -43,7 +43,7 @@ where
|
||||
}
|
||||
};
|
||||
|
||||
Ok(Source { kind })
|
||||
Source { kind }
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -16,22 +16,22 @@ use ruff_python_ast::visitor::preorder::PreorderVisitor;
|
||||
|
||||
use crate::ast_ids::TypedNodeKey;
|
||||
use crate::cache::KeyValueCache;
|
||||
use crate::db::{HasJar, QueryResult, SemanticDb, SemanticJar};
|
||||
use crate::db::{HasJar, SemanticDb, SemanticJar};
|
||||
use crate::files::FileId;
|
||||
use crate::module::ModuleName;
|
||||
use crate::Name;
|
||||
|
||||
#[allow(unreachable_pub)]
|
||||
#[tracing::instrument(level = "debug", skip(db))]
|
||||
pub fn symbol_table<Db>(db: &Db, file_id: FileId) -> QueryResult<Arc<SymbolTable>>
|
||||
pub fn symbol_table<Db>(db: &Db, file_id: FileId) -> Arc<SymbolTable>
|
||||
where
|
||||
Db: SemanticDb + HasJar<SemanticJar>,
|
||||
{
|
||||
let jar = db.jar()?;
|
||||
let jar = db.jar();
|
||||
|
||||
jar.symbol_tables.get(&file_id, |_| {
|
||||
let parsed = db.parse(file_id)?;
|
||||
Ok(Arc::from(SymbolTable::from_ast(parsed.ast())))
|
||||
let parsed = db.parse(file_id);
|
||||
Arc::from(SymbolTable::from_ast(parsed.ast()))
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
use ruff_python_ast::AstNode;
|
||||
|
||||
use crate::db::{HasJar, QueryResult, SemanticDb, SemanticJar};
|
||||
use crate::db::{HasJar, SemanticDb, SemanticJar};
|
||||
use crate::module::ModuleName;
|
||||
use crate::symbols::{Definition, ImportFromDefinition, SymbolId};
|
||||
use crate::types::Type;
|
||||
@@ -11,24 +11,23 @@ use ruff_python_ast as ast;
|
||||
|
||||
// FIXME: Figure out proper dead-lock free synchronisation now that this takes `&db` instead of `&mut db`.
|
||||
#[tracing::instrument(level = "trace", skip(db))]
|
||||
pub fn infer_symbol_type<Db>(db: &Db, file_id: FileId, symbol_id: SymbolId) -> QueryResult<Type>
|
||||
pub fn infer_symbol_type<Db>(db: &Db, file_id: FileId, symbol_id: SymbolId) -> Type
|
||||
where
|
||||
Db: SemanticDb + HasJar<SemanticJar>,
|
||||
{
|
||||
let symbols = db.symbol_table(file_id)?;
|
||||
let symbols = db.symbol_table(file_id);
|
||||
let defs = symbols.definitions(symbol_id);
|
||||
|
||||
if let Some(ty) = db
|
||||
.jar()?
|
||||
.jar()
|
||||
.type_store
|
||||
.get_cached_symbol_type(file_id, symbol_id)
|
||||
{
|
||||
return Ok(ty);
|
||||
return ty;
|
||||
}
|
||||
|
||||
// TODO handle multiple defs, conditional defs...
|
||||
assert_eq!(defs.len(), 1);
|
||||
let type_store = &db.jar()?.type_store;
|
||||
|
||||
let ty = match &defs[0] {
|
||||
Definition::ImportFrom(ImportFromDefinition {
|
||||
@@ -39,11 +38,11 @@ where
|
||||
// TODO relative imports
|
||||
assert!(matches!(level, 0));
|
||||
let module_name = ModuleName::new(module.as_ref().expect("TODO relative imports"));
|
||||
if let Some(module) = db.resolve_module(module_name)? {
|
||||
let remote_file_id = module.path(db)?.file();
|
||||
let remote_symbols = db.symbol_table(remote_file_id)?;
|
||||
if let Some(module) = db.resolve_module(module_name) {
|
||||
let remote_file_id = module.path(db).file();
|
||||
let remote_symbols = db.symbol_table(remote_file_id);
|
||||
if let Some(remote_symbol_id) = remote_symbols.root_symbol_id_by_name(name) {
|
||||
db.infer_symbol_type(remote_file_id, remote_symbol_id)?
|
||||
db.infer_symbol_type(remote_file_id, remote_symbol_id)
|
||||
} else {
|
||||
Type::Unknown
|
||||
}
|
||||
@@ -51,68 +50,71 @@ where
|
||||
Type::Unknown
|
||||
}
|
||||
}
|
||||
Definition::ClassDef(node_key) => {
|
||||
if let Some(ty) = type_store.get_cached_node_type(file_id, node_key.erased()) {
|
||||
ty
|
||||
} else {
|
||||
let parsed = db.parse(file_id)?;
|
||||
Definition::ClassDef(node_key) => db
|
||||
.jar()
|
||||
.type_store
|
||||
.get_cached_node_type(file_id, node_key.erased())
|
||||
.unwrap_or_else(|| {
|
||||
let parsed = db.parse(file_id);
|
||||
let ast = parsed.ast();
|
||||
let node = node_key.resolve_unwrap(ast.as_any_node_ref());
|
||||
|
||||
let mut bases = Vec::with_capacity(node.bases().len());
|
||||
let bases: Vec<_> = node
|
||||
.bases()
|
||||
.iter()
|
||||
.map(|base_expr| infer_expr_type(db, file_id, base_expr))
|
||||
.collect();
|
||||
|
||||
for base in node.bases() {
|
||||
bases.push(infer_expr_type(db, file_id, base)?);
|
||||
}
|
||||
|
||||
let ty = Type::Class(type_store.add_class(file_id, &node.name.id, bases));
|
||||
type_store.cache_node_type(file_id, *node_key.erased(), ty);
|
||||
let store = &db.jar().type_store;
|
||||
let ty = Type::Class(store.add_class(file_id, &node.name.id, bases));
|
||||
store.cache_node_type(file_id, *node_key.erased(), ty);
|
||||
ty
|
||||
}
|
||||
}
|
||||
Definition::FunctionDef(node_key) => {
|
||||
if let Some(ty) = type_store.get_cached_node_type(file_id, node_key.erased()) {
|
||||
ty
|
||||
} else {
|
||||
let parsed = db.parse(file_id)?;
|
||||
}),
|
||||
Definition::FunctionDef(node_key) => db
|
||||
.jar()
|
||||
.type_store
|
||||
.get_cached_node_type(file_id, node_key.erased())
|
||||
.unwrap_or_else(|| {
|
||||
let parsed = db.parse(file_id);
|
||||
let ast = parsed.ast();
|
||||
let node = node_key
|
||||
.resolve(ast.as_any_node_ref())
|
||||
.expect("node key should resolve");
|
||||
|
||||
let ty = type_store.add_function(file_id, &node.name.id).into();
|
||||
type_store.cache_node_type(file_id, *node_key.erased(), ty);
|
||||
let store = &db.jar().type_store;
|
||||
let ty = store.add_function(file_id, &node.name.id).into();
|
||||
store.cache_node_type(file_id, *node_key.erased(), ty);
|
||||
ty
|
||||
}
|
||||
}
|
||||
}),
|
||||
Definition::Assignment(node_key) => {
|
||||
let parsed = db.parse(file_id)?;
|
||||
let parsed = db.parse(file_id);
|
||||
let ast = parsed.ast();
|
||||
let node = node_key.resolve_unwrap(ast.as_any_node_ref());
|
||||
// TODO handle unpacking assignment correctly
|
||||
infer_expr_type(db, file_id, &node.value)?
|
||||
infer_expr_type(db, file_id, &node.value)
|
||||
}
|
||||
_ => todo!("other kinds of definitions"),
|
||||
};
|
||||
|
||||
type_store.cache_symbol_type(file_id, symbol_id, ty);
|
||||
|
||||
db.jar()
|
||||
.type_store
|
||||
.cache_symbol_type(file_id, symbol_id, ty);
|
||||
// TODO record dependencies
|
||||
Ok(ty)
|
||||
ty
|
||||
}
|
||||
|
||||
fn infer_expr_type<Db>(db: &Db, file_id: FileId, expr: &ast::Expr) -> QueryResult<Type>
|
||||
fn infer_expr_type<Db>(db: &Db, file_id: FileId, expr: &ast::Expr) -> Type
|
||||
where
|
||||
Db: SemanticDb + HasJar<SemanticJar>,
|
||||
{
|
||||
// TODO cache the resolution of the type on the node
|
||||
let symbols = db.symbol_table(file_id)?;
|
||||
let symbols = db.symbol_table(file_id);
|
||||
match expr {
|
||||
ast::Expr::Name(name) => {
|
||||
if let Some(symbol_id) = symbols.root_symbol_id_by_name(&name.id) {
|
||||
db.infer_symbol_type(file_id, symbol_id)
|
||||
} else {
|
||||
Ok(Type::Unknown)
|
||||
Type::Unknown
|
||||
}
|
||||
}
|
||||
_ => todo!("full expression type resolution"),
|
||||
@@ -152,7 +154,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn follow_import_to_class() -> anyhow::Result<()> {
|
||||
fn follow_import_to_class() -> std::io::Result<()> {
|
||||
let case = create_test()?;
|
||||
let db = &case.db;
|
||||
|
||||
@@ -161,18 +163,18 @@ mod tests {
|
||||
std::fs::write(a_path, "from b import C as D; E = D")?;
|
||||
std::fs::write(b_path, "class C: pass")?;
|
||||
let a_file = db
|
||||
.resolve_module(ModuleName::new("a"))?
|
||||
.resolve_module(ModuleName::new("a"))
|
||||
.expect("module should be found")
|
||||
.path(db)?
|
||||
.path(db)
|
||||
.file();
|
||||
let a_syms = db.symbol_table(a_file)?;
|
||||
let a_syms = db.symbol_table(a_file);
|
||||
let e_sym = a_syms
|
||||
.root_symbol_id_by_name("E")
|
||||
.expect("E symbol should be found");
|
||||
|
||||
let ty = db.infer_symbol_type(a_file, e_sym)?;
|
||||
let ty = db.infer_symbol_type(a_file, e_sym);
|
||||
|
||||
let jar = HasJar::<SemanticJar>::jar(db)?;
|
||||
let jar = HasJar::<SemanticJar>::jar(db);
|
||||
assert!(matches!(ty, Type::Class(_)));
|
||||
assert_eq!(format!("{}", ty.display(&jar.type_store)), "Literal[C]");
|
||||
|
||||
@@ -180,28 +182,28 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_base_class_by_name() -> anyhow::Result<()> {
|
||||
fn resolve_base_class_by_name() -> std::io::Result<()> {
|
||||
let case = create_test()?;
|
||||
let db = &case.db;
|
||||
|
||||
let path = case.src.path().join("mod.py");
|
||||
std::fs::write(path, "class Base: pass\nclass Sub(Base): pass")?;
|
||||
let file = db
|
||||
.resolve_module(ModuleName::new("mod"))?
|
||||
.resolve_module(ModuleName::new("mod"))
|
||||
.expect("module should be found")
|
||||
.path(db)?
|
||||
.path(db)
|
||||
.file();
|
||||
let syms = db.symbol_table(file)?;
|
||||
let syms = db.symbol_table(file);
|
||||
let sym = syms
|
||||
.root_symbol_id_by_name("Sub")
|
||||
.expect("Sub symbol should be found");
|
||||
|
||||
let ty = db.infer_symbol_type(file, sym)?;
|
||||
let ty = db.infer_symbol_type(file, sym);
|
||||
|
||||
let Type::Class(class_id) = ty else {
|
||||
panic!("Sub is not a Class")
|
||||
};
|
||||
let jar = HasJar::<SemanticJar>::jar(db)?;
|
||||
let jar = HasJar::<SemanticJar>::jar(db);
|
||||
let base_names: Vec<_> = jar
|
||||
.type_store
|
||||
.get_class(class_id)
|
||||
|
||||
@@ -115,25 +115,25 @@ class non_keyword_abcmeta_2(abc.ABCMeta): # safe
|
||||
|
||||
|
||||
# very invalid code, but that's up to mypy et al to check
|
||||
class keyword_abc_1(metaclass=ABC): # incorrect but outside scope of this check
|
||||
class keyword_abc_1(metaclass=ABC): # safe
|
||||
def method(self):
|
||||
foo()
|
||||
|
||||
|
||||
class keyword_abc_2(metaclass=abc.ABC): # incorrect but outside scope of this check
|
||||
class keyword_abc_2(metaclass=abc.ABC): # safe
|
||||
def method(self):
|
||||
foo()
|
||||
|
||||
|
||||
class abc_set_class_variable_1(ABC): # safe (abstract attribute)
|
||||
class abc_set_class_variable_1(ABC): # safe
|
||||
foo: int
|
||||
|
||||
|
||||
class abc_set_class_variable_2(ABC): # error (not an abstract attribute)
|
||||
class abc_set_class_variable_2(ABC): # safe
|
||||
foo = 2
|
||||
|
||||
|
||||
class abc_set_class_variable_3(ABC): # error (not an abstract attribute)
|
||||
class abc_set_class_variable_3(ABC): # safe
|
||||
foo: int = 2
|
||||
|
||||
|
||||
|
||||
@@ -28,13 +28,3 @@ class MyClassBase(metaclass=ABCMeta):
|
||||
@abstractmethod
|
||||
def example(self, value):
|
||||
"""Setter."""
|
||||
|
||||
|
||||
class VariadicParameters:
|
||||
@property
|
||||
def attribute_var_args(self, *args): # [property-with-parameters]
|
||||
return sum(args)
|
||||
|
||||
@property
|
||||
def attribute_var_kwargs(self, **kwargs): #[property-with-parameters]
|
||||
return {key: value * 2 for key, value in kwargs.items()}
|
||||
|
||||
@@ -73,8 +73,3 @@ def foo():
|
||||
|
||||
async def test():
|
||||
return [check async for check in async_func()]
|
||||
|
||||
|
||||
async def test() -> str:
|
||||
vals = [str(val) for val in await async_func(1)]
|
||||
return ",".join(vals)
|
||||
|
||||
@@ -971,32 +971,32 @@ pub fn code_to_rule(linter: Linter, code: &str) -> Option<(RuleGroup, Rule)> {
|
||||
(Ruff, "100") => (RuleGroup::Stable, rules::ruff::rules::UnusedNOQA),
|
||||
(Ruff, "101") => (RuleGroup::Preview, rules::ruff::rules::RedirectedNOQA),
|
||||
(Ruff, "200") => (RuleGroup::Stable, rules::ruff::rules::InvalidPyprojectToml),
|
||||
#[cfg(any(feature = "test-rules", test))]
|
||||
#[cfg(feature = "test-rules")]
|
||||
(Ruff, "900") => (RuleGroup::Stable, rules::ruff::rules::StableTestRule),
|
||||
#[cfg(any(feature = "test-rules", test))]
|
||||
#[cfg(feature = "test-rules")]
|
||||
(Ruff, "901") => (RuleGroup::Stable, rules::ruff::rules::StableTestRuleSafeFix),
|
||||
#[cfg(any(feature = "test-rules", test))]
|
||||
#[cfg(feature = "test-rules")]
|
||||
(Ruff, "902") => (RuleGroup::Stable, rules::ruff::rules::StableTestRuleUnsafeFix),
|
||||
#[cfg(any(feature = "test-rules", test))]
|
||||
#[cfg(feature = "test-rules")]
|
||||
(Ruff, "903") => (RuleGroup::Stable, rules::ruff::rules::StableTestRuleDisplayOnlyFix),
|
||||
#[cfg(any(feature = "test-rules", test))]
|
||||
#[cfg(feature = "test-rules")]
|
||||
(Ruff, "911") => (RuleGroup::Preview, rules::ruff::rules::PreviewTestRule),
|
||||
#[cfg(any(feature = "test-rules", test))]
|
||||
#[cfg(feature = "test-rules")]
|
||||
#[allow(deprecated)]
|
||||
(Ruff, "912") => (RuleGroup::Nursery, rules::ruff::rules::NurseryTestRule),
|
||||
#[cfg(any(feature = "test-rules", test))]
|
||||
#[cfg(feature = "test-rules")]
|
||||
(Ruff, "920") => (RuleGroup::Deprecated, rules::ruff::rules::DeprecatedTestRule),
|
||||
#[cfg(any(feature = "test-rules", test))]
|
||||
#[cfg(feature = "test-rules")]
|
||||
(Ruff, "921") => (RuleGroup::Deprecated, rules::ruff::rules::AnotherDeprecatedTestRule),
|
||||
#[cfg(any(feature = "test-rules", test))]
|
||||
#[cfg(feature = "test-rules")]
|
||||
(Ruff, "930") => (RuleGroup::Removed, rules::ruff::rules::RemovedTestRule),
|
||||
#[cfg(any(feature = "test-rules", test))]
|
||||
#[cfg(feature = "test-rules")]
|
||||
(Ruff, "931") => (RuleGroup::Removed, rules::ruff::rules::AnotherRemovedTestRule),
|
||||
#[cfg(any(feature = "test-rules", test))]
|
||||
#[cfg(feature = "test-rules")]
|
||||
(Ruff, "940") => (RuleGroup::Removed, rules::ruff::rules::RedirectedFromTestRule),
|
||||
#[cfg(any(feature = "test-rules", test))]
|
||||
#[cfg(feature = "test-rules")]
|
||||
(Ruff, "950") => (RuleGroup::Stable, rules::ruff::rules::RedirectedToTestRule),
|
||||
#[cfg(any(feature = "test-rules", test))]
|
||||
#[cfg(feature = "test-rules")]
|
||||
(Ruff, "960") => (RuleGroup::Removed, rules::ruff::rules::RedirectedFromPrefixTestRule),
|
||||
|
||||
|
||||
|
||||
@@ -33,7 +33,7 @@ use crate::message::Message;
|
||||
use crate::noqa::add_noqa;
|
||||
use crate::registry::{AsRule, Rule, RuleSet};
|
||||
use crate::rules::pycodestyle;
|
||||
#[cfg(any(feature = "test-rules", test))]
|
||||
#[cfg(feature = "test-rules")]
|
||||
use crate::rules::ruff::rules::test_rules::{self, TestRule, TEST_RULES};
|
||||
use crate::settings::types::UnsafeFixes;
|
||||
use crate::settings::{flags, LinterSettings};
|
||||
@@ -218,7 +218,7 @@ pub fn check_path(
|
||||
}
|
||||
|
||||
// Raise violations for internal test rules
|
||||
#[cfg(any(feature = "test-rules", test))]
|
||||
#[cfg(feature = "test-rules")]
|
||||
{
|
||||
for test_rule in TEST_RULES {
|
||||
if !settings.rules.enabled(*test_rule) {
|
||||
|
||||
@@ -104,10 +104,10 @@ static REDIRECTS: Lazy<HashMap<&'static str, &'static str>> = Lazy::new(|| {
|
||||
("PGH001", "S307"),
|
||||
("PGH002", "G010"),
|
||||
// Test redirect by exact code
|
||||
#[cfg(any(feature = "test-rules", test))]
|
||||
#[cfg(feature = "test-rules")]
|
||||
("RUF940", "RUF950"),
|
||||
// Test redirect by prefix
|
||||
#[cfg(any(feature = "test-rules", test))]
|
||||
#[cfg(feature = "test-rules")]
|
||||
("RUF96", "RUF95"),
|
||||
// See: https://github.com/astral-sh/ruff/issues/10791
|
||||
("PLW0117", "PLW0177"),
|
||||
|
||||
@@ -323,7 +323,7 @@ mod schema {
|
||||
})
|
||||
.filter(|_rule| {
|
||||
// Filter out all test-only rules
|
||||
#[cfg(any(feature = "test-rules", test))]
|
||||
#[cfg(feature = "test-rules")]
|
||||
#[allow(clippy::used_underscore_binding)]
|
||||
if _rule.starts_with("RUF9") {
|
||||
return false;
|
||||
|
||||
@@ -56,7 +56,6 @@ impl Violation for AbstractBaseClassWithoutAbstractMethod {
|
||||
format!("`{name}` is an abstract base class, but it has no abstract methods")
|
||||
}
|
||||
}
|
||||
|
||||
/// ## What it does
|
||||
/// Checks for empty methods in abstract base classes without an abstract
|
||||
/// decorator.
|
||||
@@ -157,13 +156,8 @@ pub(crate) fn abstract_base_class(
|
||||
let mut has_abstract_method = false;
|
||||
for stmt in body {
|
||||
// https://github.com/PyCQA/flake8-bugbear/issues/293
|
||||
// If an ABC declares an attribute by providing a type annotation
|
||||
// but does not actually assign a value for that attribute,
|
||||
// assume it is intended to be an "abstract attribute"
|
||||
if matches!(
|
||||
stmt,
|
||||
Stmt::AnnAssign(ast::StmtAnnAssign { value: None, .. })
|
||||
) {
|
||||
// Ignore abc's that declares a class attribute that must be set
|
||||
if let Stmt::AnnAssign(_) | Stmt::Assign(_) = stmt {
|
||||
has_abstract_method = true;
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -41,20 +41,6 @@ B024.py:92:7: B024 `notabc_Base_1` is an abstract base class, but it has no abst
|
||||
94 | foo()
|
||||
|
|
||||
|
||||
B024.py:132:7: B024 `abc_set_class_variable_2` is an abstract base class, but it has no abstract methods
|
||||
|
|
||||
132 | class abc_set_class_variable_2(ABC): # error (not an abstract attribute)
|
||||
| ^^^^^^^^^^^^^^^^^^^^^^^^ B024
|
||||
133 | foo = 2
|
||||
|
|
||||
|
||||
B024.py:136:7: B024 `abc_set_class_variable_3` is an abstract base class, but it has no abstract methods
|
||||
|
|
||||
136 | class abc_set_class_variable_3(ABC): # error (not an abstract attribute)
|
||||
| ^^^^^^^^^^^^^^^^^^^^^^^^ B024
|
||||
137 | foo: int = 2
|
||||
|
|
||||
|
||||
B024.py:141:7: B024 `abc_set_class_variable_4` is an abstract base class, but it has no abstract methods
|
||||
|
|
||||
140 | # this doesn't actually declare a class variable, it's just an expression
|
||||
@@ -62,3 +48,5 @@ B024.py:141:7: B024 `abc_set_class_variable_4` is an abstract base class, but it
|
||||
| ^^^^^^^^^^^^^^^^^^^^^^^^ B024
|
||||
142 | foo
|
||||
|
|
||||
|
||||
|
||||
|
||||
@@ -51,13 +51,20 @@ pub(crate) fn property_with_parameters(
|
||||
decorator_list: &[Decorator],
|
||||
parameters: &Parameters,
|
||||
) {
|
||||
if parameters.len() <= 1 {
|
||||
return;
|
||||
}
|
||||
let semantic = checker.semantic();
|
||||
if decorator_list
|
||||
if !decorator_list
|
||||
.iter()
|
||||
.any(|decorator| semantic.match_builtin_expr(&decorator.expression, "property"))
|
||||
{
|
||||
return;
|
||||
}
|
||||
if parameters
|
||||
.posonlyargs
|
||||
.iter()
|
||||
.chain(¶meters.args)
|
||||
.chain(¶meters.kwonlyargs)
|
||||
.count()
|
||||
> 1
|
||||
{
|
||||
checker
|
||||
.diagnostics
|
||||
|
||||
@@ -26,19 +26,4 @@ property_with_parameters.py:15:9: PLR0206 Cannot have defined parameters for pro
|
||||
16 | return param + param1
|
||||
|
|
||||
|
||||
property_with_parameters.py:35:9: PLR0206 Cannot have defined parameters for properties
|
||||
|
|
||||
33 | class VariadicParameters:
|
||||
34 | @property
|
||||
35 | def attribute_var_args(self, *args): # [property-with-parameters]
|
||||
| ^^^^^^^^^^^^^^^^^^ PLR0206
|
||||
36 | return sum(args)
|
||||
|
|
||||
|
||||
property_with_parameters.py:39:9: PLR0206 Cannot have defined parameters for properties
|
||||
|
|
||||
38 | @property
|
||||
39 | def attribute_var_kwargs(self, **kwargs): #[property-with-parameters]
|
||||
| ^^^^^^^^^^^^^^^^^^^^ PLR0206
|
||||
40 | return {key: value * 2 for key, value in kwargs.items()}
|
||||
|
|
||||
|
||||
@@ -21,7 +21,7 @@ pub(crate) use redirected_noqa::*;
|
||||
pub(crate) use sort_dunder_all::*;
|
||||
pub(crate) use sort_dunder_slots::*;
|
||||
pub(crate) use static_key_dict_comprehension::*;
|
||||
#[cfg(any(feature = "test-rules", test))]
|
||||
#[cfg(feature = "test-rules")]
|
||||
pub(crate) use test_rules::*;
|
||||
pub(crate) use unnecessary_dict_comprehension_for_iterable::*;
|
||||
pub(crate) use unnecessary_iterable_allocation_for_first_element::*;
|
||||
@@ -56,7 +56,7 @@ mod sort_dunder_all;
|
||||
mod sort_dunder_slots;
|
||||
mod static_key_dict_comprehension;
|
||||
mod suppression_comment_visitor;
|
||||
#[cfg(any(feature = "test-rules", test))]
|
||||
#[cfg(feature = "test-rules")]
|
||||
pub(crate) mod test_rules;
|
||||
mod unnecessary_dict_comprehension_for_iterable;
|
||||
mod unnecessary_iterable_allocation_for_first_element;
|
||||
|
||||
@@ -58,6 +58,14 @@ impl<'a> preorder::PreorderVisitor<'a> for AsyncExprVisitor {
|
||||
preorder::TraversalSignal::Traverse
|
||||
}
|
||||
}
|
||||
fn visit_expr(&mut self, expr: &'a Expr) {
|
||||
match expr {
|
||||
Expr::Await(_) => {
|
||||
self.found_await_or_async = true;
|
||||
}
|
||||
_ => preorder::walk_expr(self, expr),
|
||||
}
|
||||
}
|
||||
fn visit_stmt(&mut self, stmt: &'a Stmt) {
|
||||
match stmt {
|
||||
Stmt::With(ast::StmtWith { is_async: true, .. }) => {
|
||||
@@ -76,19 +84,9 @@ impl<'a> preorder::PreorderVisitor<'a> for AsyncExprVisitor {
|
||||
_ => preorder::walk_stmt(self, stmt),
|
||||
}
|
||||
}
|
||||
fn visit_expr(&mut self, expr: &'a Expr) {
|
||||
match expr {
|
||||
Expr::Await(_) => {
|
||||
self.found_await_or_async = true;
|
||||
}
|
||||
_ => preorder::walk_expr(self, expr),
|
||||
}
|
||||
}
|
||||
fn visit_comprehension(&mut self, comprehension: &'a ast::Comprehension) {
|
||||
if comprehension.is_async {
|
||||
self.found_await_or_async = true;
|
||||
} else {
|
||||
preorder::walk_comprehension(self, comprehension);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,21 +1,31 @@
|
||||
use ruff_text_size::{TextLen, TextSize};
|
||||
use std::str::Chars;
|
||||
|
||||
pub(crate) const EOF_CHAR: char = '\0';
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub(super) struct Cursor<'a> {
|
||||
chars: Chars<'a>,
|
||||
source_length: TextSize,
|
||||
pub(super) struct Cursor<'src> {
|
||||
/// Source text.
|
||||
source: &'src str,
|
||||
|
||||
/// The start position in the source text of the next character.
|
||||
position: usize,
|
||||
|
||||
/// Offset of the current token from the start of the source. The length of the current
|
||||
/// token can be computed by `self.position - self.current_start`.
|
||||
current_start: TextSize,
|
||||
|
||||
#[cfg(debug_assertions)]
|
||||
prev_char: char,
|
||||
}
|
||||
|
||||
impl<'a> Cursor<'a> {
|
||||
pub(crate) fn new(source: &'a str) -> Self {
|
||||
impl<'src> Cursor<'src> {
|
||||
pub(crate) fn new(source: &'src str) -> Self {
|
||||
assert!(TextSize::try_from(source.len()).is_ok());
|
||||
|
||||
Self {
|
||||
source_length: source.text_len(),
|
||||
chars: source.chars(),
|
||||
source,
|
||||
position: 0,
|
||||
current_start: TextSize::new(0),
|
||||
#[cfg(debug_assertions)]
|
||||
prev_char: EOF_CHAR,
|
||||
}
|
||||
@@ -30,43 +40,48 @@ impl<'a> Cursor<'a> {
|
||||
/// Peeks the next character from the input stream without consuming it.
|
||||
/// Returns [`EOF_CHAR`] if the file is at the end of the file.
|
||||
pub(super) fn first(&self) -> char {
|
||||
self.chars.clone().next().unwrap_or(EOF_CHAR)
|
||||
self.rest().chars().next().unwrap_or(EOF_CHAR)
|
||||
}
|
||||
|
||||
/// Peeks the second character from the input stream without consuming it.
|
||||
/// Returns [`EOF_CHAR`] if the position is past the end of the file.
|
||||
pub(super) fn second(&self) -> char {
|
||||
let mut chars = self.chars.clone();
|
||||
let mut chars = self.rest().chars();
|
||||
chars.next();
|
||||
chars.next().unwrap_or(EOF_CHAR)
|
||||
}
|
||||
|
||||
/// Returns the remaining text to lex.
|
||||
pub(super) fn rest(&self) -> &'a str {
|
||||
self.chars.as_str()
|
||||
pub(super) fn rest(&self) -> &'src str {
|
||||
self.source.get(self.position..).unwrap_or("")
|
||||
}
|
||||
|
||||
// SAFETY: The `source.text_len` call in `new` would panic if the string length is larger than a `u32`.
|
||||
// SAFETY: The `Cursor::new` call would panic if the string length is larger than a `u32`.
|
||||
#[allow(clippy::cast_possible_truncation)]
|
||||
pub(super) fn text_len(&self) -> TextSize {
|
||||
TextSize::new(self.chars.as_str().len() as u32)
|
||||
TextSize::new(self.rest().len() as u32)
|
||||
}
|
||||
|
||||
// SAFETY: The `Cursor::new` call would panic if the string length is larger than a `u32`.
|
||||
#[allow(clippy::cast_possible_truncation)]
|
||||
pub(super) fn token_len(&self) -> TextSize {
|
||||
self.source_length - self.text_len()
|
||||
TextSize::new(self.position as u32) - self.current_start
|
||||
}
|
||||
|
||||
// SAFETY: The `Cursor::new` call would panic if the string length is larger than a `u32`.
|
||||
#[allow(clippy::cast_possible_truncation)]
|
||||
pub(super) fn start_token(&mut self) {
|
||||
self.source_length = self.text_len();
|
||||
self.current_start = TextSize::new(self.position as u32);
|
||||
}
|
||||
|
||||
pub(super) fn is_eof(&self) -> bool {
|
||||
self.chars.as_str().is_empty()
|
||||
self.position >= self.source.len()
|
||||
}
|
||||
|
||||
/// Consumes the next character
|
||||
pub(super) fn bump(&mut self) -> Option<char> {
|
||||
let prev = self.chars.next()?;
|
||||
let prev = self.rest().chars().next()?;
|
||||
self.position += prev.text_len().to_usize();
|
||||
|
||||
#[cfg(debug_assertions)]
|
||||
{
|
||||
@@ -86,7 +101,7 @@ impl<'a> Cursor<'a> {
|
||||
}
|
||||
|
||||
pub(super) fn eat_char2(&mut self, c1: char, c2: char) -> bool {
|
||||
let mut chars = self.chars.clone();
|
||||
let mut chars = self.rest().chars();
|
||||
if chars.next() == Some(c1) && chars.next() == Some(c2) {
|
||||
self.bump();
|
||||
self.bump();
|
||||
@@ -97,7 +112,7 @@ impl<'a> Cursor<'a> {
|
||||
}
|
||||
|
||||
pub(super) fn eat_char3(&mut self, c1: char, c2: char, c3: char) -> bool {
|
||||
let mut chars = self.chars.clone();
|
||||
let mut chars = self.rest().chars();
|
||||
if chars.next() == Some(c1) && chars.next() == Some(c2) && chars.next() == Some(c3) {
|
||||
self.bump();
|
||||
self.bump();
|
||||
@@ -137,17 +152,14 @@ impl<'a> Cursor<'a> {
|
||||
pub(super) fn skip_bytes(&mut self, count: usize) {
|
||||
#[cfg(debug_assertions)]
|
||||
{
|
||||
self.prev_char = self.chars.as_str()[..count]
|
||||
.chars()
|
||||
.next_back()
|
||||
.unwrap_or('\0');
|
||||
self.prev_char = self.rest()[..count].chars().next_back().unwrap_or('\0');
|
||||
}
|
||||
|
||||
self.chars = self.chars.as_str()[count..].chars();
|
||||
self.position += count;
|
||||
}
|
||||
|
||||
/// Skips to the end of the input stream.
|
||||
pub(super) fn skip_to_end(&mut self) {
|
||||
self.chars = "".chars();
|
||||
self.position = self.source.len();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -26,7 +26,7 @@ ruff_text_size = { path = "../ruff_text_size" }
|
||||
ruff_workspace = { path = "../ruff_workspace" }
|
||||
|
||||
anyhow = { workspace = true }
|
||||
crossbeam = { workspace = true }
|
||||
crossbeam-channel = { workspace = true }
|
||||
jod-thread = { workspace = true }
|
||||
libc = { workspace = true }
|
||||
lsp-server = { workspace = true }
|
||||
|
||||
@@ -1,61 +0,0 @@
|
||||
## Helix Setup Guide for `ruff server`
|
||||
|
||||
First, open the language configuration file for Helix. On Linux and macOS, this will be at `~/.config/helix/languages.toml`,
|
||||
and on Windows this will be at `%AppData%\helix\languages.toml`.
|
||||
|
||||
Add the language server by adding:
|
||||
|
||||
```toml
|
||||
[language-server.ruff]
|
||||
command = "ruff"
|
||||
args = ["server", "--preview"]
|
||||
```
|
||||
|
||||
Then, you'll register the language server as the one to use with Python.
|
||||
If you don't already have a language server registered to use with Python, add this to `languages.toml`:
|
||||
|
||||
```toml
|
||||
[[language]]
|
||||
name = "python"
|
||||
language-servers = ["ruff"]
|
||||
```
|
||||
|
||||
Otherwise, if you already have `language-servers` defined, you can simply add `"ruff"` to the list. For example,
|
||||
if you already have `pylsp` as a language server, you can modify the language entry as follows:
|
||||
|
||||
```toml
|
||||
[[language]]
|
||||
name = "python"
|
||||
language-servers = ["ruff", "pylsp"]
|
||||
```
|
||||
|
||||
> \[!NOTE\]
|
||||
> Multiple language servers for a single language are only supported in Helix version [`23.10`](https://github.com/helix-editor/helix/blob/master/CHANGELOG.md#2310-2023-10-24) and later.
|
||||
|
||||
Once you've set up the server, you should see diagnostics in your Python files. Code actions and other LSP features should also be available.
|
||||
|
||||

|
||||
*This screenshot is using `select=["ALL]"` for demonstration purposes.*
|
||||
|
||||
If you want to, as an example, turn on auto-formatting, add `auto-format = true`:
|
||||
|
||||
```toml
|
||||
[[language]]
|
||||
name = "python"
|
||||
language-servers = ["ruff", "pylsp"]
|
||||
auto-format = true
|
||||
```
|
||||
|
||||
See the [Helix documentation](https://docs.helix-editor.com/languages.html) for more settings you can use here.
|
||||
|
||||
You can pass settings into `ruff server` using `[language-server.ruff.config.settings]`. For example:
|
||||
|
||||
```toml
|
||||
[language-server.ruff.config.settings]
|
||||
line-length = 80
|
||||
[language-server.ruff.config.settings.lint]
|
||||
select = ["E4", "E7"]
|
||||
preview = false
|
||||
[language-server.ruff.config.settings.format]
|
||||
preview = true
|
||||
```
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 195 KiB |
@@ -6,7 +6,7 @@ use serde_json::Value;
|
||||
|
||||
use super::schedule::Task;
|
||||
|
||||
pub(crate) type ClientSender = crossbeam::channel::Sender<lsp_server::Message>;
|
||||
pub(crate) type ClientSender = crossbeam_channel::Sender<lsp_server::Message>;
|
||||
|
||||
type ResponseBuilder<'s> = Box<dyn FnOnce(lsp_server::Response) -> Task<'s>>;
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
use std::num::NonZeroUsize;
|
||||
|
||||
use crossbeam::channel::Sender;
|
||||
use crossbeam_channel::Sender;
|
||||
|
||||
use crate::session::Session;
|
||||
|
||||
|
||||
@@ -21,7 +21,7 @@ use std::{
|
||||
},
|
||||
};
|
||||
|
||||
use crossbeam::channel::{Receiver, Sender};
|
||||
use crossbeam_channel::{Receiver, Sender};
|
||||
|
||||
use super::{Builder, JoinHandle, ThreadPriority};
|
||||
|
||||
@@ -52,7 +52,7 @@ impl Pool {
|
||||
let threads = usize::from(threads);
|
||||
|
||||
// Channel buffer capacity is between 2 and 4, depending on the pool size.
|
||||
let (job_sender, job_receiver) = crossbeam::channel::bounded(std::cmp::min(threads * 2, 4));
|
||||
let (job_sender, job_receiver) = crossbeam_channel::bounded(std::cmp::min(threads * 2, 4));
|
||||
let extant_tasks = Arc::new(AtomicUsize::new(0));
|
||||
|
||||
let mut handles = Vec::with_capacity(threads);
|
||||
|
||||
Reference in New Issue
Block a user