Compare commits

..

2 Commits

Author SHA1 Message Date
Dhruv Manilawala
ea48d1068d Use usize, simplify is_eof 2024-05-01 09:46:31 +05:30
Dhruv Manilawala
afe4cdb691 Update Cursor to use position instead of iterator 2024-05-01 09:24:43 +05:30
42 changed files with 770 additions and 1573 deletions

28
Cargo.lock generated
View File

@@ -501,19 +501,6 @@ dependencies = [
"itertools 0.10.5",
]
[[package]]
name = "crossbeam"
version = "0.8.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1137cd7e7fc0fb5d3c5a8678be38ec56e819125d8d7907411fe24ccb943faca8"
dependencies = [
"crossbeam-channel",
"crossbeam-deque",
"crossbeam-epoch",
"crossbeam-queue",
"crossbeam-utils",
]
[[package]]
name = "crossbeam-channel"
version = "0.5.12"
@@ -542,15 +529,6 @@ dependencies = [
"crossbeam-utils",
]
[[package]]
name = "crossbeam-queue"
version = "0.3.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "df0346b5d5e76ac2fe4e327c5fd1118d6be7c51dfb18f9b7922923f287471e35"
dependencies = [
"crossbeam-utils",
]
[[package]]
name = "crossbeam-utils"
version = "0.8.19"
@@ -1826,7 +1804,7 @@ version = "0.1.0"
dependencies = [
"anyhow",
"bitflags 2.5.0",
"crossbeam",
"crossbeam-channel",
"ctrlc",
"dashmap",
"hashbrown 0.14.5",
@@ -1835,11 +1813,9 @@ dependencies = [
"notify",
"parking_lot",
"rayon",
"ruff_formatter",
"ruff_index",
"ruff_notebook",
"ruff_python_ast",
"ruff_python_formatter",
"ruff_python_parser",
"ruff_python_trivia",
"ruff_text_size",
@@ -2365,7 +2341,7 @@ name = "ruff_server"
version = "0.2.2"
dependencies = [
"anyhow",
"crossbeam",
"crossbeam-channel",
"insta",
"jod-thread",
"libc",

View File

@@ -30,7 +30,7 @@ console_error_panic_hook = { version = "0.1.7" }
console_log = { version = "1.0.0" }
countme = { version = "3.0.1" }
criterion = { version = "0.5.1", default-features = false }
crossbeam = { version = "0.8.4" }
crossbeam-channel = { version = "0.5.12" }
dashmap = { version = "5.5.3" }
dirs = { version = "5.0.0" }
drop_bomb = { version = "0.1.5" }

View File

@@ -12,19 +12,17 @@ license.workspace = true
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
ruff_formatter = { path = "../ruff_formatter" }
ruff_index = { path = "../ruff_index" }
ruff_notebook = { path = "../ruff_notebook" }
ruff_python_ast = { path = "../ruff_python_ast" }
ruff_python_formatter = { path = "../ruff_python_formatter" }
ruff_python_parser = { path = "../ruff_python_parser" }
ruff_python_ast = { path = "../ruff_python_ast" }
ruff_python_trivia = { path = "../ruff_python_trivia" }
ruff_text_size = { path = "../ruff_text_size" }
ruff_index = { path = "../ruff_index" }
ruff_notebook = { path = "../ruff_notebook" }
anyhow = { workspace = true }
bitflags = { workspace = true }
ctrlc = "3.4.4"
crossbeam = { workspace = true }
crossbeam-channel = { workspace = true }
dashmap = { workspace = true }
hashbrown = { workspace = true }
indexmap = { workspace = true }

View File

@@ -2,7 +2,6 @@ use std::fmt::Formatter;
use std::hash::Hash;
use std::sync::atomic::{AtomicUsize, Ordering};
use crate::db::QueryResult;
use dashmap::mapref::entry::Entry;
use crate::FxDashMap;
@@ -28,11 +27,11 @@ where
}
}
pub fn get<F>(&self, key: &K, compute: F) -> QueryResult<V>
pub fn get<F>(&self, key: &K, compute: F) -> V
where
F: FnOnce(&K) -> QueryResult<V>,
F: FnOnce(&K) -> V,
{
Ok(match self.map.entry(key.clone()) {
match self.map.entry(key.clone()) {
Entry::Occupied(cached) => {
self.statistics.hit();
@@ -41,11 +40,11 @@ where
Entry::Vacant(vacant) => {
self.statistics.miss();
let value = compute(key)?;
let value = compute(key);
vacant.insert(value.clone());
value
}
})
}
}
pub fn set(&mut self, key: K, value: V) {
@@ -118,29 +117,23 @@ pub type CacheStatistics = DebugStatistics;
#[cfg(not(debug_assertions))]
pub type CacheStatistics = ReleaseStatistics;
pub trait StatisticsRecorder {
fn hit(&self);
fn miss(&self);
fn to_statistics(&self) -> Option<Statistics>;
}
#[derive(Debug, Default)]
pub struct DebugStatistics {
hits: AtomicUsize,
misses: AtomicUsize,
}
impl StatisticsRecorder for DebugStatistics {
impl DebugStatistics {
// TODO figure out appropriate Ordering
fn hit(&self) {
pub fn hit(&self) {
self.hits.fetch_add(1, Ordering::SeqCst);
}
fn miss(&self) {
pub fn miss(&self) {
self.misses.fetch_add(1, Ordering::SeqCst);
}
fn to_statistics(&self) -> Option<Statistics> {
pub fn to_statistics(&self) -> Option<Statistics> {
let hits = self.hits.load(Ordering::SeqCst);
let misses = self.misses.load(Ordering::SeqCst);
@@ -151,15 +144,15 @@ impl StatisticsRecorder for DebugStatistics {
#[derive(Debug, Default)]
pub struct ReleaseStatistics;
impl StatisticsRecorder for ReleaseStatistics {
impl ReleaseStatistics {
#[inline]
fn hit(&self) {}
pub const fn hit(&self) {}
#[inline]
fn miss(&self) {}
pub const fn miss(&self) {}
#[inline]
fn to_statistics(&self) -> Option<Statistics> {
pub const fn to_statistics(&self) -> Option<Statistics> {
None
}
}

View File

@@ -1,25 +1,35 @@
use std::sync::atomic::AtomicBool;
use std::sync::Arc;
use std::sync::{Arc, Condvar, Mutex};
#[derive(Debug, Clone, Default)]
#[derive(Debug, Default)]
pub struct CancellationTokenSource {
signal: Arc<AtomicBool>,
signal: Arc<(Mutex<bool>, Condvar)>,
}
impl CancellationTokenSource {
pub fn new() -> Self {
Self {
signal: Arc::new(AtomicBool::new(false)),
signal: Arc::new((Mutex::new(false), Condvar::default())),
}
}
#[tracing::instrument(level = "trace", skip_all)]
pub fn cancel(&self) {
self.signal.store(true, std::sync::atomic::Ordering::SeqCst);
let (cancelled, condvar) = &*self.signal;
let mut cancelled = cancelled.lock().unwrap();
if *cancelled {
return;
}
*cancelled = true;
condvar.notify_all();
}
pub fn is_cancelled(&self) -> bool {
self.signal.load(std::sync::atomic::Ordering::SeqCst)
let (cancelled, _) = &*self.signal;
*cancelled.lock().unwrap()
}
pub fn token(&self) -> CancellationToken {
@@ -31,12 +41,26 @@ impl CancellationTokenSource {
#[derive(Clone, Debug)]
pub struct CancellationToken {
signal: Arc<AtomicBool>,
signal: Arc<(Mutex<bool>, Condvar)>,
}
impl CancellationToken {
/// Returns `true` if cancellation has been requested.
pub fn is_cancelled(&self) -> bool {
self.signal.load(std::sync::atomic::Ordering::SeqCst)
let (cancelled, _) = &*self.signal;
*cancelled.lock().unwrap()
}
pub fn wait(&self) {
let (bool, condvar) = &*self.signal;
let lock = condvar
.wait_while(bool.lock().unwrap(), |bool| !*bool)
.unwrap();
debug_assert!(*lock);
drop(lock);
}
}

View File

@@ -1,11 +1,6 @@
use std::path::Path;
use std::sync::Arc;
pub use jars::{HasJar, HasJars};
pub use query::{QueryError, QueryResult};
pub use runtime::DbRuntime;
pub use storage::JarsStorage;
use crate::files::FileId;
use crate::lint::{Diagnostics, LintSemanticStorage, LintSyntaxStorage};
use crate::module::{Module, ModuleData, ModuleName, ModuleResolver, ModuleSearchPath};
@@ -14,111 +9,32 @@ use crate::source::{Source, SourceStorage};
use crate::symbols::{SymbolId, SymbolTable, SymbolTablesStorage};
use crate::types::{Type, TypeStore};
mod jars;
mod query;
mod runtime;
mod storage;
pub trait Database {
/// Returns a reference to the runtime of the current worker.
fn runtime(&self) -> &DbRuntime;
/// Returns a mutable reference to the runtime. Only one worker can hold a mutable reference to the runtime.
fn runtime_mut(&mut self) -> &mut DbRuntime;
/// Returns `Ok` if the queries have not been cancelled and `Err(QueryError::Cancelled)` otherwise.
fn cancelled(&self) -> QueryResult<()> {
self.runtime().cancelled()
}
/// Returns `true` if the queries have been cancelled.
fn is_cancelled(&self) -> bool {
self.runtime().is_cancelled()
}
}
/// Database that supports running queries from multiple threads.
pub trait ParallelDatabase: Database + Send {
/// Creates a snapshot of the database state that can be used to query the database in another thread.
///
/// The snapshot is a read-only view of the database but query results are shared between threads.
/// All queries will be automatically cancelled when applying any mutations (calling [`HasJars::jars_mut`])
/// to the database (not the snapshot, because they're readonly).
///
/// ## Creating a snapshot
///
/// Creating a snapshot of the database's jars is cheap but creating a snapshot of
/// other state stored on the database might require deep-cloning data. That's why you should
/// avoid creating snapshots in a hot function (e.g. don't create a snapshot for each file, instead
/// create a snapshot when scheduling the check of an entire program).
///
/// ## Salsa compatibility
/// Salsa prohibits creating a snapshot while running a local query (it's fine if other workers run a query) [[source](https://github.com/salsa-rs/salsa/issues/80)].
/// We should avoid creating snapshots while running a query because we might want to adopt Salsa in the future (if we can figure out persistent caching).
/// Unfortunately, the infrastructure doesn't provide an automated way of knowing when a query is run, that's
/// why we have to "enforce" this constraint manually.
fn snapshot(&self) -> Snapshot<Self>;
}
/// Readonly snapshot of a database.
///
/// ## Dead locks
/// A snapshot should always be dropped as soon as it is no longer necessary to run queries.
/// Storing the snapshot without running a query or periodically checking if cancellation was requested
/// can lead to deadlocks because mutating the [`Database`] requires cancels all pending queries
/// and waiting for all [`Snapshot`]s to be dropped.
#[derive(Debug)]
pub struct Snapshot<DB: ?Sized>
where
DB: ParallelDatabase,
{
db: DB,
}
impl<DB> Snapshot<DB>
where
DB: ParallelDatabase,
{
pub fn new(db: DB) -> Self {
Snapshot { db }
}
}
impl<DB> std::ops::Deref for Snapshot<DB>
where
DB: ParallelDatabase,
{
type Target = DB;
fn deref(&self) -> &DB {
&self.db
}
}
// Red knot specific databases code.
pub trait SourceDb: Database {
pub trait SourceDb {
// queries
fn file_id(&self, path: &std::path::Path) -> FileId;
fn file_path(&self, file_id: FileId) -> Arc<std::path::Path>;
fn source(&self, file_id: FileId) -> QueryResult<Source>;
fn source(&self, file_id: FileId) -> Source;
fn parse(&self, file_id: FileId) -> QueryResult<Parsed>;
fn parse(&self, file_id: FileId) -> Parsed;
fn lint_syntax(&self, file_id: FileId) -> Diagnostics;
}
pub trait SemanticDb: SourceDb {
// queries
fn resolve_module(&self, name: ModuleName) -> QueryResult<Option<Module>>;
fn resolve_module(&self, name: ModuleName) -> Option<Module>;
fn file_to_module(&self, file_id: FileId) -> QueryResult<Option<Module>>;
fn file_to_module(&self, file_id: FileId) -> Option<Module>;
fn path_to_module(&self, path: &Path) -> QueryResult<Option<Module>>;
fn path_to_module(&self, path: &Path) -> Option<Module>;
fn symbol_table(&self, file_id: FileId) -> QueryResult<Arc<SymbolTable>>;
fn symbol_table(&self, file_id: FileId) -> Arc<SymbolTable>;
fn infer_symbol_type(&self, file_id: FileId, symbol_id: SymbolId) -> QueryResult<Type>;
fn infer_symbol_type(&self, file_id: FileId, symbol_id: SymbolId) -> Type;
fn lint_semantic(&self, file_id: FileId) -> Diagnostics;
// mutations
@@ -127,18 +43,13 @@ pub trait SemanticDb: SourceDb {
fn set_module_search_paths(&mut self, paths: Vec<ModuleSearchPath>);
}
pub trait LintDb: SemanticDb {
fn lint_syntax(&self, file_id: FileId) -> QueryResult<Diagnostics>;
fn lint_semantic(&self, file_id: FileId) -> QueryResult<Diagnostics>;
}
pub trait Db: LintDb {}
pub trait Db: SemanticDb {}
#[derive(Debug, Default)]
pub struct SourceJar {
pub sources: SourceStorage,
pub parsed: ParsedStorage,
pub lint_syntax: LintSyntaxStorage,
}
#[derive(Debug, Default)]
@@ -146,12 +57,27 @@ pub struct SemanticJar {
pub module_resolver: ModuleResolver,
pub symbol_tables: SymbolTablesStorage,
pub type_store: TypeStore,
pub lint_semantic: LintSemanticStorage,
}
#[derive(Debug, Default)]
pub struct LintJar {
pub lint_syntax: LintSyntaxStorage,
pub lint_semantic: LintSemanticStorage,
/// Gives access to a specific jar in the database.
///
/// Nope, the terminology isn't borrowed from Java but from Salsa <https://salsa-rs.github.io/salsa/>,
/// which is an analogy to storing the salsa in different jars.
///
/// The basic idea is that each crate can define its own jar and the jars can be combined to a single
/// database in the top level crate. Each crate also defines its own `Database` trait. The combination of
/// `Database` trait and the jar allows to write queries in isolation without having to know how they get composed at the upper levels.
///
/// Salsa further defines a `HasIngredient` trait which slices the jar to a specific storage (e.g. a specific cache).
/// We don't need this just jet because we write our queries by hand. We may want a similar trait if we decide
/// to use a macro to generate the queries.
pub trait HasJar<T> {
/// Gives a read-only reference to the jar.
fn jar(&self) -> &T;
/// Gives a mutable reference to the jar.
fn jar_mut(&mut self) -> &mut T;
}
#[cfg(test)]
@@ -159,10 +85,7 @@ pub(crate) mod tests {
use std::path::Path;
use std::sync::Arc;
use crate::db::{
Database, DbRuntime, HasJar, HasJars, JarsStorage, LintDb, LintJar, QueryResult, SourceDb,
SourceJar,
};
use crate::db::{HasJar, SourceDb, SourceJar};
use crate::files::{FileId, Files};
use crate::lint::{lint_semantic, lint_syntax, Diagnostics};
use crate::module::{
@@ -181,36 +104,27 @@ pub(crate) mod tests {
#[derive(Debug, Default)]
pub(crate) struct TestDb {
files: Files,
jars: JarsStorage<Self>,
source: SourceJar,
semantic: SemanticJar,
}
impl HasJar<SourceJar> for TestDb {
fn jar(&self) -> QueryResult<&SourceJar> {
Ok(&self.jars()?.0)
fn jar(&self) -> &SourceJar {
&self.source
}
fn jar_mut(&mut self) -> &mut SourceJar {
&mut self.jars_mut().0
&mut self.source
}
}
impl HasJar<SemanticJar> for TestDb {
fn jar(&self) -> QueryResult<&SemanticJar> {
Ok(&self.jars()?.1)
fn jar(&self) -> &SemanticJar {
&self.semantic
}
fn jar_mut(&mut self) -> &mut SemanticJar {
&mut self.jars_mut().1
}
}
impl HasJar<LintJar> for TestDb {
fn jar(&self) -> QueryResult<&LintJar> {
Ok(&self.jars()?.2)
}
fn jar_mut(&mut self) -> &mut LintJar {
&mut self.jars_mut().2
&mut self.semantic
}
}
@@ -223,36 +137,44 @@ pub(crate) mod tests {
self.files.path(file_id)
}
fn source(&self, file_id: FileId) -> QueryResult<Source> {
fn source(&self, file_id: FileId) -> Source {
source_text(self, file_id)
}
fn parse(&self, file_id: FileId) -> QueryResult<Parsed> {
fn parse(&self, file_id: FileId) -> Parsed {
parse(self, file_id)
}
fn lint_syntax(&self, file_id: FileId) -> Diagnostics {
lint_syntax(self, file_id)
}
}
impl SemanticDb for TestDb {
fn resolve_module(&self, name: ModuleName) -> QueryResult<Option<Module>> {
fn resolve_module(&self, name: ModuleName) -> Option<Module> {
resolve_module(self, name)
}
fn file_to_module(&self, file_id: FileId) -> QueryResult<Option<Module>> {
fn file_to_module(&self, file_id: FileId) -> Option<Module> {
file_to_module(self, file_id)
}
fn path_to_module(&self, path: &Path) -> QueryResult<Option<Module>> {
fn path_to_module(&self, path: &Path) -> Option<Module> {
path_to_module(self, path)
}
fn symbol_table(&self, file_id: FileId) -> QueryResult<Arc<SymbolTable>> {
fn symbol_table(&self, file_id: FileId) -> Arc<SymbolTable> {
symbol_table(self, file_id)
}
fn infer_symbol_type(&self, file_id: FileId, symbol_id: SymbolId) -> QueryResult<Type> {
fn infer_symbol_type(&self, file_id: FileId, symbol_id: SymbolId) -> Type {
infer_symbol_type(self, file_id, symbol_id)
}
fn lint_semantic(&self, file_id: FileId) -> Diagnostics {
lint_semantic(self, file_id)
}
fn add_module(&mut self, path: &Path) -> Option<(Module, Vec<Arc<ModuleData>>)> {
add_module(self, path)
}
@@ -261,36 +183,4 @@ pub(crate) mod tests {
set_module_search_paths(self, paths);
}
}
impl LintDb for TestDb {
fn lint_syntax(&self, file_id: FileId) -> QueryResult<Diagnostics> {
lint_syntax(self, file_id)
}
fn lint_semantic(&self, file_id: FileId) -> QueryResult<Diagnostics> {
lint_semantic(self, file_id)
}
}
impl HasJars for TestDb {
type Jars = (SourceJar, SemanticJar, LintJar);
fn jars(&self) -> QueryResult<&Self::Jars> {
self.jars.jars()
}
fn jars_mut(&mut self) -> &mut Self::Jars {
self.jars.jars_mut()
}
}
impl Database for TestDb {
fn runtime(&self) -> &DbRuntime {
self.jars.runtime()
}
fn runtime_mut(&mut self) -> &mut DbRuntime {
self.jars.runtime_mut()
}
}
}

View File

@@ -1,37 +0,0 @@
use crate::db::query::QueryResult;
/// Gives access to a specific jar in the database.
///
/// Nope, the terminology isn't borrowed from Java but from Salsa <https://salsa-rs.github.io/salsa/>,
/// which is an analogy to storing the salsa in different jars.
///
/// The basic idea is that each crate can define its own jar and the jars can be combined to a single
/// database in the top level crate. Each crate also defines its own `Database` trait. The combination of
/// `Database` trait and the jar allows to write queries in isolation without having to know how they get composed at the upper levels.
///
/// Salsa further defines a `HasIngredient` trait which slices the jar to a specific storage (e.g. a specific cache).
/// We don't need this just jet because we write our queries by hand. We may want a similar trait if we decide
/// to use a macro to generate the queries.
pub trait HasJar<T> {
/// Gives a read-only reference to the jar.
fn jar(&self) -> QueryResult<&T>;
/// Gives a mutable reference to the jar.
fn jar_mut(&mut self) -> &mut T;
}
/// Gives access to the jars in a database.
pub trait HasJars {
/// A type storing the jars.
///
/// Most commonly, this is a tuple where each jar is a tuple element.
type Jars: Default;
/// Gives access to the underlying jars but tests if the queries have been cancelled.
///
/// Returns `Err(QueryError::Cancelled)` if the queries have been cancelled.
fn jars(&self) -> QueryResult<&Self::Jars>;
/// Gives mutable access to the underlying jars.
fn jars_mut(&mut self) -> &mut Self::Jars;
}

View File

@@ -1,20 +0,0 @@
use std::fmt::{Display, Formatter};
/// Reason why a db query operation failed.
#[derive(Debug, Clone, Copy)]
pub enum QueryError {
/// The query was cancelled because the DB was mutated or the query was cancelled by the host (e.g. on a file change or when pressing CTRL+C).
Cancelled,
}
impl Display for QueryError {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
match self {
QueryError::Cancelled => f.write_str("query was cancelled"),
}
}
}
impl std::error::Error for QueryError {}
pub type QueryResult<T> = Result<T, QueryError>;

View File

@@ -1,41 +0,0 @@
use crate::cancellation::CancellationTokenSource;
use crate::db::{QueryError, QueryResult};
/// Holds the jar agnostic state of the database.
#[derive(Debug, Default)]
pub struct DbRuntime {
/// The cancellation token source used to signal other works that the queries should be aborted and
/// exit at the next possible point.
cancellation_token: CancellationTokenSource,
}
impl DbRuntime {
pub(super) fn snapshot(&self) -> Self {
Self {
cancellation_token: self.cancellation_token.clone(),
}
}
/// Cancels the pending queries of other workers. The current worker cannot have any pending
/// queries because we're holding a mutable reference to the runtime.
pub(super) fn cancel_other_workers(&mut self) {
self.cancellation_token.cancel();
// Set a new cancellation token so that we're in a non-cancelled state again when running the next
// query.
self.cancellation_token = CancellationTokenSource::default();
}
/// Returns `Ok` if the queries have not been cancelled and `Err(QueryError::Cancelled)` otherwise.
pub(super) fn cancelled(&self) -> QueryResult<()> {
if self.cancellation_token.is_cancelled() {
Err(QueryError::Cancelled)
} else {
Ok(())
}
}
/// Returns `true` if the queries have been cancelled.
pub(super) fn is_cancelled(&self) -> bool {
self.cancellation_token.is_cancelled()
}
}

View File

@@ -1,117 +0,0 @@
use std::fmt::Formatter;
use std::sync::Arc;
use crossbeam::sync::WaitGroup;
use crate::db::query::QueryResult;
use crate::db::runtime::DbRuntime;
use crate::db::{HasJars, ParallelDatabase};
/// Stores the jars of a database and the state for each worker.
///
/// Today, all state is shared across all workers, but it may be desired to store data per worker in the future.
pub struct JarsStorage<T>
where
T: HasJars + Sized,
{
// It's important that `jars_wait_group` is declared after `jars` to ensure that `jars` is dropped first.
// See https://doc.rust-lang.org/reference/destructors.html
/// Stores the jars of the database.
jars: Arc<T::Jars>,
/// Used to count the references to `jars`. Allows implementing `jars_mut` without requiring to clone `jars`.
jars_wait_group: WaitGroup,
/// The data agnostic state.
runtime: DbRuntime,
}
impl<Db> JarsStorage<Db>
where
Db: HasJars,
{
pub(super) fn new() -> Self {
Self {
jars: Arc::new(Db::Jars::default()),
jars_wait_group: WaitGroup::default(),
runtime: DbRuntime::default(),
}
}
/// Creates a snapshot of the jars.
///
/// Creating the snapshot is cheap because it doesn't clone the jars, it only increments a ref counter.
#[must_use]
pub fn snapshot(&self) -> JarsStorage<Db>
where
Db: ParallelDatabase,
{
Self {
jars: self.jars.clone(),
jars_wait_group: self.jars_wait_group.clone(),
runtime: self.runtime.snapshot(),
}
}
pub(crate) fn jars(&self) -> QueryResult<&Db::Jars> {
self.runtime.cancelled()?;
Ok(&self.jars)
}
/// Returns a mutable reference to the jars without cloning their content.
///
/// The method cancels any pending queries of other works and waits for them to complete so that
/// this instance is the only instance holding a reference to the jars.
pub(crate) fn jars_mut(&mut self) -> &mut Db::Jars {
// We have a mutable ref here, so no more workers can be spawned between calling this function and taking the mut ref below.
self.cancel_other_workers();
// Now all other references to `self.jars` should have been released. We can now safely return a mutable reference
// to the Arc's content.
let jars =
Arc::get_mut(&mut self.jars).expect("All references to jars should have been released");
jars
}
pub(crate) fn runtime(&self) -> &DbRuntime {
&self.runtime
}
pub(crate) fn runtime_mut(&mut self) -> &mut DbRuntime {
// Note: This method may need to use a similar trick to `jars_mut` if `DbRuntime` is ever to store data that is shared between workers.
&mut self.runtime
}
#[tracing::instrument(level = "trace", skip(self))]
fn cancel_other_workers(&mut self) {
self.runtime.cancel_other_workers();
// Wait for all other works to complete.
let existing_wait = std::mem::take(&mut self.jars_wait_group);
existing_wait.wait();
}
}
impl<Db> Default for JarsStorage<Db>
where
Db: HasJars,
{
fn default() -> Self {
Self::new()
}
}
impl<T> std::fmt::Debug for JarsStorage<T>
where
T: HasJars,
<T as HasJars>::Jars: std::fmt::Debug,
{
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
f.debug_struct("SharedStorage")
.field("jars", &self.jars)
.field("jars_wait_group", &self.jars_wait_group)
.field("runtime", &self.runtime)
.finish()
}
}

View File

@@ -1,135 +0,0 @@
use std::ops::{Deref, DerefMut};
use ruff_formatter::PrintedRange;
use ruff_python_formatter::{FormatModuleError, PyFormatOptions};
use ruff_text_size::TextRange;
use crate::cache::KeyValueCache;
use crate::db::{HasJar, QueryError, SourceDb};
use crate::files::FileId;
use crate::lint::Diagnostics;
use crate::FxDashSet;
pub(crate) trait FormatDb: SourceDb {
/// Formats a file and returns its formatted content or an indicator that it is unchanged.
fn format_file(&self, file_id: FileId) -> Result<FormattedFile, FormatError>;
/// Formats a range in a file.
fn format_file_range(
&self,
file_id: FileId,
range: TextRange,
) -> Result<PrintedRange, FormatError>;
fn check_file_formatted(&self, file_id: FileId) -> Result<Diagnostics, FormatError>;
}
#[tracing::instrument(level = "trace", skip(db))]
pub(crate) fn format_file<Db>(db: &Db, file_id: FileId) -> Result<FormattedFile, FormatError>
where
Db: FormatDb + HasJar<FormatJar>,
{
let formatted = &db.jar()?.formatted;
if formatted.contains(&file_id) {
return Ok(FormattedFile::Unchanged);
}
let source = db.source(file_id)?;
// TODO use the `format_module` method here to re-use the AST.
let printed =
ruff_python_formatter::format_module_source(source.text(), PyFormatOptions::default())?;
Ok(if printed.as_code() == source.text() {
formatted.insert(file_id);
FormattedFile::Unchanged
} else {
FormattedFile::Formatted(printed.into_code())
})
}
#[tracing::instrument(level = "trace", skip(db))]
pub(crate) fn format_file_range<Db: FormatDb + HasJar<FormatJar>>(
db: &Db,
file_id: FileId,
range: TextRange,
) -> Result<PrintedRange, FormatError> {
let formatted = &db.jar()?.formatted;
let source = db.source(file_id)?;
if formatted.contains(&file_id) {
return Ok(PrintedRange::new(source.text()[range].into(), range));
}
// TODO use the `format_module` method here to re-use the AST.
let result =
ruff_python_formatter::format_range(source.text(), range, PyFormatOptions::default())?;
Ok(result)
}
/// Checks if the file is correctly formatted. It creates a diagnostic for formatting issues.
#[tracing::instrument(level = "trace", skip(db))]
pub(crate) fn check_formatted<Db>(db: &Db, file_id: FileId) -> Result<Diagnostics, FormatError>
where
Db: FormatDb + HasJar<FormatJar>,
{
Ok(if db.format_file(file_id)?.is_unchanged() {
Diagnostics::Empty
} else {
Diagnostics::from(vec!["File is not formatted".to_string()])
})
}
#[derive(Debug)]
pub(crate) enum FormatError {
Format(FormatModuleError),
Query(QueryError),
}
impl From<FormatModuleError> for FormatError {
fn from(value: FormatModuleError) -> Self {
Self::Format(value)
}
}
impl From<QueryError> for FormatError {
fn from(value: QueryError) -> Self {
Self::Query(value)
}
}
#[derive(Clone, Eq, PartialEq, Debug)]
pub(crate) enum FormattedFile {
Formatted(String),
Unchanged,
}
impl FormattedFile {
pub(crate) const fn is_unchanged(&self) -> bool {
matches!(self, FormattedFile::Unchanged)
}
}
#[derive(Debug, Default)]
pub struct FormatJar {
pub formatted: FxDashSet<FileId>,
}
#[derive(Default, Debug)]
pub(crate) struct FormattedStorage(KeyValueCache<FileId, ()>);
impl Deref for FormattedStorage {
type Target = KeyValueCache<FileId, ()>;
fn deref(&self) -> &Self::Target {
&self.0
}
}
impl DerefMut for FormattedStorage {
fn deref_mut(&mut self) -> &mut Self::Target {
&mut self.0
}
}

View File

@@ -12,7 +12,6 @@ pub mod cache;
pub mod cancellation;
pub mod db;
pub mod files;
mod format;
pub mod hir;
pub mod lint;
pub mod module;
@@ -28,7 +27,7 @@ pub(crate) type FxDashMap<K, V> = dashmap::DashMap<K, V, BuildHasherDefault<FxHa
pub(crate) type FxDashSet<V> = dashmap::DashSet<V, BuildHasherDefault<FxHasher>>;
pub(crate) type FxIndexSet<V> = indexmap::set::IndexSet<V, BuildHasherDefault<FxHasher>>;
#[derive(Debug, Clone)]
#[derive(Debug)]
pub struct Workspace {
/// TODO this should be a resolved path. We should probably use a newtype wrapper that guarantees that
/// PATH is a UTF-8 path and is normalized.

View File

@@ -1,13 +1,12 @@
use std::cell::RefCell;
use std::ops::{Deref, DerefMut};
use std::sync::Arc;
use std::time::Duration;
use ruff_python_ast::visitor::Visitor;
use ruff_python_ast::{ModModule, StringLiteral};
use crate::cache::KeyValueCache;
use crate::db::{HasJar, LintDb, LintJar, QueryResult, SemanticDb};
use crate::db::{HasJar, SemanticDb, SemanticJar, SourceDb, SourceJar};
use crate::files::FileId;
use crate::parse::Parsed;
use crate::source::Source;
@@ -15,28 +14,19 @@ use crate::symbols::{Definition, SymbolId, SymbolTable};
use crate::types::Type;
#[tracing::instrument(level = "debug", skip(db))]
pub(crate) fn lint_syntax<Db>(db: &Db, file_id: FileId) -> QueryResult<Diagnostics>
pub(crate) fn lint_syntax<Db>(db: &Db, file_id: FileId) -> Diagnostics
where
Db: LintDb + HasJar<LintJar>,
Db: SourceDb + HasJar<SourceJar>,
{
let storage = &db.jar()?.lint_syntax;
#[allow(clippy::print_stdout)]
if std::env::var("RED_KNOT_SLOW_LINT").is_ok() {
for i in 0..10 {
db.cancelled()?;
println!("RED_KNOT_SLOW_LINT is set, sleeping for {i}/10 seconds");
std::thread::sleep(Duration::from_secs(1));
}
}
let storage = &db.jar().lint_syntax;
storage.get(&file_id, |file_id| {
let mut diagnostics = Vec::new();
let source = db.source(*file_id)?;
let source = db.source(*file_id);
lint_lines(source.text(), &mut diagnostics);
let parsed = db.parse(*file_id)?;
let parsed = db.parse(*file_id);
if parsed.errors().is_empty() {
let ast = parsed.ast();
@@ -51,7 +41,7 @@ where
diagnostics.extend(parsed.errors().iter().map(std::string::ToString::to_string));
}
Ok(Diagnostics::from(diagnostics))
Diagnostics::from(diagnostics)
})
}
@@ -73,16 +63,16 @@ fn lint_lines(source: &str, diagnostics: &mut Vec<String>) {
}
#[tracing::instrument(level = "debug", skip(db))]
pub(crate) fn lint_semantic<Db>(db: &Db, file_id: FileId) -> QueryResult<Diagnostics>
pub(crate) fn lint_semantic<Db>(db: &Db, file_id: FileId) -> Diagnostics
where
Db: LintDb + HasJar<LintJar>,
Db: SemanticDb + HasJar<SemanticJar>,
{
let storage = &db.jar()?.lint_semantic;
let storage = &db.jar().lint_semantic;
storage.get(&file_id, |file_id| {
let source = db.source(*file_id)?;
let parsed = db.parse(*file_id)?;
let symbols = db.symbol_table(*file_id)?;
let source = db.source(*file_id);
let parsed = db.parse(*file_id);
let symbols = db.symbol_table(*file_id);
let context = SemanticLintContext {
file_id: *file_id,
@@ -93,25 +83,25 @@ where
diagnostics: RefCell::new(Vec::new()),
};
lint_unresolved_imports(&context)?;
lint_unresolved_imports(&context);
Ok(Diagnostics::from(context.diagnostics.take()))
Diagnostics::from(context.diagnostics.take())
})
}
fn lint_unresolved_imports(context: &SemanticLintContext) -> QueryResult<()> {
fn lint_unresolved_imports(context: &SemanticLintContext) {
// TODO: Consider iterating over the dependencies (imports) only instead of all definitions.
for (symbol, definition) in context.symbols().all_definitions() {
match definition {
Definition::Import(import) => {
let ty = context.infer_symbol_type(symbol)?;
let ty = context.eval_symbol(symbol);
if ty.is_unknown() {
context.push_diagnostic(format!("Unresolved module {}", import.module));
}
}
Definition::ImportFrom(import) => {
let ty = context.infer_symbol_type(symbol)?;
let ty = context.eval_symbol(symbol);
if ty.is_unknown() {
let module_name = import.module().map(Deref::deref).unwrap_or_default();
@@ -136,8 +126,6 @@ fn lint_unresolved_imports(context: &SemanticLintContext) -> QueryResult<()> {
_ => {}
}
}
Ok(())
}
pub struct SemanticLintContext<'a> {
@@ -166,7 +154,7 @@ impl<'a> SemanticLintContext<'a> {
&self.symbols
}
pub fn infer_symbol_type(&self, symbol_id: SymbolId) -> QueryResult<Type> {
pub fn eval_symbol(&self, symbol_id: SymbolId) -> Type {
self.db.infer_symbol_type(self.file_id, symbol_id)
}

View File

@@ -4,7 +4,6 @@ use std::collections::hash_map::Entry;
use std::path::Path;
use std::sync::Mutex;
use crossbeam::channel as crossbeam_channel;
use rustc_hash::FxHashMap;
use tracing::subscriber::Interest;
use tracing::{Level, Metadata};
@@ -13,12 +12,11 @@ use tracing_subscriber::layer::{Context, Filter, SubscriberExt};
use tracing_subscriber::{Layer, Registry};
use tracing_tree::time::Uptime;
use red_knot::db::{
Database, HasJar, ParallelDatabase, QueryError, SemanticDb, SourceDb, SourceJar,
};
use red_knot::cancellation::CancellationTokenSource;
use red_knot::db::{HasJar, SourceDb, SourceJar};
use red_knot::files::FileId;
use red_knot::module::{ModuleSearchPath, ModuleSearchPathKind};
use red_knot::program::check::ExecutionMode;
use red_knot::program::check::{CheckError, RayonCheckScheduler};
use red_knot::program::{FileChange, FileChangeKind, Program};
use red_knot::watch::FileWatcher;
use red_knot::Workspace;
@@ -53,8 +51,7 @@ fn main() -> anyhow::Result<()> {
workspace.root().to_path_buf(),
ModuleSearchPathKind::FirstParty,
);
let mut program = Program::new(workspace);
program.set_module_search_paths(vec![workspace_search_path]);
let mut program = Program::new(workspace, vec![workspace_search_path]);
let entry_id = program.file_id(entry_point);
program.workspace_mut().open_file(entry_id);
@@ -85,7 +82,7 @@ fn main() -> anyhow::Result<()> {
main_loop.run(&mut program);
let source_jar: &SourceJar = program.jar().unwrap();
let source_jar: &SourceJar = program.jar();
dbg!(source_jar.parsed.statistics());
dbg!(source_jar.sources.statistics());
@@ -104,9 +101,10 @@ impl MainLoop {
let (main_loop_sender, main_loop_receiver) = crossbeam_channel::bounded(1);
let mut orchestrator = Orchestrator {
pending_analysis: None,
receiver: orchestrator_receiver,
sender: main_loop_sender.clone(),
revision: 0,
aggregated_changes: AggregatedChanges::default(),
};
std::thread::spawn(move || {
@@ -139,32 +137,34 @@ impl MainLoop {
tracing::trace!("Main Loop: Tick");
match message {
MainLoopMessage::CheckProgram { revision } => {
{
let program = program.snapshot();
let sender = self.orchestrator_sender.clone();
MainLoopMessage::CheckProgram => {
// Remove mutability from program.
let program = &*program;
let run_cancellation_token_source = CancellationTokenSource::new();
let run_cancellation_token = run_cancellation_token_source.token();
let sender = &self.orchestrator_sender;
// Spawn a new task that checks the program. This needs to be done in a separate thread
// to prevent blocking the main loop here.
rayon::spawn(move || match program.check(ExecutionMode::ThreadPool) {
Ok(result) => {
sender
.send(OrchestratorMessage::CheckProgramCompleted {
diagnostics: result,
revision,
})
.unwrap();
}
Err(QueryError::Cancelled) => {}
});
}
sender
.send(OrchestratorMessage::CheckProgramStarted {
cancellation_token: run_cancellation_token_source,
})
.unwrap();
if !program.is_cancelled() {
let _ = program.format();
}
rayon::in_place_scope(|scope| {
let scheduler = RayonCheckScheduler::new(program, scope);
let result = program.check(&scheduler, run_cancellation_token);
match result {
Ok(result) => sender
.send(OrchestratorMessage::CheckProgramCompleted(result))
.unwrap(),
Err(CheckError::Cancelled) => sender
.send(OrchestratorMessage::CheckProgramCancelled)
.unwrap(),
}
});
}
MainLoopMessage::ApplyChanges(changes) => {
// Automatically cancels any pending queries and waits for them to complete.
program.apply_changes(changes.iter());
}
MainLoopMessage::CheckCompleted(diagnostics) => {
@@ -211,11 +211,13 @@ impl MainLoopCancellationToken {
}
struct Orchestrator {
aggregated_changes: AggregatedChanges,
pending_analysis: Option<PendingAnalysisState>,
/// Sends messages to the main loop.
sender: crossbeam_channel::Sender<MainLoopMessage>,
/// Receives messages from the main loop.
receiver: crossbeam_channel::Receiver<OrchestratorMessage>,
revision: usize,
}
impl Orchestrator {
@@ -223,33 +225,51 @@ impl Orchestrator {
while let Ok(message) = self.receiver.recv() {
match message {
OrchestratorMessage::Run => {
self.pending_analysis = None;
self.sender.send(MainLoopMessage::CheckProgram).unwrap();
}
OrchestratorMessage::CheckProgramStarted { cancellation_token } => {
debug_assert!(self.pending_analysis.is_none());
self.pending_analysis = Some(PendingAnalysisState { cancellation_token });
}
OrchestratorMessage::CheckProgramCompleted(diagnostics) => {
self.pending_analysis
.take()
.expect("Expected a pending analysis.");
self.sender
.send(MainLoopMessage::CheckProgram {
revision: self.revision,
})
.send(MainLoopMessage::CheckCompleted(diagnostics))
.unwrap();
}
OrchestratorMessage::CheckProgramCompleted {
diagnostics,
revision,
} => {
// Only take the diagnostics if they are for the latest revision.
if self.revision == revision {
self.sender
.send(MainLoopMessage::CheckCompleted(diagnostics))
.unwrap();
} else {
tracing::debug!("Discarding diagnostics for outdated revision {revision} (current: {}).", self.revision);
}
OrchestratorMessage::CheckProgramCancelled => {
self.pending_analysis
.take()
.expect("Expected a pending analysis.");
self.debounce_changes();
}
OrchestratorMessage::FileChanges(changes) => {
// Request cancellation, but wait until all analysis tasks have completed to
// avoid stale messages in the next main loop.
let pending = if let Some(pending_state) = self.pending_analysis.as_ref() {
pending_state.cancellation_token.cancel();
true
} else {
false
};
self.revision += 1;
self.debounce_changes(changes);
self.aggregated_changes.extend(changes);
// If there are no pending analysis tasks, apply the file changes. Otherwise
// keep running until all file checks have completed.
if !pending {
self.debounce_changes();
}
}
OrchestratorMessage::Shutdown => {
return self.shutdown();
@@ -258,9 +278,8 @@ impl Orchestrator {
}
}
fn debounce_changes(&self, changes: Vec<FileChange>) {
let mut aggregated_changes = AggregatedChanges::default();
aggregated_changes.extend(changes);
fn debounce_changes(&mut self) {
debug_assert!(self.pending_analysis.is_none());
loop {
// Consume possibly incoming file change messages before running a new analysis, but don't wait for more than 100ms.
@@ -271,12 +290,10 @@ impl Orchestrator {
return self.shutdown();
}
Ok(OrchestratorMessage::FileChanges(file_changes)) => {
aggregated_changes.extend(file_changes);
self.aggregated_changes.extend(file_changes);
}
Ok(OrchestratorMessage::CheckProgramCompleted { .. })=> {
// disregard any outdated completion message.
}
Ok(OrchestratorMessage::CheckProgramStarted {..}| OrchestratorMessage::CheckProgramCompleted(_) | OrchestratorMessage::CheckProgramCancelled) => unreachable!("No program check should be running while debouncing changes."),
Ok(OrchestratorMessage::Run) => unreachable!("The orchestrator is already running."),
Err(_) => {
@@ -285,10 +302,10 @@ impl Orchestrator {
}
}
},
default(std::time::Duration::from_millis(10)) => {
// No more file changes after 10 ms, send the changes and schedule a new analysis
self.sender.send(MainLoopMessage::ApplyChanges(aggregated_changes)).unwrap();
self.sender.send(MainLoopMessage::CheckProgram { revision: self.revision}).unwrap();
default(std::time::Duration::from_millis(100)) => {
// No more file changes after 100 ms, send the changes and schedule a new analysis
self.sender.send(MainLoopMessage::ApplyChanges(std::mem::take(&mut self.aggregated_changes))).unwrap();
self.sender.send(MainLoopMessage::CheckProgram).unwrap();
return;
}
}
@@ -301,10 +318,15 @@ impl Orchestrator {
}
}
#[derive(Debug)]
struct PendingAnalysisState {
cancellation_token: CancellationTokenSource,
}
/// Message sent from the orchestrator to the main loop.
#[derive(Debug)]
enum MainLoopMessage {
CheckProgram { revision: usize },
CheckProgram,
CheckCompleted(Vec<String>),
ApplyChanges(AggregatedChanges),
Exit,
@@ -315,10 +337,11 @@ enum OrchestratorMessage {
Run,
Shutdown,
CheckProgramCompleted {
diagnostics: Vec<String>,
revision: usize,
CheckProgramStarted {
cancellation_token: CancellationTokenSource,
},
CheckProgramCompleted(Vec<String>),
CheckProgramCancelled,
FileChanges(Vec<FileChange>),
}

View File

@@ -7,7 +7,7 @@ use std::sync::Arc;
use dashmap::mapref::entry::Entry;
use smol_str::SmolStr;
use crate::db::{HasJar, QueryResult, SemanticDb, SemanticJar};
use crate::db::{HasJar, SemanticDb, SemanticJar};
use crate::files::FileId;
use crate::symbols::Dependency;
use crate::FxDashMap;
@@ -17,48 +17,44 @@ use crate::FxDashMap;
pub struct Module(u32);
impl Module {
pub fn name<Db>(&self, db: &Db) -> QueryResult<ModuleName>
pub fn name<Db>(&self, db: &Db) -> ModuleName
where
Db: HasJar<SemanticJar>,
{
let modules = &db.jar()?.module_resolver;
let modules = &db.jar().module_resolver;
Ok(modules.modules.get(self).unwrap().name.clone())
modules.modules.get(self).unwrap().name.clone()
}
pub fn path<Db>(&self, db: &Db) -> QueryResult<ModulePath>
pub fn path<Db>(&self, db: &Db) -> ModulePath
where
Db: HasJar<SemanticJar>,
{
let modules = &db.jar()?.module_resolver;
let modules = &db.jar().module_resolver;
Ok(modules.modules.get(self).unwrap().path.clone())
modules.modules.get(self).unwrap().path.clone()
}
pub fn kind<Db>(&self, db: &Db) -> QueryResult<ModuleKind>
pub fn kind<Db>(&self, db: &Db) -> ModuleKind
where
Db: HasJar<SemanticJar>,
{
let modules = &db.jar()?.module_resolver;
let modules = &db.jar().module_resolver;
Ok(modules.modules.get(self).unwrap().kind)
modules.modules.get(self).unwrap().kind
}
pub fn resolve_dependency<Db>(
&self,
db: &Db,
dependency: &Dependency,
) -> QueryResult<Option<ModuleName>>
pub fn resolve_dependency<Db>(&self, db: &Db, dependency: &Dependency) -> Option<ModuleName>
where
Db: HasJar<SemanticJar>,
{
let (level, module) = match dependency {
Dependency::Module(module) => return Ok(Some(module.clone())),
Dependency::Module(module) => return Some(module.clone()),
Dependency::Relative { level, module } => (*level, module.as_deref()),
};
let name = self.name(db)?;
let kind = self.kind(db)?;
let name = self.name(db);
let kind = self.kind(db);
let mut components = name.components().peekable();
@@ -71,9 +67,7 @@ impl Module {
// Skip over the relative parts.
for _ in start..level.get() {
if components.next_back().is_none() {
return Ok(None);
}
components.next_back()?;
}
let mut name = String::new();
@@ -86,11 +80,11 @@ impl Module {
name.push_str(part);
}
Ok(if name.is_empty() {
if name.is_empty() {
None
} else {
Some(ModuleName(SmolStr::new(name)))
})
}
}
}
@@ -244,25 +238,20 @@ pub struct ModuleData {
/// TODO: This would not work with Salsa because `ModuleName` isn't an ingredient and, therefore, cannot be used as part of a query.
/// For this to work with salsa, it would be necessary to intern all `ModuleName`s.
#[tracing::instrument(level = "debug", skip(db))]
pub fn resolve_module<Db>(db: &Db, name: ModuleName) -> QueryResult<Option<Module>>
pub fn resolve_module<Db>(db: &Db, name: ModuleName) -> Option<Module>
where
Db: SemanticDb + HasJar<SemanticJar>,
{
let jar = db.jar();
let modules = &jar?.module_resolver;
let modules = &jar.module_resolver;
let entry = modules.by_name.entry(name.clone());
match entry {
Entry::Occupied(entry) => Ok(Some(*entry.get())),
Entry::Occupied(entry) => Some(*entry.get()),
Entry::Vacant(entry) => {
let Some((root_path, absolute_path, kind)) = resolve_name(&name, &modules.search_paths)
else {
return Ok(None);
};
let Ok(normalized) = absolute_path.canonicalize() else {
return Ok(None);
};
let (root_path, absolute_path, kind) = resolve_name(&name, &modules.search_paths)?;
let normalized = absolute_path.canonicalize().ok()?;
let file_id = db.file_id(&normalized);
let path = ModulePath::new(root_path.clone(), file_id);
@@ -284,76 +273,59 @@ where
// ```
// Here, both `foo` and `bar` resolve to the same module but through different paths.
// That's why we need to insert the absolute path and not the normalized path here.
let absolute_id = if absolute_path == normalized {
file_id
} else {
db.file_id(&absolute_path)
};
modules.by_file.insert(absolute_id, id);
modules.by_path.insert(absolute_path, id);
entry.insert_entry(id);
Ok(Some(id))
Some(id)
}
}
}
/// Resolves the module id for the given path.
///
/// Returns `None` if the path is not a module in `sys.path`.
#[tracing::instrument(level = "debug", skip(db))]
pub fn path_to_module<Db>(db: &Db, path: &Path) -> QueryResult<Option<Module>>
where
Db: SemanticDb + HasJar<SemanticJar>,
{
let file = db.file_id(path);
file_to_module(db, file)
}
/// Resolves the module id for the file with the given id.
///
/// Returns `None` if the file is not a module in `sys.path`.
#[tracing::instrument(level = "debug", skip(db))]
pub fn file_to_module<Db>(db: &Db, file: FileId) -> QueryResult<Option<Module>>
pub fn file_to_module<Db>(db: &Db, file: FileId) -> Option<Module>
where
Db: SemanticDb + HasJar<SemanticJar>,
{
let jar = db.jar()?;
let modules = &jar.module_resolver;
if let Some(existing) = modules.by_file.get(&file) {
return Ok(Some(*existing));
}
let path = db.file_path(file);
path_to_module(db, &path)
}
/// Resolves the module id for the given path.
///
/// Returns `None` if the path is not a module in `sys.path`.
#[tracing::instrument(level = "debug", skip(db))]
pub fn path_to_module<Db>(db: &Db, path: &Path) -> Option<Module>
where
Db: SemanticDb + HasJar<SemanticJar>,
{
let jar = db.jar();
let modules = &jar.module_resolver;
debug_assert!(path.is_absolute());
let Some((root_path, relative_path)) = modules.search_paths.iter().find_map(|root| {
if let Some(existing) = modules.by_path.get(path) {
return Some(*existing);
}
let (root_path, relative_path) = modules.search_paths.iter().find_map(|root| {
let relative_path = path.strip_prefix(root.path()).ok()?;
Some((root.clone(), relative_path))
}) else {
return Ok(None);
};
})?;
let Some(module_name) = ModuleName::from_relative_path(relative_path) else {
return Ok(None);
};
let module_name = ModuleName::from_relative_path(relative_path)?;
// Resolve the module name to see if Python would resolve the name to the same path.
// If it doesn't, then that means that multiple modules have the same in different
// root paths, but that the module corresponding to the past path is in a lower priority path,
// in which case we ignore it.
let Some(module_id) = resolve_module(db, module_name)? else {
return Ok(None);
};
let module_path = module_id.path(db)?;
let module_id = resolve_module(db, module_name)?;
let module_path = module_id.path(db);
if module_path.root() == &root_path {
let Ok(normalized) = path.canonicalize() else {
return Ok(None);
};
let normalized = path.canonicalize().ok()?;
let interned_normalized = db.file_id(&normalized);
if interned_normalized != module_path.file() {
@@ -364,15 +336,15 @@ where
// ```
// The module name of `src/foo.py` is `foo`, but the module loaded by Python is `src/foo/__init__.py`.
// That means we need to ignore `src/foo.py` even though it resolves to the same module name.
return Ok(None);
return None;
}
// Path has been inserted by `resolved`
Ok(Some(module_id))
Some(module_id)
} else {
// This path is for a module with the same name but in a module search path with a lower priority.
// Ignore it.
Ok(None)
None
}
}
@@ -406,7 +378,7 @@ where
// TODO This needs tests
// Note: Intentionally by-pass caching here. Module should not be in the cache yet.
let module = path_to_module(db, path).ok()??;
let module = path_to_module(db, path)?;
// The code below is to handle the addition of `__init__.py` files.
// When an `__init__.py` file is added, we need to remove all modules that are part of the same package.
@@ -420,7 +392,7 @@ where
return Some((module, Vec::new()));
}
let Some(parent_name) = module.name(db).ok()?.parent() else {
let Some(parent_name) = module.name(db).parent() else {
return Some((module, Vec::new()));
};
@@ -429,7 +401,7 @@ where
let jar = db.jar_mut();
let modules = &mut jar.module_resolver;
modules.by_file.retain(|_, id| {
modules.by_path.retain(|_, id| {
if modules
.modules
.get(id)
@@ -468,7 +440,7 @@ pub struct ModuleResolver {
/// Lookup from absolute path to module.
/// The same module might be reachable from different paths when symlinks are involved.
by_file: FxDashMap<FileId, Module>,
by_path: FxDashMap<PathBuf, Module>,
next_module_id: AtomicU32,
}
@@ -478,14 +450,14 @@ impl ModuleResolver {
search_paths,
modules: FxDashMap::default(),
by_name: FxDashMap::default(),
by_file: FxDashMap::default(),
by_path: FxDashMap::default(),
next_module_id: AtomicU32::new(0),
}
}
pub(crate) fn remove_module(&mut self, file_id: FileId) {
pub fn remove_module(&mut self, path: &Path) {
// No locking is required because we're holding a mutable reference to `self`.
let Some((_, id)) = self.by_file.remove(&file_id) else {
let Some((_, id)) = self.by_path.remove(path) else {
return;
};
@@ -498,7 +470,7 @@ impl ModuleResolver {
self.by_name.remove(&module.name).unwrap();
// It's possible that multiple paths map to the same id. Search all other paths referencing the same module id.
self.by_file.retain(|_, current_id| *current_id != id);
self.by_path.retain(|_, current_id| *current_id != id);
module
}
@@ -719,7 +691,7 @@ mod tests {
}
#[test]
fn first_party_module() -> anyhow::Result<()> {
fn first_party_module() -> std::io::Result<()> {
let TestCase {
db,
src,
@@ -730,22 +702,22 @@ mod tests {
let foo_path = src.path().join("foo.py");
std::fs::write(&foo_path, "print('Hello, world!')")?;
let foo_module = db.resolve_module(ModuleName::new("foo"))?.unwrap();
let foo_module = db.resolve_module(ModuleName::new("foo")).unwrap();
assert_eq!(Some(foo_module), db.resolve_module(ModuleName::new("foo"))?);
assert_eq!(Some(foo_module), db.resolve_module(ModuleName::new("foo")));
assert_eq!(ModuleName::new("foo"), foo_module.name(&db)?);
assert_eq!(&src, foo_module.path(&db)?.root());
assert_eq!(ModuleKind::Module, foo_module.kind(&db)?);
assert_eq!(&foo_path, &*db.file_path(foo_module.path(&db)?.file()));
assert_eq!(ModuleName::new("foo"), foo_module.name(&db));
assert_eq!(&src, foo_module.path(&db).root());
assert_eq!(ModuleKind::Module, foo_module.kind(&db));
assert_eq!(&foo_path, &*db.file_path(foo_module.path(&db).file()));
assert_eq!(Some(foo_module), db.path_to_module(&foo_path)?);
assert_eq!(Some(foo_module), db.path_to_module(&foo_path));
Ok(())
}
#[test]
fn resolve_package() -> anyhow::Result<()> {
fn resolve_package() -> std::io::Result<()> {
let TestCase {
src,
db,
@@ -758,22 +730,22 @@ mod tests {
std::fs::create_dir(&foo_dir)?;
std::fs::write(&foo_path, "print('Hello, world!')")?;
let foo_module = db.resolve_module(ModuleName::new("foo"))?.unwrap();
let foo_module = db.resolve_module(ModuleName::new("foo")).unwrap();
assert_eq!(ModuleName::new("foo"), foo_module.name(&db)?);
assert_eq!(&src, foo_module.path(&db)?.root());
assert_eq!(&foo_path, &*db.file_path(foo_module.path(&db)?.file()));
assert_eq!(ModuleName::new("foo"), foo_module.name(&db));
assert_eq!(&src, foo_module.path(&db).root());
assert_eq!(&foo_path, &*db.file_path(foo_module.path(&db).file()));
assert_eq!(Some(foo_module), db.path_to_module(&foo_path)?);
assert_eq!(Some(foo_module), db.path_to_module(&foo_path));
// Resolving by directory doesn't resolve to the init file.
assert_eq!(None, db.path_to_module(&foo_dir)?);
assert_eq!(None, db.path_to_module(&foo_dir));
Ok(())
}
#[test]
fn package_priority_over_module() -> anyhow::Result<()> {
fn package_priority_over_module() -> std::io::Result<()> {
let TestCase {
db,
temp_dir: _temp_dir,
@@ -789,20 +761,20 @@ mod tests {
let foo_py = src.path().join("foo.py");
std::fs::write(&foo_py, "print('Hello, world!')")?;
let foo_module = db.resolve_module(ModuleName::new("foo"))?.unwrap();
let foo_module = db.resolve_module(ModuleName::new("foo")).unwrap();
assert_eq!(&src, foo_module.path(&db)?.root());
assert_eq!(&foo_init, &*db.file_path(foo_module.path(&db)?.file()));
assert_eq!(ModuleKind::Package, foo_module.kind(&db)?);
assert_eq!(&src, foo_module.path(&db).root());
assert_eq!(&foo_init, &*db.file_path(foo_module.path(&db).file()));
assert_eq!(ModuleKind::Package, foo_module.kind(&db));
assert_eq!(Some(foo_module), db.path_to_module(&foo_init)?);
assert_eq!(None, db.path_to_module(&foo_py)?);
assert_eq!(Some(foo_module), db.path_to_module(&foo_init));
assert_eq!(None, db.path_to_module(&foo_py));
Ok(())
}
#[test]
fn typing_stub_over_module() -> anyhow::Result<()> {
fn typing_stub_over_module() -> std::io::Result<()> {
let TestCase {
db,
src,
@@ -815,19 +787,19 @@ mod tests {
std::fs::write(&foo_stub, "x: int")?;
std::fs::write(&foo_py, "print('Hello, world!')")?;
let foo = db.resolve_module(ModuleName::new("foo"))?.unwrap();
let foo = db.resolve_module(ModuleName::new("foo")).unwrap();
assert_eq!(&src, foo.path(&db)?.root());
assert_eq!(&foo_stub, &*db.file_path(foo.path(&db)?.file()));
assert_eq!(&src, foo.path(&db).root());
assert_eq!(&foo_stub, &*db.file_path(foo.path(&db).file()));
assert_eq!(Some(foo), db.path_to_module(&foo_stub)?);
assert_eq!(None, db.path_to_module(&foo_py)?);
assert_eq!(Some(foo), db.path_to_module(&foo_stub));
assert_eq!(None, db.path_to_module(&foo_py));
Ok(())
}
#[test]
fn sub_packages() -> anyhow::Result<()> {
fn sub_packages() -> std::io::Result<()> {
let TestCase {
db,
src,
@@ -844,18 +816,18 @@ mod tests {
std::fs::write(bar.join("__init__.py"), "")?;
std::fs::write(&baz, "print('Hello, world!')")?;
let baz_module = db.resolve_module(ModuleName::new("foo.bar.baz"))?.unwrap();
let baz_module = db.resolve_module(ModuleName::new("foo.bar.baz")).unwrap();
assert_eq!(&src, baz_module.path(&db)?.root());
assert_eq!(&baz, &*db.file_path(baz_module.path(&db)?.file()));
assert_eq!(&src, baz_module.path(&db).root());
assert_eq!(&baz, &*db.file_path(baz_module.path(&db).file()));
assert_eq!(Some(baz_module), db.path_to_module(&baz)?);
assert_eq!(Some(baz_module), db.path_to_module(&baz));
Ok(())
}
#[test]
fn namespace_package() -> anyhow::Result<()> {
fn namespace_package() -> std::io::Result<()> {
let TestCase {
db,
temp_dir: _,
@@ -891,21 +863,21 @@ mod tests {
std::fs::write(&two, "print('Hello, world!')")?;
let one_module = db
.resolve_module(ModuleName::new("parent.child.one"))?
.resolve_module(ModuleName::new("parent.child.one"))
.unwrap();
assert_eq!(Some(one_module), db.path_to_module(&one)?);
assert_eq!(Some(one_module), db.path_to_module(&one));
let two_module = db
.resolve_module(ModuleName::new("parent.child.two"))?
.resolve_module(ModuleName::new("parent.child.two"))
.unwrap();
assert_eq!(Some(two_module), db.path_to_module(&two)?);
assert_eq!(Some(two_module), db.path_to_module(&two));
Ok(())
}
#[test]
fn regular_package_in_namespace_package() -> anyhow::Result<()> {
fn regular_package_in_namespace_package() -> std::io::Result<()> {
let TestCase {
db,
temp_dir: _,
@@ -942,20 +914,17 @@ mod tests {
std::fs::write(two, "print('Hello, world!')")?;
let one_module = db
.resolve_module(ModuleName::new("parent.child.one"))?
.resolve_module(ModuleName::new("parent.child.one"))
.unwrap();
assert_eq!(Some(one_module), db.path_to_module(&one)?);
assert_eq!(Some(one_module), db.path_to_module(&one));
assert_eq!(
None,
db.resolve_module(ModuleName::new("parent.child.two"))?
);
assert_eq!(None, db.resolve_module(ModuleName::new("parent.child.two")));
Ok(())
}
#[test]
fn module_search_path_priority() -> anyhow::Result<()> {
fn module_search_path_priority() -> std::io::Result<()> {
let TestCase {
db,
src,
@@ -969,20 +938,20 @@ mod tests {
std::fs::write(&foo_src, "")?;
std::fs::write(&foo_site_packages, "")?;
let foo_module = db.resolve_module(ModuleName::new("foo"))?.unwrap();
let foo_module = db.resolve_module(ModuleName::new("foo")).unwrap();
assert_eq!(&src, foo_module.path(&db)?.root());
assert_eq!(&foo_src, &*db.file_path(foo_module.path(&db)?.file()));
assert_eq!(&src, foo_module.path(&db).root());
assert_eq!(&foo_src, &*db.file_path(foo_module.path(&db).file()));
assert_eq!(Some(foo_module), db.path_to_module(&foo_src)?);
assert_eq!(None, db.path_to_module(&foo_site_packages)?);
assert_eq!(Some(foo_module), db.path_to_module(&foo_src));
assert_eq!(None, db.path_to_module(&foo_site_packages));
Ok(())
}
#[test]
#[cfg(target_family = "unix")]
fn symlink() -> anyhow::Result<()> {
fn symlink() -> std::io::Result<()> {
let TestCase {
db,
src,
@@ -996,28 +965,28 @@ mod tests {
std::fs::write(&foo, "")?;
std::os::unix::fs::symlink(&foo, &bar)?;
let foo_module = db.resolve_module(ModuleName::new("foo"))?.unwrap();
let bar_module = db.resolve_module(ModuleName::new("bar"))?.unwrap();
let foo_module = db.resolve_module(ModuleName::new("foo")).unwrap();
let bar_module = db.resolve_module(ModuleName::new("bar")).unwrap();
assert_ne!(foo_module, bar_module);
assert_eq!(&src, foo_module.path(&db)?.root());
assert_eq!(&foo, &*db.file_path(foo_module.path(&db)?.file()));
assert_eq!(&src, foo_module.path(&db).root());
assert_eq!(&foo, &*db.file_path(foo_module.path(&db).file()));
// Bar has a different name but it should point to the same file.
assert_eq!(&src, bar_module.path(&db)?.root());
assert_eq!(foo_module.path(&db)?.file(), bar_module.path(&db)?.file());
assert_eq!(&foo, &*db.file_path(bar_module.path(&db)?.file()));
assert_eq!(&src, bar_module.path(&db).root());
assert_eq!(foo_module.path(&db).file(), bar_module.path(&db).file());
assert_eq!(&foo, &*db.file_path(bar_module.path(&db).file()));
assert_eq!(Some(foo_module), db.path_to_module(&foo)?);
assert_eq!(Some(bar_module), db.path_to_module(&bar)?);
assert_eq!(Some(foo_module), db.path_to_module(&foo));
assert_eq!(Some(bar_module), db.path_to_module(&bar));
Ok(())
}
#[test]
fn resolve_dependency() -> anyhow::Result<()> {
fn resolve_dependency() -> std::io::Result<()> {
let TestCase {
src,
db,
@@ -1033,8 +1002,8 @@ mod tests {
std::fs::write(foo_path, "from .bar import test")?;
std::fs::write(bar_path, "test = 'Hello world'")?;
let foo_module = db.resolve_module(ModuleName::new("foo"))?.unwrap();
let bar_module = db.resolve_module(ModuleName::new("foo.bar"))?.unwrap();
let foo_module = db.resolve_module(ModuleName::new("foo")).unwrap();
let bar_module = db.resolve_module(ModuleName::new("foo.bar")).unwrap();
// `from . import bar` in `foo/__init__.py` resolves to `foo`
assert_eq!(
@@ -1045,13 +1014,13 @@ mod tests {
level: NonZeroU32::new(1).unwrap(),
module: None,
}
)?
)
);
// `from baz import bar` in `foo/__init__.py` should resolve to `baz.py`
assert_eq!(
Some(ModuleName::new("baz")),
foo_module.resolve_dependency(&db, &Dependency::Module(ModuleName::new("baz")))?
foo_module.resolve_dependency(&db, &Dependency::Module(ModuleName::new("baz")))
);
// from .bar import test in `foo/__init__.py` should resolve to `foo/bar.py`
@@ -1063,7 +1032,7 @@ mod tests {
level: NonZeroU32::new(1).unwrap(),
module: Some(ModuleName::new("bar"))
}
)?
)
);
// from .. import test in `foo/__init__.py` resolves to `` which is not a module
@@ -1075,7 +1044,7 @@ mod tests {
level: NonZeroU32::new(2).unwrap(),
module: None
}
)?
)
);
// `from . import test` in `foo/bar.py` resolves to `foo`
@@ -1087,13 +1056,13 @@ mod tests {
level: NonZeroU32::new(1).unwrap(),
module: None
}
)?
)
);
// `from baz import test` in `foo/bar.py` resolves to `baz`
assert_eq!(
Some(ModuleName::new("baz")),
bar_module.resolve_dependency(&db, &Dependency::Module(ModuleName::new("baz")))?
bar_module.resolve_dependency(&db, &Dependency::Module(ModuleName::new("baz")))
);
// `from .baz import test` in `foo/bar.py` resolves to `foo.baz`.
@@ -1105,7 +1074,7 @@ mod tests {
level: NonZeroU32::new(1).unwrap(),
module: Some(ModuleName::new("baz"))
}
)?
)
);
Ok(())

View File

@@ -6,7 +6,7 @@ use ruff_python_parser::{Mode, ParseError};
use ruff_text_size::{Ranged, TextRange};
use crate::cache::KeyValueCache;
use crate::db::{HasJar, QueryResult, SourceDb, SourceJar};
use crate::db::{HasJar, SourceDb, SourceJar};
use crate::files::FileId;
#[derive(Debug, Clone, PartialEq)]
@@ -64,16 +64,16 @@ impl Parsed {
}
#[tracing::instrument(level = "debug", skip(db))]
pub(crate) fn parse<Db>(db: &Db, file_id: FileId) -> QueryResult<Parsed>
pub(crate) fn parse<Db>(db: &Db, file_id: FileId) -> Parsed
where
Db: SourceDb + HasJar<SourceJar>,
{
let parsed = db.jar()?;
let parsed = db.jar();
parsed.parsed.get(&file_id, |file_id| {
let source = db.source(*file_id)?;
let source = db.source(*file_id);
Ok(Parsed::from_text(source.text()))
Parsed::from_text(source.text())
})
}

View File

@@ -1,9 +1,11 @@
use rayon::{current_num_threads, yield_local};
use std::num::NonZeroUsize;
use rayon::max_num_threads;
use rustc_hash::FxHashSet;
use crate::db::{Database, LintDb, QueryError, QueryResult, SemanticDb};
use crate::cancellation::CancellationToken;
use crate::db::{SemanticDb, SourceDb};
use crate::files::FileId;
use crate::format::{FormatDb, FormatError};
use crate::lint::Diagnostics;
use crate::program::Program;
use crate::symbols::Dependency;
@@ -11,28 +13,42 @@ use crate::symbols::Dependency;
impl Program {
/// Checks all open files in the workspace and its dependencies.
#[tracing::instrument(level = "debug", skip_all)]
pub fn check(&self, mode: ExecutionMode) -> QueryResult<Vec<String>> {
self.cancelled()?;
pub fn check(
&self,
scheduler: &dyn CheckScheduler,
cancellation_token: CancellationToken,
) -> Result<Vec<String>, CheckError> {
let check_loop = CheckFilesLoop::new(scheduler, cancellation_token);
let mut context = CheckContext::new(self);
check_loop.run(self.workspace().open_files.iter().copied())
}
match mode {
ExecutionMode::SingleThreaded => SingleThreadedExecutor.run(&mut context)?,
ExecutionMode::ThreadPool => ThreadPoolExecutor.run(&mut context)?,
};
/// Checks a single file and its dependencies.
#[tracing::instrument(level = "debug", skip(self, scheduler, cancellation_token))]
pub fn check_file(
&self,
file: FileId,
scheduler: &dyn CheckScheduler,
cancellation_token: CancellationToken,
) -> Result<Vec<String>, CheckError> {
let check_loop = CheckFilesLoop::new(scheduler, cancellation_token);
Ok(context.finish())
check_loop.run([file].into_iter())
}
#[tracing::instrument(level = "debug", skip(self, context))]
fn check_file(&self, file: FileId, context: &CheckFileContext) -> QueryResult<Diagnostics> {
self.cancelled()?;
fn do_check_file(
&self,
file: FileId,
context: &CheckContext,
) -> Result<Diagnostics, CheckError> {
context.cancelled_ok()?;
let symbol_table = self.symbol_table(file)?;
let symbol_table = self.symbol_table(file);
let dependencies = symbol_table.dependencies();
if !dependencies.is_empty() {
let module = self.file_to_module(file)?;
let module = self.file_to_module(file);
// TODO scheduling all dependencies here is wasteful if we don't infer any types on them
// but I think that's unlikely, so it is okay?
@@ -41,19 +57,18 @@ impl Program {
for dependency in dependencies {
let dependency_name = match dependency {
Dependency::Module(name) => Some(name.clone()),
Dependency::Relative { .. } => match &module {
Some(module) => module.resolve_dependency(self, dependency)?,
None => None,
},
Dependency::Relative { .. } => module
.as_ref()
.and_then(|module| module.resolve_dependency(self, dependency)),
};
if let Some(dependency_name) = dependency_name {
// TODO We may want to have a different check functions for non-first-party
// files because we only need to index them and not check them.
// Supporting non-first-party code also requires supporting typing stubs.
if let Some(dependency) = self.resolve_module(dependency_name)? {
if dependency.path(self)?.root().kind().is_first_party() {
context.schedule_dependency(dependency.path(self)?.file());
if let Some(dependency) = self.resolve_module(dependency_name) {
if dependency.path(self).root().kind().is_first_party() {
context.schedule_check_file(dependency.path(self).file());
}
}
}
@@ -63,363 +78,238 @@ impl Program {
let mut diagnostics = Vec::new();
if self.workspace().is_file_open(file) {
diagnostics.extend_from_slice(&self.lint_syntax(file)?);
diagnostics.extend_from_slice(&self.lint_semantic(file)?);
match self.check_file_formatted(file) {
Ok(format_diagnostics) => {
diagnostics.extend_from_slice(&format_diagnostics);
}
Err(FormatError::Query(err)) => {
return Err(err);
}
Err(FormatError::Format(error)) => {
diagnostics.push(format!("Error formatting file: {error}"));
}
}
diagnostics.extend_from_slice(&self.lint_syntax(file));
diagnostics.extend_from_slice(&self.lint_semantic(file));
}
Ok(Diagnostics::from(diagnostics))
}
}
#[derive(Copy, Clone, Debug, PartialEq, Eq)]
pub enum ExecutionMode {
SingleThreaded,
ThreadPool,
}
/// Context that stores state information about the entire check operation.
struct CheckContext<'a> {
/// IDs of the files that have been queued for checking.
/// Schedules checks for files.
pub trait CheckScheduler {
/// Schedules a check for a file.
///
/// Used to avoid queuing the same file twice.
scheduled_files: FxHashSet<FileId>,
/// The check can either be run immediately on the current thread or the check can be queued
/// in a thread pool and ran asynchronously.
///
/// The order in which scheduled checks are executed is not guaranteed.
///
/// The implementation should call [`CheckFileTask::run`] to execute the check.
fn check_file(&self, file_task: CheckFileTask);
/// Reference to the program that is checked.
program: &'a Program,
/// The aggregated diagnostics
diagnostics: Vec<String>,
/// The maximum number of checks that can be run concurrently.
///
/// Returns `None` if the checks run on the current thread (no concurrency).
fn max_concurrency(&self) -> Option<NonZeroUsize>;
}
impl<'a> CheckContext<'a> {
fn new(program: &'a Program) -> Self {
Self {
scheduled_files: FxHashSet::default(),
program,
diagnostics: Vec::new(),
}
}
/// Scheduler that runs checks on a rayon thread pool.
pub struct RayonCheckScheduler<'program, 'scope_ref, 'scope> {
program: &'program Program,
scope: &'scope_ref rayon::Scope<'scope>,
}
/// Returns the tasks to check all open files in the workspace.
fn check_open_files(&mut self) -> Vec<CheckOpenFileTask> {
self.scheduled_files
.extend(self.program.workspace().open_files());
self.program
.workspace()
.open_files()
.map(|file_id| CheckOpenFileTask { file_id })
.collect()
}
/// Returns the task to check a dependency.
fn check_dependency(&mut self, file_id: FileId) -> Option<CheckDependencyTask> {
if self.scheduled_files.insert(file_id) {
Some(CheckDependencyTask { file_id })
} else {
None
}
}
/// Pushes the result for a single file check operation
fn push_diagnostics(&mut self, diagnostics: &Diagnostics) {
self.diagnostics.extend_from_slice(diagnostics);
}
/// Returns a reference to the program that is being checked.
fn program(&self) -> &'a Program {
self.program
}
/// Creates a task context that is used to check a single file.
fn task_context<'b, S>(&self, dependency_scheduler: &'b S) -> CheckTaskContext<'a, 'b, S>
where
S: ScheduleDependency,
{
CheckTaskContext {
program: self.program,
dependency_scheduler,
}
}
fn finish(self) -> Vec<String> {
self.diagnostics
impl<'program, 'scope_ref, 'scope> RayonCheckScheduler<'program, 'scope_ref, 'scope> {
pub fn new(program: &'program Program, scope: &'scope_ref rayon::Scope<'scope>) -> Self {
Self { program, scope }
}
}
/// Trait that abstracts away how a dependency of a file gets scheduled for checking.
trait ScheduleDependency {
/// Schedules the file with the given ID for checking.
fn schedule(&self, file_id: FileId);
}
impl<T> ScheduleDependency for T
impl<'program, 'scope_ref, 'scope> CheckScheduler
for RayonCheckScheduler<'program, 'scope_ref, 'scope>
where
T: Fn(FileId),
'program: 'scope,
{
fn schedule(&self, file_id: FileId) {
let f = self;
f(file_id);
fn check_file(&self, check_file_task: CheckFileTask) {
let child_span =
tracing::trace_span!("check_file", file_id = check_file_task.file_id.as_u32());
let program = self.program;
self.scope
.spawn(move |_| child_span.in_scope(|| check_file_task.run(program)));
}
fn max_concurrency(&self) -> Option<NonZeroUsize> {
Some(NonZeroUsize::new(max_num_threads()).unwrap_or(NonZeroUsize::MIN))
}
}
/// Context that is used to run a single file check task.
///
/// The task is generic over `S` because it is passed across thread boundaries and
/// we don't want to add the requirement that [`ScheduleDependency`] must be [`Send`].
struct CheckTaskContext<'a, 'scheduler, S>
where
S: ScheduleDependency,
{
dependency_scheduler: &'scheduler S,
/// Scheduler that runs all checks on the current thread.
pub struct SameThreadCheckScheduler<'a> {
program: &'a Program,
}
impl<'a, 'scheduler, S> CheckTaskContext<'a, 'scheduler, S>
where
S: ScheduleDependency,
{
fn as_file_context(&self) -> CheckFileContext<'scheduler> {
CheckFileContext {
dependency_scheduler: self.dependency_scheduler,
}
impl<'a> SameThreadCheckScheduler<'a> {
pub fn new(program: &'a Program) -> Self {
Self { program }
}
}
/// Context passed when checking a single file.
///
/// This is a trimmed down version of [`CheckTaskContext`] with the type parameter `S` erased
/// to avoid monomorphization of [`Program:check_file`].
struct CheckFileContext<'a> {
dependency_scheduler: &'a dyn ScheduleDependency,
impl CheckScheduler for SameThreadCheckScheduler<'_> {
fn check_file(&self, task: CheckFileTask) {
task.run(self.program);
}
fn max_concurrency(&self) -> Option<NonZeroUsize> {
None
}
}
impl<'a> CheckFileContext<'a> {
fn schedule_dependency(&self, file_id: FileId) {
self.dependency_scheduler.schedule(file_id);
}
#[derive(Debug, Clone)]
pub enum CheckError {
Cancelled,
}
#[derive(Debug)]
enum CheckFileTask {
OpenFile(CheckOpenFileTask),
Dependency(CheckDependencyTask),
pub struct CheckFileTask {
file_id: FileId,
context: CheckContext,
}
impl CheckFileTask {
/// Runs the task and returns the results for checking this file.
fn run<S>(&self, context: &CheckTaskContext<S>) -> QueryResult<Diagnostics>
where
S: ScheduleDependency,
{
match self {
Self::OpenFile(task) => task.run(context),
Self::Dependency(task) => task.run(context),
}
}
fn file_id(&self) -> FileId {
match self {
CheckFileTask::OpenFile(task) => task.file_id,
CheckFileTask::Dependency(task) => task.file_id,
/// Runs the check and communicates the result to the orchestrator.
pub fn run(self, program: &Program) {
match program.do_check_file(self.file_id, &self.context) {
Ok(diagnostics) => self
.context
.sender
.send(CheckFileMessage::Completed(diagnostics))
.unwrap(),
Err(CheckError::Cancelled) => self
.context
.sender
.send(CheckFileMessage::Cancelled)
.unwrap(),
}
}
}
/// Task to check an open file.
#[derive(Debug)]
struct CheckOpenFileTask {
file_id: FileId,
#[derive(Clone, Debug)]
struct CheckContext {
cancellation_token: CancellationToken,
sender: crossbeam_channel::Sender<CheckFileMessage>,
}
impl CheckOpenFileTask {
fn run<S>(&self, context: &CheckTaskContext<S>) -> QueryResult<Diagnostics>
where
S: ScheduleDependency,
{
context
.program
.check_file(self.file_id, &context.as_file_context())
impl CheckContext {
fn new(
cancellation_token: CancellationToken,
sender: crossbeam_channel::Sender<CheckFileMessage>,
) -> Self {
Self {
cancellation_token,
sender,
}
}
/// Queues a new file for checking using the [`CheckScheduler`].
#[allow(unused)]
fn schedule_check_file(&self, file_id: FileId) {
self.sender.send(CheckFileMessage::Queue(file_id)).unwrap();
}
/// Returns `true` if the check has been cancelled.
fn is_cancelled(&self) -> bool {
self.cancellation_token.is_cancelled()
}
fn cancelled_ok(&self) -> Result<(), CheckError> {
if self.is_cancelled() {
Err(CheckError::Cancelled)
} else {
Ok(())
}
}
}
/// Task to check a dependency file.
#[derive(Debug)]
struct CheckDependencyTask {
file_id: FileId,
struct CheckFilesLoop<'a> {
scheduler: &'a dyn CheckScheduler,
cancellation_token: CancellationToken,
pending: usize,
queued_files: FxHashSet<FileId>,
}
impl CheckDependencyTask {
fn run<S>(&self, context: &CheckTaskContext<S>) -> QueryResult<Diagnostics>
where
S: ScheduleDependency,
{
context
.program
.check_file(self.file_id, &context.as_file_context())
impl<'a> CheckFilesLoop<'a> {
fn new(scheduler: &'a dyn CheckScheduler, cancellation_token: CancellationToken) -> Self {
Self {
scheduler,
cancellation_token,
queued_files: FxHashSet::default(),
pending: 0,
}
}
}
/// Executor that schedules the checking of individual program files.
trait CheckExecutor {
fn run(self, context: &mut CheckContext) -> QueryResult<()>;
}
fn run(mut self, files: impl Iterator<Item = FileId>) -> Result<Vec<String>, CheckError> {
let (sender, receiver) = if let Some(max_concurrency) = self.scheduler.max_concurrency() {
crossbeam_channel::bounded(max_concurrency.get())
} else {
// The checks run on the current thread. That means it is necessary to store all messages
// or we risk deadlocking when the main loop never gets a chance to read the messages.
crossbeam_channel::unbounded()
};
/// Executor that runs all check operations on the current thread.
///
/// The executor does not schedule dependencies for checking.
/// The main motivation for scheduling dependencies
/// in a multithreaded environment is to parse and index the dependencies concurrently.
/// However, that doesn't make sense in a single threaded environment, because the dependencies then compute
/// with checking the open files. Checking dependencies in a single threaded environment is more likely
/// to hurt performance because we end up analyzing files in their entirety, even if we only need to type check parts of them.
#[derive(Debug, Default)]
struct SingleThreadedExecutor;
let context = CheckContext::new(self.cancellation_token.clone(), sender.clone());
impl CheckExecutor for SingleThreadedExecutor {
fn run(self, context: &mut CheckContext) -> QueryResult<()> {
let mut queue = context.check_open_files();
for file in files {
self.queue_file(file, context.clone())?;
}
let noop_schedule_dependency = |_| {};
self.run_impl(receiver, &context)
}
while let Some(file) = queue.pop() {
context.program().cancelled()?;
fn run_impl(
mut self,
receiver: crossbeam_channel::Receiver<CheckFileMessage>,
context: &CheckContext,
) -> Result<Vec<String>, CheckError> {
if self.cancellation_token.is_cancelled() {
return Err(CheckError::Cancelled);
}
let task_context = context.task_context(&noop_schedule_dependency);
context.push_diagnostics(&file.run(&task_context)?);
let mut result = Vec::default();
for message in receiver {
match message {
CheckFileMessage::Completed(diagnostics) => {
result.extend_from_slice(&diagnostics);
self.pending -= 1;
if self.pending == 0 {
break;
}
}
CheckFileMessage::Queue(id) => {
self.queue_file(id, context.clone())?;
}
CheckFileMessage::Cancelled => {
return Err(CheckError::Cancelled);
}
}
}
Ok(result)
}
fn queue_file(&mut self, file_id: FileId, context: CheckContext) -> Result<(), CheckError> {
if context.is_cancelled() {
return Err(CheckError::Cancelled);
}
if self.queued_files.insert(file_id) {
self.pending += 1;
self.scheduler
.check_file(CheckFileTask { file_id, context });
}
Ok(())
}
}
/// Executor that runs the check operations on a thread pool.
///
/// The executor runs each check operation as its own task using a thread pool.
///
/// Other than [`SingleThreadedExecutor`], this executor schedules dependencies for checking. It
/// even schedules dependencies for checking when the thread pool size is 1 for a better debugging experience.
#[derive(Debug, Default)]
struct ThreadPoolExecutor;
impl CheckExecutor for ThreadPoolExecutor {
fn run(self, context: &mut CheckContext) -> QueryResult<()> {
let num_threads = current_num_threads();
let single_threaded = num_threads == 1;
let span = tracing::trace_span!("ThreadPoolExecutor::run", num_threads);
let _ = span.enter();
let mut queue: Vec<_> = context
.check_open_files()
.into_iter()
.map(CheckFileTask::OpenFile)
.collect();
let (sender, receiver) = if single_threaded {
// Use an unbounded queue for single threaded execution to prevent deadlocks
// when a single file schedules multiple dependencies.
crossbeam::channel::unbounded()
} else {
// Use a bounded queue to apply backpressure when the orchestration thread isn't able to keep
// up processing messages from the worker threads.
crossbeam::channel::bounded(num_threads)
};
let schedule_sender = sender.clone();
let schedule_dependency = move |file_id| {
schedule_sender
.send(ThreadPoolMessage::ScheduleDependency(file_id))
.unwrap();
};
let result = rayon::in_place_scope(|scope| {
let mut pending = 0usize;
loop {
context.program().cancelled()?;
// 1. Try to get a queued message to ensure that we have always remaining space in the channel to prevent blocking the worker threads.
// 2. Try to process a queued file
// 3. If there's no queued file wait for the next incoming message.
// 4. Exit if there are no more messages and no senders.
let message = if let Ok(message) = receiver.try_recv() {
message
} else if let Some(task) = queue.pop() {
pending += 1;
let task_context = context.task_context(&schedule_dependency);
let sender = sender.clone();
let task_span = tracing::trace_span!(
parent: &span,
"CheckFileTask::run",
file_id = task.file_id().as_u32(),
);
scope.spawn(move |_| {
task_span.in_scope(|| match task.run(&task_context) {
Ok(result) => {
sender.send(ThreadPoolMessage::Completed(result)).unwrap();
}
Err(err) => sender.send(ThreadPoolMessage::Errored(err)).unwrap(),
});
});
// If this is a single threaded rayon thread pool, yield the current thread
// or we never start processing the work items.
if single_threaded {
yield_local();
}
continue;
} else if let Ok(message) = receiver.recv() {
message
} else {
break;
};
match message {
ThreadPoolMessage::ScheduleDependency(dependency) => {
if let Some(task) = context.check_dependency(dependency) {
queue.push(CheckFileTask::Dependency(task));
}
}
ThreadPoolMessage::Completed(diagnostics) => {
context.push_diagnostics(&diagnostics);
pending -= 1;
if pending == 0 && queue.is_empty() {
break;
}
}
ThreadPoolMessage::Errored(err) => {
return Err(err);
}
}
}
Ok(())
});
result
}
}
#[derive(Debug)]
enum ThreadPoolMessage {
ScheduleDependency(FileId),
enum CheckFileMessage {
Completed(Diagnostics),
Errored(QueryError),
Queue(FileId),
Cancelled,
}

View File

@@ -1,44 +0,0 @@
use crate::db::{QueryResult, SourceDb};
use crate::format::{FormatDb, FormatError, FormattedFile};
use crate::program::Program;
impl Program {
#[tracing::instrument(level = "trace", skip(self))]
pub fn format(&mut self) -> QueryResult<()> {
// Formats all open files
// TODO make `Executor` from `check` reusable.
for file in self.workspace.open_files() {
match self.format_file(file) {
Ok(FormattedFile::Formatted(content)) => {
let path = self.file_path(file);
// TODO: This is problematic because it immediately re-triggers the file watcher.
// A possible solution is to track the self "inflicted" changes inside of programs
// by tracking the file revision right after the write. It could then use the revision
// to determine which changes are safe to ignore (and in which context).
// An other alternative is to not write as part of the `format` command and instead
// return a Vec with the format results and leave the writing to the caller.
// I think that's undesired because a) we still need a way to tell the formatter
// that it won't be necessary to format the content again and
// b) it would reduce concurrency because the writing would need to wait for the file
// formatting to be complete, unless we use some form of communication channel.
std::fs::write(path, content).expect("Unable to write file");
}
Ok(FormattedFile::Unchanged) => {
// No op
}
Err(FormatError::Query(error)) => {
return Err(error);
}
Err(FormatError::Format(error)) => {
// TODO proper error handling. We should either propagate this error or
// emit a diagnostic (probably this).
tracing::warn!("Failed to format file: {}", error);
}
}
}
Ok(())
}
}

View File

@@ -1,42 +1,45 @@
use ruff_formatter::PrintedRange;
use ruff_text_size::TextRange;
pub mod check;
use std::path::Path;
use std::sync::Arc;
use crate::db::{
Database, Db, DbRuntime, HasJar, HasJars, JarsStorage, LintDb, LintJar, ParallelDatabase,
QueryResult, SemanticDb, SemanticJar, Snapshot, SourceDb, SourceJar,
};
use crate::db::{Db, HasJar, SemanticDb, SemanticJar, SourceDb, SourceJar};
use crate::files::{FileId, Files};
use crate::format::{
check_formatted, format_file, format_file_range, FormatDb, FormatError, FormatJar,
FormattedFile,
use crate::lint::{
lint_semantic, lint_syntax, Diagnostics, LintSemanticStorage, LintSyntaxStorage,
};
use crate::lint::{lint_semantic, lint_syntax, Diagnostics};
use crate::module::{
add_module, file_to_module, path_to_module, resolve_module, set_module_search_paths, Module,
ModuleData, ModuleName, ModuleSearchPath,
ModuleData, ModuleName, ModuleResolver, ModuleSearchPath,
};
use crate::parse::{parse, Parsed};
use crate::source::{source_text, Source};
use crate::symbols::{symbol_table, SymbolId, SymbolTable};
use crate::types::{infer_symbol_type, Type};
use crate::parse::{parse, Parsed, ParsedStorage};
use crate::source::{source_text, Source, SourceStorage};
use crate::symbols::{symbol_table, SymbolId, SymbolTable, SymbolTablesStorage};
use crate::types::{infer_symbol_type, Type, TypeStore};
use crate::Workspace;
pub mod check;
mod format;
#[derive(Debug)]
pub struct Program {
jars: JarsStorage<Program>,
files: Files,
source: SourceJar,
semantic: SemanticJar,
workspace: Workspace,
}
impl Program {
pub fn new(workspace: Workspace) -> Self {
pub fn new(workspace: Workspace, module_search_paths: Vec<ModuleSearchPath>) -> Self {
Self {
jars: JarsStorage::default(),
source: SourceJar {
sources: SourceStorage::default(),
parsed: ParsedStorage::default(),
lint_syntax: LintSyntaxStorage::default(),
},
semantic: SemanticJar {
module_resolver: ModuleResolver::new(module_search_paths),
symbol_tables: SymbolTablesStorage::default(),
type_store: TypeStore::default(),
lint_semantic: LintSemanticStorage::default(),
},
files: Files::default(),
workspace,
}
@@ -46,17 +49,17 @@ impl Program {
where
I: IntoIterator<Item = FileChange>,
{
let (source, semantic, lint, format) = self.jars_mut();
for change in changes {
semantic.module_resolver.remove_module(change.id);
semantic.symbol_tables.remove(&change.id);
source.sources.remove(&change.id);
source.parsed.remove(&change.id);
self.semantic
.module_resolver
.remove_module(&self.file_path(change.id));
self.semantic.symbol_tables.remove(&change.id);
self.source.sources.remove(&change.id);
self.source.parsed.remove(&change.id);
self.source.lint_syntax.remove(&change.id);
// TODO: remove all dependent modules as well
semantic.type_store.remove_module(change.id);
lint.lint_syntax.remove(&change.id);
lint.lint_semantic.remove(&change.id);
format.formatted.remove(&change.id);
self.semantic.type_store.remove_module(change.id);
self.semantic.lint_semantic.remove(&change.id);
}
}
@@ -82,36 +85,44 @@ impl SourceDb for Program {
self.files.path(file_id)
}
fn source(&self, file_id: FileId) -> QueryResult<Source> {
fn source(&self, file_id: FileId) -> Source {
source_text(self, file_id)
}
fn parse(&self, file_id: FileId) -> QueryResult<Parsed> {
fn parse(&self, file_id: FileId) -> Parsed {
parse(self, file_id)
}
fn lint_syntax(&self, file_id: FileId) -> Diagnostics {
lint_syntax(self, file_id)
}
}
impl SemanticDb for Program {
fn resolve_module(&self, name: ModuleName) -> QueryResult<Option<Module>> {
fn resolve_module(&self, name: ModuleName) -> Option<Module> {
resolve_module(self, name)
}
fn file_to_module(&self, file_id: FileId) -> QueryResult<Option<Module>> {
fn file_to_module(&self, file_id: FileId) -> Option<Module> {
file_to_module(self, file_id)
}
fn path_to_module(&self, path: &Path) -> QueryResult<Option<Module>> {
fn path_to_module(&self, path: &Path) -> Option<Module> {
path_to_module(self, path)
}
fn symbol_table(&self, file_id: FileId) -> QueryResult<Arc<SymbolTable>> {
fn symbol_table(&self, file_id: FileId) -> Arc<SymbolTable> {
symbol_table(self, file_id)
}
fn infer_symbol_type(&self, file_id: FileId, symbol_id: SymbolId) -> QueryResult<Type> {
fn infer_symbol_type(&self, file_id: FileId, symbol_id: SymbolId) -> Type {
infer_symbol_type(self, file_id, symbol_id)
}
fn lint_semantic(&self, file_id: FileId) -> Diagnostics {
lint_semantic(self, file_id)
}
// Mutations
fn add_module(&mut self, path: &Path) -> Option<(Module, Vec<Arc<ModuleData>>)> {
add_module(self, path)
@@ -122,105 +133,25 @@ impl SemanticDb for Program {
}
}
impl LintDb for Program {
fn lint_syntax(&self, file_id: FileId) -> QueryResult<Diagnostics> {
lint_syntax(self, file_id)
}
fn lint_semantic(&self, file_id: FileId) -> QueryResult<Diagnostics> {
lint_semantic(self, file_id)
}
}
impl FormatDb for Program {
fn format_file(&self, file_id: FileId) -> Result<FormattedFile, FormatError> {
format_file(self, file_id)
}
fn format_file_range(
&self,
file_id: FileId,
range: TextRange,
) -> Result<PrintedRange, FormatError> {
format_file_range(self, file_id, range)
}
fn check_file_formatted(&self, file_id: FileId) -> Result<Diagnostics, FormatError> {
check_formatted(self, file_id)
}
}
impl Db for Program {}
impl Database for Program {
fn runtime(&self) -> &DbRuntime {
self.jars.runtime()
}
fn runtime_mut(&mut self) -> &mut DbRuntime {
self.jars.runtime_mut()
}
}
impl ParallelDatabase for Program {
fn snapshot(&self) -> Snapshot<Self> {
Snapshot::new(Self {
jars: self.jars.snapshot(),
files: self.files.clone(),
workspace: self.workspace.clone(),
})
}
}
impl HasJars for Program {
type Jars = (SourceJar, SemanticJar, LintJar, FormatJar);
fn jars(&self) -> QueryResult<&Self::Jars> {
self.jars.jars()
}
fn jars_mut(&mut self) -> &mut Self::Jars {
self.jars.jars_mut()
}
}
impl HasJar<SourceJar> for Program {
fn jar(&self) -> QueryResult<&SourceJar> {
Ok(&self.jars()?.0)
fn jar(&self) -> &SourceJar {
&self.source
}
fn jar_mut(&mut self) -> &mut SourceJar {
&mut self.jars_mut().0
&mut self.source
}
}
impl HasJar<SemanticJar> for Program {
fn jar(&self) -> QueryResult<&SemanticJar> {
Ok(&self.jars()?.1)
fn jar(&self) -> &SemanticJar {
&self.semantic
}
fn jar_mut(&mut self) -> &mut SemanticJar {
&mut self.jars_mut().1
}
}
impl HasJar<LintJar> for Program {
fn jar(&self) -> QueryResult<&LintJar> {
Ok(&self.jars()?.2)
}
fn jar_mut(&mut self) -> &mut LintJar {
&mut self.jars_mut().2
}
}
impl HasJar<FormatJar> for Program {
fn jar(&self) -> QueryResult<&FormatJar> {
Ok(&self.jars()?.3)
}
fn jar_mut(&mut self) -> &mut FormatJar {
&mut self.jars_mut().3
&mut self.semantic
}
}

View File

@@ -1,5 +1,5 @@
use crate::cache::KeyValueCache;
use crate::db::{HasJar, QueryResult, SourceDb, SourceJar};
use crate::db::{HasJar, SourceDb, SourceJar};
use ruff_notebook::Notebook;
use ruff_python_ast::PySourceType;
use std::ops::{Deref, DerefMut};
@@ -8,11 +8,11 @@ use std::sync::Arc;
use crate::files::FileId;
#[tracing::instrument(level = "debug", skip(db))]
pub(crate) fn source_text<Db>(db: &Db, file_id: FileId) -> QueryResult<Source>
pub(crate) fn source_text<Db>(db: &Db, file_id: FileId) -> Source
where
Db: SourceDb + HasJar<SourceJar>,
{
let sources = &db.jar()?.sources;
let sources = &db.jar().sources;
sources.get(&file_id, |file_id| {
let path = db.file_path(*file_id);
@@ -43,7 +43,7 @@ where
}
};
Ok(Source { kind })
Source { kind }
})
}

View File

@@ -16,22 +16,22 @@ use ruff_python_ast::visitor::preorder::PreorderVisitor;
use crate::ast_ids::TypedNodeKey;
use crate::cache::KeyValueCache;
use crate::db::{HasJar, QueryResult, SemanticDb, SemanticJar};
use crate::db::{HasJar, SemanticDb, SemanticJar};
use crate::files::FileId;
use crate::module::ModuleName;
use crate::Name;
#[allow(unreachable_pub)]
#[tracing::instrument(level = "debug", skip(db))]
pub fn symbol_table<Db>(db: &Db, file_id: FileId) -> QueryResult<Arc<SymbolTable>>
pub fn symbol_table<Db>(db: &Db, file_id: FileId) -> Arc<SymbolTable>
where
Db: SemanticDb + HasJar<SemanticJar>,
{
let jar = db.jar()?;
let jar = db.jar();
jar.symbol_tables.get(&file_id, |_| {
let parsed = db.parse(file_id)?;
Ok(Arc::from(SymbolTable::from_ast(parsed.ast())))
let parsed = db.parse(file_id);
Arc::from(SymbolTable::from_ast(parsed.ast()))
})
}

View File

@@ -2,7 +2,7 @@
use ruff_python_ast::AstNode;
use crate::db::{HasJar, QueryResult, SemanticDb, SemanticJar};
use crate::db::{HasJar, SemanticDb, SemanticJar};
use crate::module::ModuleName;
use crate::symbols::{Definition, ImportFromDefinition, SymbolId};
use crate::types::Type;
@@ -11,24 +11,23 @@ use ruff_python_ast as ast;
// FIXME: Figure out proper dead-lock free synchronisation now that this takes `&db` instead of `&mut db`.
#[tracing::instrument(level = "trace", skip(db))]
pub fn infer_symbol_type<Db>(db: &Db, file_id: FileId, symbol_id: SymbolId) -> QueryResult<Type>
pub fn infer_symbol_type<Db>(db: &Db, file_id: FileId, symbol_id: SymbolId) -> Type
where
Db: SemanticDb + HasJar<SemanticJar>,
{
let symbols = db.symbol_table(file_id)?;
let symbols = db.symbol_table(file_id);
let defs = symbols.definitions(symbol_id);
if let Some(ty) = db
.jar()?
.jar()
.type_store
.get_cached_symbol_type(file_id, symbol_id)
{
return Ok(ty);
return ty;
}
// TODO handle multiple defs, conditional defs...
assert_eq!(defs.len(), 1);
let type_store = &db.jar()?.type_store;
let ty = match &defs[0] {
Definition::ImportFrom(ImportFromDefinition {
@@ -39,11 +38,11 @@ where
// TODO relative imports
assert!(matches!(level, 0));
let module_name = ModuleName::new(module.as_ref().expect("TODO relative imports"));
if let Some(module) = db.resolve_module(module_name)? {
let remote_file_id = module.path(db)?.file();
let remote_symbols = db.symbol_table(remote_file_id)?;
if let Some(module) = db.resolve_module(module_name) {
let remote_file_id = module.path(db).file();
let remote_symbols = db.symbol_table(remote_file_id);
if let Some(remote_symbol_id) = remote_symbols.root_symbol_id_by_name(name) {
db.infer_symbol_type(remote_file_id, remote_symbol_id)?
db.infer_symbol_type(remote_file_id, remote_symbol_id)
} else {
Type::Unknown
}
@@ -51,68 +50,71 @@ where
Type::Unknown
}
}
Definition::ClassDef(node_key) => {
if let Some(ty) = type_store.get_cached_node_type(file_id, node_key.erased()) {
ty
} else {
let parsed = db.parse(file_id)?;
Definition::ClassDef(node_key) => db
.jar()
.type_store
.get_cached_node_type(file_id, node_key.erased())
.unwrap_or_else(|| {
let parsed = db.parse(file_id);
let ast = parsed.ast();
let node = node_key.resolve_unwrap(ast.as_any_node_ref());
let mut bases = Vec::with_capacity(node.bases().len());
let bases: Vec<_> = node
.bases()
.iter()
.map(|base_expr| infer_expr_type(db, file_id, base_expr))
.collect();
for base in node.bases() {
bases.push(infer_expr_type(db, file_id, base)?);
}
let ty = Type::Class(type_store.add_class(file_id, &node.name.id, bases));
type_store.cache_node_type(file_id, *node_key.erased(), ty);
let store = &db.jar().type_store;
let ty = Type::Class(store.add_class(file_id, &node.name.id, bases));
store.cache_node_type(file_id, *node_key.erased(), ty);
ty
}
}
Definition::FunctionDef(node_key) => {
if let Some(ty) = type_store.get_cached_node_type(file_id, node_key.erased()) {
ty
} else {
let parsed = db.parse(file_id)?;
}),
Definition::FunctionDef(node_key) => db
.jar()
.type_store
.get_cached_node_type(file_id, node_key.erased())
.unwrap_or_else(|| {
let parsed = db.parse(file_id);
let ast = parsed.ast();
let node = node_key
.resolve(ast.as_any_node_ref())
.expect("node key should resolve");
let ty = type_store.add_function(file_id, &node.name.id).into();
type_store.cache_node_type(file_id, *node_key.erased(), ty);
let store = &db.jar().type_store;
let ty = store.add_function(file_id, &node.name.id).into();
store.cache_node_type(file_id, *node_key.erased(), ty);
ty
}
}
}),
Definition::Assignment(node_key) => {
let parsed = db.parse(file_id)?;
let parsed = db.parse(file_id);
let ast = parsed.ast();
let node = node_key.resolve_unwrap(ast.as_any_node_ref());
// TODO handle unpacking assignment correctly
infer_expr_type(db, file_id, &node.value)?
infer_expr_type(db, file_id, &node.value)
}
_ => todo!("other kinds of definitions"),
};
type_store.cache_symbol_type(file_id, symbol_id, ty);
db.jar()
.type_store
.cache_symbol_type(file_id, symbol_id, ty);
// TODO record dependencies
Ok(ty)
ty
}
fn infer_expr_type<Db>(db: &Db, file_id: FileId, expr: &ast::Expr) -> QueryResult<Type>
fn infer_expr_type<Db>(db: &Db, file_id: FileId, expr: &ast::Expr) -> Type
where
Db: SemanticDb + HasJar<SemanticJar>,
{
// TODO cache the resolution of the type on the node
let symbols = db.symbol_table(file_id)?;
let symbols = db.symbol_table(file_id);
match expr {
ast::Expr::Name(name) => {
if let Some(symbol_id) = symbols.root_symbol_id_by_name(&name.id) {
db.infer_symbol_type(file_id, symbol_id)
} else {
Ok(Type::Unknown)
Type::Unknown
}
}
_ => todo!("full expression type resolution"),
@@ -152,7 +154,7 @@ mod tests {
}
#[test]
fn follow_import_to_class() -> anyhow::Result<()> {
fn follow_import_to_class() -> std::io::Result<()> {
let case = create_test()?;
let db = &case.db;
@@ -161,18 +163,18 @@ mod tests {
std::fs::write(a_path, "from b import C as D; E = D")?;
std::fs::write(b_path, "class C: pass")?;
let a_file = db
.resolve_module(ModuleName::new("a"))?
.resolve_module(ModuleName::new("a"))
.expect("module should be found")
.path(db)?
.path(db)
.file();
let a_syms = db.symbol_table(a_file)?;
let a_syms = db.symbol_table(a_file);
let e_sym = a_syms
.root_symbol_id_by_name("E")
.expect("E symbol should be found");
let ty = db.infer_symbol_type(a_file, e_sym)?;
let ty = db.infer_symbol_type(a_file, e_sym);
let jar = HasJar::<SemanticJar>::jar(db)?;
let jar = HasJar::<SemanticJar>::jar(db);
assert!(matches!(ty, Type::Class(_)));
assert_eq!(format!("{}", ty.display(&jar.type_store)), "Literal[C]");
@@ -180,28 +182,28 @@ mod tests {
}
#[test]
fn resolve_base_class_by_name() -> anyhow::Result<()> {
fn resolve_base_class_by_name() -> std::io::Result<()> {
let case = create_test()?;
let db = &case.db;
let path = case.src.path().join("mod.py");
std::fs::write(path, "class Base: pass\nclass Sub(Base): pass")?;
let file = db
.resolve_module(ModuleName::new("mod"))?
.resolve_module(ModuleName::new("mod"))
.expect("module should be found")
.path(db)?
.path(db)
.file();
let syms = db.symbol_table(file)?;
let syms = db.symbol_table(file);
let sym = syms
.root_symbol_id_by_name("Sub")
.expect("Sub symbol should be found");
let ty = db.infer_symbol_type(file, sym)?;
let ty = db.infer_symbol_type(file, sym);
let Type::Class(class_id) = ty else {
panic!("Sub is not a Class")
};
let jar = HasJar::<SemanticJar>::jar(db)?;
let jar = HasJar::<SemanticJar>::jar(db);
let base_names: Vec<_> = jar
.type_store
.get_class(class_id)

View File

@@ -115,25 +115,25 @@ class non_keyword_abcmeta_2(abc.ABCMeta): # safe
# very invalid code, but that's up to mypy et al to check
class keyword_abc_1(metaclass=ABC): # incorrect but outside scope of this check
class keyword_abc_1(metaclass=ABC): # safe
def method(self):
foo()
class keyword_abc_2(metaclass=abc.ABC): # incorrect but outside scope of this check
class keyword_abc_2(metaclass=abc.ABC): # safe
def method(self):
foo()
class abc_set_class_variable_1(ABC): # safe (abstract attribute)
class abc_set_class_variable_1(ABC): # safe
foo: int
class abc_set_class_variable_2(ABC): # error (not an abstract attribute)
class abc_set_class_variable_2(ABC): # safe
foo = 2
class abc_set_class_variable_3(ABC): # error (not an abstract attribute)
class abc_set_class_variable_3(ABC): # safe
foo: int = 2

View File

@@ -28,13 +28,3 @@ class MyClassBase(metaclass=ABCMeta):
@abstractmethod
def example(self, value):
"""Setter."""
class VariadicParameters:
@property
def attribute_var_args(self, *args): # [property-with-parameters]
return sum(args)
@property
def attribute_var_kwargs(self, **kwargs): #[property-with-parameters]
return {key: value * 2 for key, value in kwargs.items()}

View File

@@ -73,8 +73,3 @@ def foo():
async def test():
return [check async for check in async_func()]
async def test() -> str:
vals = [str(val) for val in await async_func(1)]
return ",".join(vals)

View File

@@ -971,32 +971,32 @@ pub fn code_to_rule(linter: Linter, code: &str) -> Option<(RuleGroup, Rule)> {
(Ruff, "100") => (RuleGroup::Stable, rules::ruff::rules::UnusedNOQA),
(Ruff, "101") => (RuleGroup::Preview, rules::ruff::rules::RedirectedNOQA),
(Ruff, "200") => (RuleGroup::Stable, rules::ruff::rules::InvalidPyprojectToml),
#[cfg(any(feature = "test-rules", test))]
#[cfg(feature = "test-rules")]
(Ruff, "900") => (RuleGroup::Stable, rules::ruff::rules::StableTestRule),
#[cfg(any(feature = "test-rules", test))]
#[cfg(feature = "test-rules")]
(Ruff, "901") => (RuleGroup::Stable, rules::ruff::rules::StableTestRuleSafeFix),
#[cfg(any(feature = "test-rules", test))]
#[cfg(feature = "test-rules")]
(Ruff, "902") => (RuleGroup::Stable, rules::ruff::rules::StableTestRuleUnsafeFix),
#[cfg(any(feature = "test-rules", test))]
#[cfg(feature = "test-rules")]
(Ruff, "903") => (RuleGroup::Stable, rules::ruff::rules::StableTestRuleDisplayOnlyFix),
#[cfg(any(feature = "test-rules", test))]
#[cfg(feature = "test-rules")]
(Ruff, "911") => (RuleGroup::Preview, rules::ruff::rules::PreviewTestRule),
#[cfg(any(feature = "test-rules", test))]
#[cfg(feature = "test-rules")]
#[allow(deprecated)]
(Ruff, "912") => (RuleGroup::Nursery, rules::ruff::rules::NurseryTestRule),
#[cfg(any(feature = "test-rules", test))]
#[cfg(feature = "test-rules")]
(Ruff, "920") => (RuleGroup::Deprecated, rules::ruff::rules::DeprecatedTestRule),
#[cfg(any(feature = "test-rules", test))]
#[cfg(feature = "test-rules")]
(Ruff, "921") => (RuleGroup::Deprecated, rules::ruff::rules::AnotherDeprecatedTestRule),
#[cfg(any(feature = "test-rules", test))]
#[cfg(feature = "test-rules")]
(Ruff, "930") => (RuleGroup::Removed, rules::ruff::rules::RemovedTestRule),
#[cfg(any(feature = "test-rules", test))]
#[cfg(feature = "test-rules")]
(Ruff, "931") => (RuleGroup::Removed, rules::ruff::rules::AnotherRemovedTestRule),
#[cfg(any(feature = "test-rules", test))]
#[cfg(feature = "test-rules")]
(Ruff, "940") => (RuleGroup::Removed, rules::ruff::rules::RedirectedFromTestRule),
#[cfg(any(feature = "test-rules", test))]
#[cfg(feature = "test-rules")]
(Ruff, "950") => (RuleGroup::Stable, rules::ruff::rules::RedirectedToTestRule),
#[cfg(any(feature = "test-rules", test))]
#[cfg(feature = "test-rules")]
(Ruff, "960") => (RuleGroup::Removed, rules::ruff::rules::RedirectedFromPrefixTestRule),

View File

@@ -33,7 +33,7 @@ use crate::message::Message;
use crate::noqa::add_noqa;
use crate::registry::{AsRule, Rule, RuleSet};
use crate::rules::pycodestyle;
#[cfg(any(feature = "test-rules", test))]
#[cfg(feature = "test-rules")]
use crate::rules::ruff::rules::test_rules::{self, TestRule, TEST_RULES};
use crate::settings::types::UnsafeFixes;
use crate::settings::{flags, LinterSettings};
@@ -218,7 +218,7 @@ pub fn check_path(
}
// Raise violations for internal test rules
#[cfg(any(feature = "test-rules", test))]
#[cfg(feature = "test-rules")]
{
for test_rule in TEST_RULES {
if !settings.rules.enabled(*test_rule) {

View File

@@ -104,10 +104,10 @@ static REDIRECTS: Lazy<HashMap<&'static str, &'static str>> = Lazy::new(|| {
("PGH001", "S307"),
("PGH002", "G010"),
// Test redirect by exact code
#[cfg(any(feature = "test-rules", test))]
#[cfg(feature = "test-rules")]
("RUF940", "RUF950"),
// Test redirect by prefix
#[cfg(any(feature = "test-rules", test))]
#[cfg(feature = "test-rules")]
("RUF96", "RUF95"),
// See: https://github.com/astral-sh/ruff/issues/10791
("PLW0117", "PLW0177"),

View File

@@ -323,7 +323,7 @@ mod schema {
})
.filter(|_rule| {
// Filter out all test-only rules
#[cfg(any(feature = "test-rules", test))]
#[cfg(feature = "test-rules")]
#[allow(clippy::used_underscore_binding)]
if _rule.starts_with("RUF9") {
return false;

View File

@@ -56,7 +56,6 @@ impl Violation for AbstractBaseClassWithoutAbstractMethod {
format!("`{name}` is an abstract base class, but it has no abstract methods")
}
}
/// ## What it does
/// Checks for empty methods in abstract base classes without an abstract
/// decorator.
@@ -157,13 +156,8 @@ pub(crate) fn abstract_base_class(
let mut has_abstract_method = false;
for stmt in body {
// https://github.com/PyCQA/flake8-bugbear/issues/293
// If an ABC declares an attribute by providing a type annotation
// but does not actually assign a value for that attribute,
// assume it is intended to be an "abstract attribute"
if matches!(
stmt,
Stmt::AnnAssign(ast::StmtAnnAssign { value: None, .. })
) {
// Ignore abc's that declares a class attribute that must be set
if let Stmt::AnnAssign(_) | Stmt::Assign(_) = stmt {
has_abstract_method = true;
continue;
}

View File

@@ -41,20 +41,6 @@ B024.py:92:7: B024 `notabc_Base_1` is an abstract base class, but it has no abst
94 | foo()
|
B024.py:132:7: B024 `abc_set_class_variable_2` is an abstract base class, but it has no abstract methods
|
132 | class abc_set_class_variable_2(ABC): # error (not an abstract attribute)
| ^^^^^^^^^^^^^^^^^^^^^^^^ B024
133 | foo = 2
|
B024.py:136:7: B024 `abc_set_class_variable_3` is an abstract base class, but it has no abstract methods
|
136 | class abc_set_class_variable_3(ABC): # error (not an abstract attribute)
| ^^^^^^^^^^^^^^^^^^^^^^^^ B024
137 | foo: int = 2
|
B024.py:141:7: B024 `abc_set_class_variable_4` is an abstract base class, but it has no abstract methods
|
140 | # this doesn't actually declare a class variable, it's just an expression
@@ -62,3 +48,5 @@ B024.py:141:7: B024 `abc_set_class_variable_4` is an abstract base class, but it
| ^^^^^^^^^^^^^^^^^^^^^^^^ B024
142 | foo
|

View File

@@ -51,13 +51,20 @@ pub(crate) fn property_with_parameters(
decorator_list: &[Decorator],
parameters: &Parameters,
) {
if parameters.len() <= 1 {
return;
}
let semantic = checker.semantic();
if decorator_list
if !decorator_list
.iter()
.any(|decorator| semantic.match_builtin_expr(&decorator.expression, "property"))
{
return;
}
if parameters
.posonlyargs
.iter()
.chain(&parameters.args)
.chain(&parameters.kwonlyargs)
.count()
> 1
{
checker
.diagnostics

View File

@@ -26,19 +26,4 @@ property_with_parameters.py:15:9: PLR0206 Cannot have defined parameters for pro
16 | return param + param1
|
property_with_parameters.py:35:9: PLR0206 Cannot have defined parameters for properties
|
33 | class VariadicParameters:
34 | @property
35 | def attribute_var_args(self, *args): # [property-with-parameters]
| ^^^^^^^^^^^^^^^^^^ PLR0206
36 | return sum(args)
|
property_with_parameters.py:39:9: PLR0206 Cannot have defined parameters for properties
|
38 | @property
39 | def attribute_var_kwargs(self, **kwargs): #[property-with-parameters]
| ^^^^^^^^^^^^^^^^^^^^ PLR0206
40 | return {key: value * 2 for key, value in kwargs.items()}
|

View File

@@ -21,7 +21,7 @@ pub(crate) use redirected_noqa::*;
pub(crate) use sort_dunder_all::*;
pub(crate) use sort_dunder_slots::*;
pub(crate) use static_key_dict_comprehension::*;
#[cfg(any(feature = "test-rules", test))]
#[cfg(feature = "test-rules")]
pub(crate) use test_rules::*;
pub(crate) use unnecessary_dict_comprehension_for_iterable::*;
pub(crate) use unnecessary_iterable_allocation_for_first_element::*;
@@ -56,7 +56,7 @@ mod sort_dunder_all;
mod sort_dunder_slots;
mod static_key_dict_comprehension;
mod suppression_comment_visitor;
#[cfg(any(feature = "test-rules", test))]
#[cfg(feature = "test-rules")]
pub(crate) mod test_rules;
mod unnecessary_dict_comprehension_for_iterable;
mod unnecessary_iterable_allocation_for_first_element;

View File

@@ -58,6 +58,14 @@ impl<'a> preorder::PreorderVisitor<'a> for AsyncExprVisitor {
preorder::TraversalSignal::Traverse
}
}
fn visit_expr(&mut self, expr: &'a Expr) {
match expr {
Expr::Await(_) => {
self.found_await_or_async = true;
}
_ => preorder::walk_expr(self, expr),
}
}
fn visit_stmt(&mut self, stmt: &'a Stmt) {
match stmt {
Stmt::With(ast::StmtWith { is_async: true, .. }) => {
@@ -76,19 +84,9 @@ impl<'a> preorder::PreorderVisitor<'a> for AsyncExprVisitor {
_ => preorder::walk_stmt(self, stmt),
}
}
fn visit_expr(&mut self, expr: &'a Expr) {
match expr {
Expr::Await(_) => {
self.found_await_or_async = true;
}
_ => preorder::walk_expr(self, expr),
}
}
fn visit_comprehension(&mut self, comprehension: &'a ast::Comprehension) {
if comprehension.is_async {
self.found_await_or_async = true;
} else {
preorder::walk_comprehension(self, comprehension);
}
}
}

View File

@@ -1,21 +1,31 @@
use ruff_text_size::{TextLen, TextSize};
use std::str::Chars;
pub(crate) const EOF_CHAR: char = '\0';
#[derive(Clone, Debug)]
pub(super) struct Cursor<'a> {
chars: Chars<'a>,
source_length: TextSize,
pub(super) struct Cursor<'src> {
/// Source text.
source: &'src str,
/// The start position in the source text of the next character.
position: usize,
/// Offset of the current token from the start of the source. The length of the current
/// token can be computed by `self.position - self.current_start`.
current_start: TextSize,
#[cfg(debug_assertions)]
prev_char: char,
}
impl<'a> Cursor<'a> {
pub(crate) fn new(source: &'a str) -> Self {
impl<'src> Cursor<'src> {
pub(crate) fn new(source: &'src str) -> Self {
assert!(TextSize::try_from(source.len()).is_ok());
Self {
source_length: source.text_len(),
chars: source.chars(),
source,
position: 0,
current_start: TextSize::new(0),
#[cfg(debug_assertions)]
prev_char: EOF_CHAR,
}
@@ -30,43 +40,48 @@ impl<'a> Cursor<'a> {
/// Peeks the next character from the input stream without consuming it.
/// Returns [`EOF_CHAR`] if the file is at the end of the file.
pub(super) fn first(&self) -> char {
self.chars.clone().next().unwrap_or(EOF_CHAR)
self.rest().chars().next().unwrap_or(EOF_CHAR)
}
/// Peeks the second character from the input stream without consuming it.
/// Returns [`EOF_CHAR`] if the position is past the end of the file.
pub(super) fn second(&self) -> char {
let mut chars = self.chars.clone();
let mut chars = self.rest().chars();
chars.next();
chars.next().unwrap_or(EOF_CHAR)
}
/// Returns the remaining text to lex.
pub(super) fn rest(&self) -> &'a str {
self.chars.as_str()
pub(super) fn rest(&self) -> &'src str {
self.source.get(self.position..).unwrap_or("")
}
// SAFETY: The `source.text_len` call in `new` would panic if the string length is larger than a `u32`.
// SAFETY: The `Cursor::new` call would panic if the string length is larger than a `u32`.
#[allow(clippy::cast_possible_truncation)]
pub(super) fn text_len(&self) -> TextSize {
TextSize::new(self.chars.as_str().len() as u32)
TextSize::new(self.rest().len() as u32)
}
// SAFETY: The `Cursor::new` call would panic if the string length is larger than a `u32`.
#[allow(clippy::cast_possible_truncation)]
pub(super) fn token_len(&self) -> TextSize {
self.source_length - self.text_len()
TextSize::new(self.position as u32) - self.current_start
}
// SAFETY: The `Cursor::new` call would panic if the string length is larger than a `u32`.
#[allow(clippy::cast_possible_truncation)]
pub(super) fn start_token(&mut self) {
self.source_length = self.text_len();
self.current_start = TextSize::new(self.position as u32);
}
pub(super) fn is_eof(&self) -> bool {
self.chars.as_str().is_empty()
self.position >= self.source.len()
}
/// Consumes the next character
pub(super) fn bump(&mut self) -> Option<char> {
let prev = self.chars.next()?;
let prev = self.rest().chars().next()?;
self.position += prev.text_len().to_usize();
#[cfg(debug_assertions)]
{
@@ -86,7 +101,7 @@ impl<'a> Cursor<'a> {
}
pub(super) fn eat_char2(&mut self, c1: char, c2: char) -> bool {
let mut chars = self.chars.clone();
let mut chars = self.rest().chars();
if chars.next() == Some(c1) && chars.next() == Some(c2) {
self.bump();
self.bump();
@@ -97,7 +112,7 @@ impl<'a> Cursor<'a> {
}
pub(super) fn eat_char3(&mut self, c1: char, c2: char, c3: char) -> bool {
let mut chars = self.chars.clone();
let mut chars = self.rest().chars();
if chars.next() == Some(c1) && chars.next() == Some(c2) && chars.next() == Some(c3) {
self.bump();
self.bump();
@@ -137,17 +152,14 @@ impl<'a> Cursor<'a> {
pub(super) fn skip_bytes(&mut self, count: usize) {
#[cfg(debug_assertions)]
{
self.prev_char = self.chars.as_str()[..count]
.chars()
.next_back()
.unwrap_or('\0');
self.prev_char = self.rest()[..count].chars().next_back().unwrap_or('\0');
}
self.chars = self.chars.as_str()[count..].chars();
self.position += count;
}
/// Skips to the end of the input stream.
pub(super) fn skip_to_end(&mut self) {
self.chars = "".chars();
self.position = self.source.len();
}
}

View File

@@ -26,7 +26,7 @@ ruff_text_size = { path = "../ruff_text_size" }
ruff_workspace = { path = "../ruff_workspace" }
anyhow = { workspace = true }
crossbeam = { workspace = true }
crossbeam-channel = { workspace = true }
jod-thread = { workspace = true }
libc = { workspace = true }
lsp-server = { workspace = true }

View File

@@ -1,61 +0,0 @@
## Helix Setup Guide for `ruff server`
First, open the language configuration file for Helix. On Linux and macOS, this will be at `~/.config/helix/languages.toml`,
and on Windows this will be at `%AppData%\helix\languages.toml`.
Add the language server by adding:
```toml
[language-server.ruff]
command = "ruff"
args = ["server", "--preview"]
```
Then, you'll register the language server as the one to use with Python.
If you don't already have a language server registered to use with Python, add this to `languages.toml`:
```toml
[[language]]
name = "python"
language-servers = ["ruff"]
```
Otherwise, if you already have `language-servers` defined, you can simply add `"ruff"` to the list. For example,
if you already have `pylsp` as a language server, you can modify the language entry as follows:
```toml
[[language]]
name = "python"
language-servers = ["ruff", "pylsp"]
```
> \[!NOTE\]
> Multiple language servers for a single language are only supported in Helix version [`23.10`](https://github.com/helix-editor/helix/blob/master/CHANGELOG.md#2310-2023-10-24) and later.
Once you've set up the server, you should see diagnostics in your Python files. Code actions and other LSP features should also be available.
![image](assets/SuccessfulHelixSetup.png "A screenshot showing an open Python file in Helix with highlighted diagnostics and a code action dropdown menu open")
*This screenshot is using `select=["ALL]"` for demonstration purposes.*
If you want to, as an example, turn on auto-formatting, add `auto-format = true`:
```toml
[[language]]
name = "python"
language-servers = ["ruff", "pylsp"]
auto-format = true
```
See the [Helix documentation](https://docs.helix-editor.com/languages.html) for more settings you can use here.
You can pass settings into `ruff server` using `[language-server.ruff.config.settings]`. For example:
```toml
[language-server.ruff.config.settings]
line-length = 80
[language-server.ruff.config.settings.lint]
select = ["E4", "E7"]
preview = false
[language-server.ruff.config.settings.format]
preview = true
```

Binary file not shown.

Before

Width:  |  Height:  |  Size: 195 KiB

View File

@@ -6,7 +6,7 @@ use serde_json::Value;
use super::schedule::Task;
pub(crate) type ClientSender = crossbeam::channel::Sender<lsp_server::Message>;
pub(crate) type ClientSender = crossbeam_channel::Sender<lsp_server::Message>;
type ResponseBuilder<'s> = Box<dyn FnOnce(lsp_server::Response) -> Task<'s>>;

View File

@@ -1,6 +1,6 @@
use std::num::NonZeroUsize;
use crossbeam::channel::Sender;
use crossbeam_channel::Sender;
use crate::session::Session;

View File

@@ -21,7 +21,7 @@ use std::{
},
};
use crossbeam::channel::{Receiver, Sender};
use crossbeam_channel::{Receiver, Sender};
use super::{Builder, JoinHandle, ThreadPriority};
@@ -52,7 +52,7 @@ impl Pool {
let threads = usize::from(threads);
// Channel buffer capacity is between 2 and 4, depending on the pool size.
let (job_sender, job_receiver) = crossbeam::channel::bounded(std::cmp::min(threads * 2, 4));
let (job_sender, job_receiver) = crossbeam_channel::bounded(std::cmp::min(threads * 2, 4));
let extant_tasks = Arc::new(AtomicUsize::new(0));
let mut handles = Vec::with_capacity(threads);