Compare commits

...

2 Commits

Author SHA1 Message Date
Charlie Marsh
9c5988f23e Add an Unrecognized variant to PySourceType 2023-08-28 10:49:17 -04:00
Charlie Marsh
a33883d51a Add TOML files to SourceType 2023-08-28 10:40:02 -04:00
8 changed files with 152 additions and 137 deletions

View File

@@ -268,9 +268,12 @@ pub fn check_path(
const MAX_ITERATIONS: usize = 100; const MAX_ITERATIONS: usize = 100;
/// Add any missing `# noqa` pragmas to the source code at the given `Path`. /// Add any missing `# noqa` pragmas to the source code at the given `Path`.
pub fn add_noqa_to_path(path: &Path, package: Option<&Path>, settings: &Settings) -> Result<usize> { pub fn add_noqa_to_path(
let source_type = PySourceType::from(path); path: &Path,
package: Option<&Path>,
source_type: PySourceType,
settings: &Settings,
) -> Result<usize> {
// Read the file from disk. // Read the file from disk.
let contents = std::fs::read_to_string(path)?; let contents = std::fs::read_to_string(path)?;

View File

@@ -3,7 +3,9 @@ use crate::jupyter::Notebook;
#[derive(Clone, Debug, PartialEq, is_macro::Is)] #[derive(Clone, Debug, PartialEq, is_macro::Is)]
pub enum SourceKind { pub enum SourceKind {
/// The source contains Python source code.
Python(String), Python(String),
/// The source contains a Jupyter notebook.
IpyNotebook(Notebook), IpyNotebook(Notebook),
} }

View File

@@ -8,7 +8,7 @@ use rayon::prelude::*;
use ruff::linter::add_noqa_to_path; use ruff::linter::add_noqa_to_path;
use ruff::warn_user_once; use ruff::warn_user_once;
use ruff_python_stdlib::path::{is_jupyter_notebook, is_project_toml}; use ruff_python_ast::{PySourceType, SourceType};
use ruff_workspace::resolver::{python_files_in_path, PyprojectConfig}; use ruff_workspace::resolver::{python_files_in_path, PyprojectConfig};
use crate::args::Overrides; use crate::args::Overrides;
@@ -46,15 +46,18 @@ pub(crate) fn add_noqa(
.flatten() .flatten()
.filter_map(|entry| { .filter_map(|entry| {
let path = entry.path(); let path = entry.path();
if is_project_toml(path) || is_jupyter_notebook(path) { let SourceType::Python(
source_type @ (PySourceType::Py | PySourceType::Pyi | PySourceType::Pyw),
) = SourceType::from(path)
else {
return None; return None;
} };
let package = path let package = path
.parent() .parent()
.and_then(|parent| package_roots.get(parent)) .and_then(|parent| package_roots.get(parent))
.and_then(|package| *package); .and_then(|package| *package);
let settings = resolver.resolve(path, pyproject_config); let settings = resolver.resolve(path, pyproject_config);
match add_noqa_to_path(path, package, settings) { match add_noqa_to_path(path, package, source_type, settings) {
Ok(count) => Some(count), Ok(count) => Some(count),
Err(e) => { Err(e) => {
error!("Failed to add noqa to {}: {e}", path.display()); error!("Failed to add noqa to {}: {e}", path.display());

View File

@@ -12,7 +12,7 @@ use tracing::{span, Level};
use ruff::fs; use ruff::fs;
use ruff::warn_user_once; use ruff::warn_user_once;
use ruff_formatter::LineWidth; use ruff_formatter::LineWidth;
use ruff_python_ast::PySourceType; use ruff_python_ast::{PySourceType, SourceType};
use ruff_python_formatter::{format_module, FormatModuleError, PyFormatOptions}; use ruff_python_formatter::{format_module, FormatModuleError, PyFormatOptions};
use ruff_workspace::resolver::python_files_in_path; use ruff_workspace::resolver::python_files_in_path;
@@ -37,23 +37,21 @@ pub(crate) fn format(cli: &Arguments, overrides: &Overrides) -> Result<ExitStatu
let result = paths let result = paths
.into_par_iter() .into_par_iter()
.map(|dir_entry| { .map(|entry| {
let dir_entry = dir_entry?; let entry = entry?;
let path = dir_entry.path(); let path = entry.path();
let source_type = PySourceType::from(path); if matches!(
if !(source_type.is_python() || source_type.is_stub()) SourceType::from(path),
|| path SourceType::Python(PySourceType::Py | PySourceType::Pyi | PySourceType::Pyw)
.extension() ) {
.is_some_and(|extension| extension == "toml") let line_length = resolver.resolve(path, &pyproject_config).line_length;
{ let options = PyFormatOptions::from_extension(path)
return Ok(()); .with_line_width(LineWidth::from(NonZeroU16::from(line_length)));
format_path(path, options)
} else {
Ok(())
} }
let line_length = resolver.resolve(path, &pyproject_config).line_length;
let options = PyFormatOptions::from_extension(path)
.with_line_width(LineWidth::from(NonZeroU16::from(line_length)));
format_path(path, options)
}) })
.map(|result| { .map(|result| {
result.map_err(|err| { result.map_err(|err| {

View File

@@ -27,8 +27,7 @@ use ruff::{fs, IOError};
use ruff_diagnostics::Diagnostic; use ruff_diagnostics::Diagnostic;
use ruff_macros::CacheKey; use ruff_macros::CacheKey;
use ruff_python_ast::imports::ImportMap; use ruff_python_ast::imports::ImportMap;
use ruff_python_ast::PySourceType; use ruff_python_ast::{PySourceType, SourceType, TomlSourceType};
use ruff_python_stdlib::path::is_project_toml;
use ruff_source_file::{LineIndex, SourceCode, SourceFileBuilder}; use ruff_source_file::{LineIndex, SourceCode, SourceFileBuilder};
use ruff_text_size::{TextRange, TextSize}; use ruff_text_size::{TextRange, TextSize};
@@ -228,36 +227,36 @@ pub(crate) fn lint_path(
debug!("Checking: {}", path.display()); debug!("Checking: {}", path.display());
// We have to special case this here since the Python tokenizer doesn't work with TOML. let source_type = match SourceType::from(path) {
if is_project_toml(path) { SourceType::Toml(TomlSourceType::Pyproject) => {
let messages = if settings let messages = if settings
.lib .lib
.rules .rules
.iter_enabled() .iter_enabled()
.any(|rule_code| rule_code.lint_source().is_pyproject_toml()) .any(|rule_code| rule_code.lint_source().is_pyproject_toml())
{ {
let contents = match std::fs::read_to_string(path) { let contents = match std::fs::read_to_string(path) {
Ok(contents) => contents, Ok(contents) => contents,
Err(err) => { Err(err) => {
return Ok(Diagnostics::from_io_error(&err, path, &settings.lib)); return Ok(Diagnostics::from_io_error(&err, path, &settings.lib));
} }
};
let source_file = SourceFileBuilder::new(path.to_string_lossy(), contents).finish();
lint_pyproject_toml(source_file, &settings.lib)
} else {
vec![]
}; };
let source_file = SourceFileBuilder::new(path.to_string_lossy(), contents).finish(); return Ok(Diagnostics {
lint_pyproject_toml(source_file, &settings.lib) messages,
} else { ..Diagnostics::default()
vec![] });
}; }
return Ok(Diagnostics { SourceType::Toml(_) => return Ok(Diagnostics::default()),
messages, SourceType::Python(source_type) => source_type,
..Diagnostics::default() };
});
}
// Extract the sources from the file. // Extract the sources from the file.
let LintSources { let LintSource(source_kind) = match LintSource::try_from_path(path, source_type) {
source_type,
source_kind,
} = match LintSources::try_from_path(path) {
Ok(sources) => sources, Ok(sources) => sources,
Err(SourceExtractionError::Io(err)) => { Err(SourceExtractionError::Io(err)) => {
return Ok(Diagnostics::from_io_error(&err, path, &settings.lib)); return Ok(Diagnostics::from_io_error(&err, path, &settings.lib));
@@ -438,21 +437,24 @@ pub(crate) fn lint_stdin(
noqa: flags::Noqa, noqa: flags::Noqa,
autofix: flags::FixMode, autofix: flags::FixMode,
) -> Result<Diagnostics> { ) -> Result<Diagnostics> {
// Extract the sources from the file. // TODO(charlie): Support `pyproject.toml`.
let LintSources { let SourceType::Python(source_type) = path.map(SourceType::from).unwrap_or_default() else {
source_type, return Ok(Diagnostics::default());
source_kind,
} = match LintSources::try_from_source_code(contents, path) {
Ok(sources) => sources,
Err(SourceExtractionError::Io(err)) => {
// SAFETY: An `io::Error` can only occur if we're reading from a path.
return Ok(Diagnostics::from_io_error(&err, path.unwrap(), settings));
}
Err(SourceExtractionError::Diagnostics(diagnostics)) => {
return Ok(*diagnostics);
}
}; };
// Extract the sources from the file.
let LintSource(source_kind) =
match LintSource::try_from_source_code(contents, path, source_type) {
Ok(sources) => sources,
Err(SourceExtractionError::Io(err)) => {
// SAFETY: An `io::Error` can only occur if we're reading from a path.
return Ok(Diagnostics::from_io_error(&err, path.unwrap(), settings));
}
Err(SourceExtractionError::Diagnostics(diagnostics)) => {
return Ok(*diagnostics);
}
};
// Lint the inputs. // Lint the inputs.
let ( let (
LinterResult { LinterResult {
@@ -554,58 +556,40 @@ pub(crate) fn lint_stdin(
} }
#[derive(Debug)] #[derive(Debug)]
struct LintSources { struct LintSource(SourceKind);
/// The "type" of source code, e.g. `.py`, `.pyi`, `.ipynb`, etc.
source_type: PySourceType,
/// The "kind" of source, e.g. Python file, Jupyter Notebook, etc.
source_kind: SourceKind,
}
impl LintSources { impl LintSource {
/// Extract the lint [`LintSources`] from the given file path. /// Extract the lint [`LintSource`] from the given file path.
fn try_from_path(path: &Path) -> Result<LintSources, SourceExtractionError> { fn try_from_path(
let source_type = PySourceType::from(path); path: &Path,
source_type: PySourceType,
// Read the file from disk. ) -> Result<LintSource, SourceExtractionError> {
if source_type.is_ipynb() { if source_type.is_ipynb() {
let notebook = notebook_from_path(path).map_err(SourceExtractionError::Diagnostics)?; let notebook = notebook_from_path(path).map_err(SourceExtractionError::Diagnostics)?;
let source_kind = SourceKind::IpyNotebook(notebook); let source_kind = SourceKind::IpyNotebook(notebook);
Ok(LintSources { Ok(LintSource(source_kind))
source_type,
source_kind,
})
} else { } else {
// This is tested by ruff_cli integration test `unreadable_file` // This is tested by ruff_cli integration test `unreadable_file`
let contents = std::fs::read_to_string(path).map_err(SourceExtractionError::Io)?; let contents = std::fs::read_to_string(path).map_err(SourceExtractionError::Io)?;
Ok(LintSources { Ok(LintSource(SourceKind::Python(contents)))
source_type,
source_kind: SourceKind::Python(contents),
})
} }
} }
/// Extract the lint [`LintSources`] from the raw string contents, optionally accompanied by a /// Extract the lint [`LintSource`] from the raw string contents, optionally accompanied by a
/// file path indicating the path to the file from which the contents were read. If provided, /// file path indicating the path to the file from which the contents were read. If provided,
/// the file path should be used for diagnostics, but not for reading the file from disk. /// the file path should be used for diagnostics, but not for reading the file from disk.
fn try_from_source_code( fn try_from_source_code(
source_code: String, source_code: String,
path: Option<&Path>, path: Option<&Path>,
) -> Result<LintSources, SourceExtractionError> { source_type: PySourceType,
let source_type = path.map(PySourceType::from).unwrap_or_default(); ) -> Result<LintSource, SourceExtractionError> {
if source_type.is_ipynb() { if source_type.is_ipynb() {
let notebook = notebook_from_source_code(&source_code, path) let notebook = notebook_from_source_code(&source_code, path)
.map_err(SourceExtractionError::Diagnostics)?; .map_err(SourceExtractionError::Diagnostics)?;
let source_kind = SourceKind::IpyNotebook(notebook); let source_kind = SourceKind::IpyNotebook(notebook);
Ok(LintSources { Ok(LintSource(source_kind))
source_type,
source_kind,
})
} else { } else {
Ok(LintSources { Ok(LintSource(SourceKind::Python(source_code)))
source_type,
source_kind: SourceKind::Python(source_code),
})
} }
} }
} }

View File

@@ -24,35 +24,78 @@ pub mod types;
pub mod visitor; pub mod visitor;
pub mod whitespace; pub mod whitespace;
#[derive(Clone, Copy, Debug, Default, PartialEq)] /// The type of a source file.
#[derive(Clone, Copy, Debug, PartialEq, is_macro::Is)]
pub enum SourceType {
/// The file contains Python source code.
Python(PySourceType),
/// The file contains TOML.
Toml(TomlSourceType),
}
impl Default for SourceType {
fn default() -> Self {
Self::Python(PySourceType::Py)
}
}
impl From<&Path> for SourceType {
fn from(path: &Path) -> Self {
match path.file_name() {
Some(filename) if filename == "pyproject.toml" => Self::Toml(TomlSourceType::Pyproject),
Some(filename) if filename == "Pipfile" => Self::Toml(TomlSourceType::Pipfile),
Some(filename) if filename == "poetry.lock" => Self::Toml(TomlSourceType::Poetry),
_ => match path.extension() {
Some(ext) if ext == "toml" => Self::Toml(TomlSourceType::Unrecognized),
_ => Self::Python(PySourceType::from(path)),
},
}
}
}
#[derive(Clone, Copy, Debug, PartialEq, is_macro::Is)]
pub enum TomlSourceType {
/// The source is a `pyproject.toml`.
Pyproject,
/// The source is a `Pipfile`.
Pipfile,
/// The source is a `poetry.lock`.
Poetry,
/// The source is an unrecognized TOML file.
Unrecognized,
}
#[derive(Clone, Copy, Debug, Default, PartialEq, is_macro::Is)]
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
pub enum PySourceType { pub enum PySourceType {
/// The source is a Python file (`.py`).
#[default] #[default]
Python, Py,
Stub, /// The source is a Python stub file (`.pyi`).
Pyi,
/// The source is a Python script with a graphical user interface (`.pyw`).
Pyw,
/// The source is a Jupyter notebook (`.ipynb`).
Ipynb, Ipynb,
/// The source is an unrecognized Python file.
Unrecognized,
} }
impl PySourceType { impl PySourceType {
pub const fn is_python(&self) -> bool { /// Return `true` if the source type is a stub file.
matches!(self, PySourceType::Python) pub const fn is_stub(self) -> bool {
} matches!(self, Self::Pyi)
pub const fn is_stub(&self) -> bool {
matches!(self, PySourceType::Stub)
}
pub const fn is_ipynb(&self) -> bool {
matches!(self, PySourceType::Ipynb)
} }
} }
impl From<&Path> for PySourceType { impl From<&Path> for PySourceType {
fn from(path: &Path) -> Self { fn from(path: &Path) -> Self {
match path.extension() { match path.extension() {
Some(ext) if ext == "pyi" => PySourceType::Stub, Some(ext) if ext == "py" => PySourceType::Py,
Some(ext) if ext == "pyi" => PySourceType::Pyi,
Some(ext) if ext == "pyw" => PySourceType::Pyw,
Some(ext) if ext == "ipynb" => PySourceType::Ipynb, Some(ext) if ext == "ipynb" => PySourceType::Ipynb,
_ => PySourceType::Python, _ => PySourceType::Py,
} }
} }
} }

View File

@@ -312,7 +312,10 @@ pub trait AsMode {
impl AsMode for PySourceType { impl AsMode for PySourceType {
fn as_mode(&self) -> Mode { fn as_mode(&self) -> Mode {
match self { match self {
PySourceType::Python | PySourceType::Stub => Mode::Module, PySourceType::Py
| PySourceType::Pyi
| PySourceType::Pyw
| PySourceType::Unrecognized => Mode::Module,
PySourceType::Ipynb => Mode::Jupyter, PySourceType::Ipynb => Mode::Jupyter,
} }
} }

View File

@@ -1,13 +1,7 @@
use std::path::Path; use std::path::Path;
/// Return `true` if the [`Path`] appears to be that of a Python file.
pub fn is_python_file(path: &Path) -> bool {
path.extension()
.is_some_and(|ext| ext == "py" || ext == "pyi")
}
/// Return `true` if the [`Path`] is named `pyproject.toml`. /// Return `true` if the [`Path`] is named `pyproject.toml`.
pub fn is_project_toml(path: &Path) -> bool { pub fn is_pyproject_toml(path: &Path) -> bool {
path.file_name() path.file_name()
.is_some_and(|name| name == "pyproject.toml") .is_some_and(|name| name == "pyproject.toml")
} }
@@ -26,22 +20,7 @@ pub fn is_jupyter_notebook(path: &Path) -> bool {
mod tests { mod tests {
use std::path::Path; use std::path::Path;
use crate::path::{is_jupyter_notebook, is_python_file}; use crate::path::is_jupyter_notebook;
#[test]
fn inclusions() {
let path = Path::new("foo/bar/baz.py");
assert!(is_python_file(path));
let path = Path::new("foo/bar/baz.pyi");
assert!(is_python_file(path));
let path = Path::new("foo/bar/baz.js");
assert!(!is_python_file(path));
let path = Path::new("foo/bar/baz");
assert!(!is_python_file(path));
}
#[test] #[test]
fn test_is_jupyter_notebook() { fn test_is_jupyter_notebook() {