Compare commits

..

89 Commits

Author SHA1 Message Date
Charlie Marsh
c23bf395e2 Warn on invalid # noqa rule codes 2024-08-11 19:23:33 -04:00
Charlie Marsh
383676e332 Avoid parsing joint rule codes as distinct codes in # noqa 2024-08-11 19:23:00 -04:00
Yury Fedotov
feba5031dc [Minor typo] Fix article in "an fix" (#12797) 2024-08-10 21:22:00 -04:00
Dylan
0c2b88f224 [flake8-simplify] Further simplify to binary in preview for if-else-block-instead-of-if-exp (SIM108) (#12796)
In most cases we should suggest a ternary operator, but there are three
edge cases where a binary operator is more appropriate.

Given an if-else block of the form

```python
if test:
    target_var = body_value
else:
    target_var = else_value
```
This PR updates the check for SIM108 to the following:

- If `test == body_value` and preview enabled, suggest to replace with
`target_var = test or else_value`
- If `test == not body_value` and preview enabled, suggest to replace
with `target_var = body_value and else_value`
- If `not test == body_value` and preview enabled, suggest to replace
with `target_var = body_value and else_value`
- Otherwise, suggest to replace with `target_var = body_value if test
else else_value`

Closes #12189.
2024-08-10 16:49:25 +00:00
Alex Waygood
cf1a57df5a Remove red_knot_python_semantic::python_version::TargetVersion (#12790) 2024-08-10 14:28:31 +01:00
renovate[bot]
597c5f9124 Update dependency black to v24 (#12728) 2024-08-10 18:04:37 +05:30
Charlie Marsh
69e1c567d4 Treat type(Protocol) et al as metaclass base (#12770)
## Summary

Closes https://github.com/astral-sh/ruff/issues/12736.
2024-08-09 20:10:12 +00:00
Alex Waygood
37b9bac403 [red-knot] Add support for --system-site-packages virtual environments (#12759) 2024-08-09 21:02:16 +01:00
Alex Waygood
83db48d316 RUF031: Ignore unparenthesized tuples in subscripts when the subscript is obviously a type annotation or type alias (#12762) 2024-08-09 20:31:27 +01:00
Alex Waygood
c4e651921b [red-knot] Move, rename and make public the PyVersion type (#12782) 2024-08-09 16:49:17 +01:00
Dylan
b595346213 [ruff] Do not remove parens for tuples with starred expressions in Python <=3.10 RUF031 (#12784) 2024-08-09 17:30:29 +02:00
Ryan Hoban
253474b312 Document that BLE001 supports both BaseException and Exception (#12788) 2024-08-09 17:28:50 +02:00
Micha Reiser
a176679b24 Log warnings when skipping editable installations (#12779) 2024-08-09 16:29:43 +02:00
Charlie Marsh
1f51048fa4 Don't enforce returns and yields in abstract methods (#12771)
## Summary

Closes https://github.com/astral-sh/ruff/issues/12685.
2024-08-09 13:34:14 +00:00
Micha Reiser
2abfab0f9b Move Program and related structs to red_knot_python_semantic (#12777) 2024-08-09 11:50:45 +02:00
Dylan
64f1f3468d [ruff] Skip tuples with slice expressions in incorrectly-parenthesized-tuple-in-subscript (RUF031) (#12768)
## Summary

Adding parentheses to a tuple in a subscript with elements that include
slice expressions causes a syntax error. For example, `d[(1,2,:)]` is a
syntax error.

So, when `lint.ruff.parenthesize-tuple-in-subscript = true` and the
tuple includes a slice expression, we skip this check and fix.

Closes #12766.
2024-08-09 09:22:58 +00:00
Micha Reiser
ffaa35eafe Add test helper to setup tracing (#12741) 2024-08-09 07:04:04 +00:00
Charlie Marsh
c906b0183b Add known problems warning to type-comparison rule (#12769)
## Summary

See: https://github.com/astral-sh/ruff/issues/4560
2024-08-09 01:41:15 +00:00
Carl Meyer
bc5b9b81dd [red-knot] add dev dependency on ruff_db os feature from red_knot_pyt… (#12760) 2024-08-08 18:10:30 +01:00
Dhruv Manilawala
221ea662e0 Bump version to 0.5.7 (#12756) 2024-08-08 20:56:15 +05:30
Alex Waygood
d28c5afd14 [red-knot] Remove mentions of Ruff from the CLI help (#12752)
Co-authored-by: Micha Reiser <micha@reiser.io>
2024-08-08 15:35:10 +01:00
Alex Waygood
f1de08c2a0 [red-knot] Merge the semantic and module-resolver crates (#12751) 2024-08-08 15:34:11 +01:00
Christian Clauss
33e9a6a54e SIM110: any() is ~3x slower than the code it replaces (#12746)
> ~Builtins are also more efficient than `for` loops.~

Let's not promise performance because this code transformation does not
deliver.

Benchmark written by @dcbaker

> `any()` seems to be about 1/3 as fast (Python 3.11.9, NixOS):
```python
loop = 'abcdef'.split()
found = 'f'
nfound = 'g'


def test1():
    for x in loop:
        if x == found:
            return True
    return False


def test2():
    return any(x == found for x in loop)


def test3():
    for x in loop:
        if x == nfound:
            return True
    return False


def test4():
    return any(x == nfound for x in loop)


if __name__ == "__main__":
    import timeit

    print('for loop (found)    :', timeit.timeit(test1))
    print('for loop (not found):', timeit.timeit(test3))
    print('any() (found)       :', timeit.timeit(test2))
    print('any() (not found)   :', timeit.timeit(test4))
```
```
for loop (found)    : 0.051076093994197436
for loop (not found): 0.04388196699437685
any() (found)       : 0.15422860698890872
any() (not found)   : 0.15568504799739458
```
I have retested with longer lists and on multiple Python versions with
similar results.
2024-08-08 08:25:43 -04:00
Dylan
f577e03021 [ruff] Ignore empty tuples for incorrectly-parenthesized-tuple-in-subscript (RUF031) (#12749) 2024-08-08 13:18:03 +02:00
Micha Reiser
f53733525c Remove all useEffect usages (#12659) 2024-08-08 13:16:38 +02:00
Micha Reiser
2daa914334 Gracefully handle errors in CLI (#12747) 2024-08-08 11:02:47 +00:00
Steve C
6d9205e346 [ruff_linter] - Use LibCST in adjust_indentation for mixed whitespace (#12740) 2024-08-08 10:49:58 +02:00
Micha Reiser
df7345e118 Exit with an error if there are check failures (#12735) 2024-08-08 07:10:18 +00:00
Micha Reiser
dc6aafecc2 Setup tracing and document tracing usage (#12730) 2024-08-08 06:28:40 +00:00
Charlie Marsh
5107a50ae7 Parenthesize conditions based on precedence when merging if arms (#12737)
## Summary

Closes https://github.com/astral-sh/ruff/issues/12732.
2024-08-07 23:03:24 -04:00
Micha Reiser
a631d600ac Fix cache invalidation for nested pyproject.toml files (#12727) 2024-08-07 21:53:45 +02:00
Alex Waygood
f34b9a77f0 [red-knot] Cleanups to logic resolving site-packages from a venv path (#12731) 2024-08-07 15:48:15 +01:00
Dylan
7997da47f5 [ruff] Implement incorrectly-parenthesized-tuple-in-subscript (RUF031) (#12480)
Implements the new fixable lint rule `RUF031` which checks for the use or omission of parentheses around tuples in subscripts, depending on the setting `lint.ruff.parenthesize-tuple-in-getitem`. By default, the use of parentheses is considered a violation.
2024-08-07 13:11:29 +00:00
Alex Waygood
d380b37a09 Add a new Binding::is_unused method (#12729) 2024-08-07 11:17:56 +01:00
Alex Waygood
b14fee9320 [ruff] Mark RUF023 fix as unsafe if __slots__ is not a set and the binding is used elsewhere (#12692) 2024-08-07 10:41:03 +01:00
Dhruv Manilawala
037e817450 Use struct instead of type alias for workspace settings index (#12726)
## Summary

Follow-up from https://github.com/astral-sh/ruff/pull/12725, this is
just a small refactor to use a wrapper struct instead of type alias for
workspace settings index. This avoids the need to have the
`register_workspace_settings` as a static method on `Index` and instead
is a method on the new struct itself.
2024-08-07 09:26:59 +00:00
Dhruv Manilawala
7fcfedd430 Ignore non-file workspace URL (#12725)
## Summary

This PR updates the server to ignore non-file workspace URL.

This is to avoid crashing the server if the URL scheme is not "file".
We'd still raise an error if the URL to file path conversion fails.

Also, as per the docs of
[`to_file_path`](https://docs.rs/url/2.5.2/url/struct.Url.html#method.to_file_path):

> Note: This does not actually check the URL’s scheme, and may give
nonsensical results for other schemes. It is the user’s responsibility
to check the URL’s scheme before calling this.

resolves: #12660

## Test Plan

I'm not sure how to test this locally but the change is small enough to
validate on its own.
2024-08-07 09:15:55 +00:00
Dhruv Manilawala
50ff5c7544 Include docs requirements for Renovate upgrades (#12724)
## Summary

This PR updates the Renovate config to account for the
`requirements*.txt` files in `docs/` directory.

The `mkdocs-material` upgrade is ignored because we use commit SHA for
the insider version and it should match the corresponding public version
as per the docs:
https://squidfunk.github.io/mkdocs-material/insiders/upgrade/
(`9.x.x-insiders-4.x.x`).

## Test Plan

```console
❯ renovate-config-validator
(node:83193) [DEP0040] DeprecationWarning: The `punycode` module is deprecated. Please use a userland alternative instead.
(Use `node --trace-deprecation ...` to show where the warning was created)
 INFO: Validating .github/renovate.json5
 INFO: Config validated successfully
```
2024-08-07 13:11:18 +05:30
Charlie Marsh
90e5bc2bd9 Avoid false-positives for list concatenations in SQL construction (#12720)
## Summary

Closes https://github.com/astral-sh/ruff/issues/12710.
2024-08-06 16:26:03 -04:00
Alex Waygood
aae9619d3d [red-knot] Fix build on Windows (#12719)
## Summary

Tests are failing on `main` because automerge landed
https://github.com/astral-sh/ruff/pull/12716 despite the Windows tests
failing.
2024-08-06 20:21:25 +01:00
Alex Waygood
7fa76a2b2b [red-knot] Derive site-packages from a venv path (#12716) 2024-08-06 18:34:37 +00:00
Dhruv Manilawala
14dd6d980e [red-knot] Keep subcommands optional for the binary (#12715)
## Summary

This PR updates the `red_knot` CLI to make the subcommand optional.

## Test Plan

Run the following commands:
* `cargo run --bin red_knot --
--current-directory=~/playground/ruff/type_inference` (no subcommand
requirement)
* `cargo run --bin red_knot -- server` (should start the server)
2024-08-06 20:24:49 +05:30
Micha Reiser
846f57fd15 Update salsa (#12711) 2024-08-06 13:17:39 +00:00
Micha Reiser
8e6aa78796 Remove 'cli' module from red_knot (#12714) 2024-08-06 12:10:36 +00:00
Dhruv Manilawala
e91a0fe94a [red-knot] Implement basic LSP server (#12624)
## Summary

This PR adds basic LSP implementation for the Red Knot project.

This is basically a fork of the existing `ruff_server` crate into a
`red_knot_server` crate. The following are the main differences:
1. The `Session` stores a map from workspace root to the corresponding
Red Knot database (`RootDatabase`).
2. The database is initialized with the newly implemented `LSPSystem`
(implementation of `System` trait)
3. The `LSPSystem` contains the server index corresponding to each
workspace and an underlying OS system implementation. For certain
methods, the system first checks if there's an open document in LSP
system and returns the information from that. Otherwise, it falls back
to the OS system to get that information. These methods are
`path_metadata`, `read_to_string` and `read_to_notebook`
4. Add `as_any_mut` method for `System`

**Why fork?**

Forking allows us to experiment with the functionalities that are
specific to Red Knot. The architecture is completely different and so
the requirements for an LSP implementation are different as well. For
example, Red Knot only supports a single workspace, so the LSP system
needs to map the multi-workspace support to each Red Knot instance. In
the end, the server code isn't too big, it will be easier to implement
Red Knot specific functionality without worrying about existing server
limitations and it shouldn't be difficult to port the existing server.

## Review

Most of the server files hasn't been changed. I'm going to list down the
files that have been changed along with highlight the specific part of
the file that's changed from the existing server code.

Changed files:
* Red Knot CLI implementation:
https://github.com/astral-sh/ruff/pull/12624/files#diff-579596339a29d3212a641232e674778c339b446de33b890c7fdad905b5eb50e1
* In
https://github.com/astral-sh/ruff/pull/12624/files#diff-b9a9041a8a2bace014bf3687c3ef0512f25e0541f112fad6131b14242f408db6,
server capabilities have been updated, dynamic capability registration
is removed
* In
https://github.com/astral-sh/ruff/pull/12624/files#diff-b9a9041a8a2bace014bf3687c3ef0512f25e0541f112fad6131b14242f408db6,
the API for `clear_diagnostics` now take in a `Url` instead of
`DocumentQuery` as the document version doesn't matter when clearing
diagnostics after a document is closed
*
[`did_close`](https://github.com/astral-sh/ruff/pull/12624/files#diff-9271370102a6f3be8defaca40c82485b0048731942520b491a3bdd2ee0e25493),
[`did_close_notebook`](https://github.com/astral-sh/ruff/pull/12624/files#diff-96fb53ffb12c1694356e17313e4bb37b3f0931e887878b5d7c896c19ff60283b),
[`did_open`](https://github.com/astral-sh/ruff/pull/12624/files#diff-60e852cf1aa771e993131cabf98eb4c467963a8328f10eccdb43b3e8f0f1fb12),
[`did_open_notebook`](https://github.com/astral-sh/ruff/pull/12624/files#diff-ac356eb5e36c3b2c1c135eda9dfbcab5c12574d1cb77c71f7da8dbcfcfb2d2f1)
are updated to open / close file from the corresponding Red Knot
workspace
* The [diagnostic
handler](https://github.com/astral-sh/ruff/pull/12624/files#diff-4475f318fd0290d0292834569a7df5699debdcc0a453b411b8c3d329f1b879d9)
is updated to request diagnostics from Red Knot
* The [`Session::new`] method in
https://github.com/astral-sh/ruff/pull/12624/files#diff-55c96201296200c1cab37c8b0407b6c733381374b94be7ae50563bfe95264e4d
is updated to construct the Red Knot databases for each workspace. It
also contains the `index_mut` and `MutIndexGuard` implementation
* And, `LSPSystem` implementation is in
https://github.com/astral-sh/ruff/pull/12624/files#diff-4ed62bd359c43b0bf1a13f04349dcd954966934bb8d544de7813f974182b489e

## Test Plan

First, configure VS Code to use the `red_knot` binary

1. Build the `red_knot` binary by `cargo build`
2. Update the VS Code extension to specify the path to this binary
```json
{
	"ruff.path": ["/path/to/ruff/target/debug/red_knot"]
}
```
3. Restart VS Code

Now, open a file containing red-knot specific diagnostics, close the
file and validate that diagnostics disappear.
2024-08-06 11:27:30 +00:00
Micha Reiser
d2c627efb3 Use standard allocator for wasm (#12713) 2024-08-06 11:20:47 +00:00
Micha Reiser
10e977d5f5 [red-knot] Add basic WASM API (#12654) 2024-08-06 09:21:42 +02:00
Auguste Lalande
f0318ff889 [pydoclint] Consider DOC201 satisfied if docstring begins with "Returns" (#12675)
<!--
Thank you for contributing to Ruff! To help us out with reviewing,
please consider the following:

- Does this pull request include a summary of the change? (See below.)
- Does this pull request include a descriptive title?
- Does this pull request include references to any relevant issues?
-->

## Summary

Resolves #12636

Consider docstrings which begin with the word "Returns" as having
satisfactorily documented they're returns. For example
```python
def f():
    """Returns 1."""
    return 1
```
is valid.

## Test Plan

Added example to test fixture.

---------

Co-authored-by: Dhruv Manilawala <dhruvmanila@gmail.com>
2024-08-06 06:46:38 +00:00
Dhruv Manilawala
5cc3fed9a8 [red-knot] Infer float and complex literal expressions (#12689)
## Summary

This PR implements type inference for float and complex literal
expressions.

## Test Plan

Add test cases for both types.
2024-08-06 06:24:28 +00:00
Steve C
39dd732e27 [refurb] - fix unused autofix for implicit-cwd (FURB177) (#12708) 2024-08-06 08:09:35 +02:00
Dylan
52630a1d55 [flake8-comprehensions] Set comprehensions not a violation for sum in unnecessary-comprehension-in-call (C419) (#12691)
## Summary

Removes set comprehension as a violation for `sum` when checking `C419`,
because set comprehension may de-duplicate entries in a generator,
thereby modifying the value of the sum.

Closes #12690.
2024-08-06 02:30:58 +00:00
Steve C
7b5fd63ce8 [flake8-pyi] - add autofix for future-annotations-in-stub (PYI044) (#12676)
## Summary

add autofix for `PYI044`

## Test Plan

`cargo test`
2024-08-05 22:27:55 -04:00
Alex Waygood
5499821c67 [red-knot] Rename workspace_root variables in the module resolver to src_root (#12697)
Fixes #12337
2024-08-05 23:07:18 +01:00
Alex Waygood
7ee7c68f36 Add a new script to generate builtin module names (#12696) 2024-08-05 21:33:36 +01:00
Carl Meyer
2393d19f91 [red-knot] infer instance types for builtins (#12695)
Previously we wrongly inferred the type of the builtin type itself (e.g.
`Literal[int]`); we need to infer the instance type instead.
2024-08-05 13:32:42 -07:00
Dhruv Manilawala
a8e2ba508e [red-knot] Infer boolean literal expression (#12688)
## Summary

This PR implements type inference for boolean literal expressions.

## Test Plan

Add test cases for `True` and `False`.
2024-08-05 11:30:53 -07:00
Alex Waygood
0b4d3ce39b TRY002: fixup docs (#12683) 2024-08-05 08:56:12 +00:00
epenet
0a345dc627 [tryceratops] Add BaseException to raise-vanilla-class rule (TRY002) (#12620) 2024-08-05 09:45:49 +01:00
Micha Reiser
ff2aa3ea00 Revert "Remove criterion/codspeed compat layer (#12524)" (#12680) 2024-08-05 07:49:04 +00:00
Micha Reiser
0d3bad877d Fix module resolver symlink test on macOs (#12682) 2024-08-05 07:22:54 +00:00
Micha Reiser
756060d676 Upgrade Salsa to a version with a 32bit compatible concurrent vec (#12679) 2024-08-05 08:50:32 +02:00
Micha Reiser
b647f3fba8 Disable testing ruff_benchmark by default (#12678) 2024-08-05 06:15:52 +00:00
Dhruv Manilawala
82e69ebf23 Update broken links in the documentation (#12677)
## Summary

Running `mkdocs server -f mkdocs.insiders.yml` gave warnings about these
broken links.

## Test plan

I built the docs locally and verified that the updated links work
properly.
2024-08-05 05:35:23 +00:00
renovate[bot]
2c79045342 Update Rust crate pep440_rs to v0.6.6 (#12666) 2024-08-04 22:42:43 -04:00
Charlie Marsh
3497f5257b Add preview note to unnecessary-comprehension-in-call (#12673) 2024-08-05 02:27:00 +00:00
Dylan
25aabec814 [flake8-comprehensions] Account for list and set comprehensions in unnecessary-literal-within-tuple-call (C409) (#12657)
## Summary

Make it a violation of `C409` to call `tuple` with a list or set
comprehension, and
implement the (unsafe) fix of calling the `tuple` with the underlying
generator instead.

Closes #12648.

## Test Plan

Test fixture updated, cargo test, docs checked for updated description.
2024-08-04 22:14:52 -04:00
renovate[bot]
0e71485ea9 Update Rust crate regex to v1.10.6 (#12667) 2024-08-04 22:10:40 -04:00
renovate[bot]
43a9d282f7 Update Rust crate ordermap to v0.5.1 (#12665) 2024-08-04 22:10:32 -04:00
renovate[bot]
6f357b8b45 Update Rust crate tempfile to v3.11.0 (#12671) 2024-08-05 02:08:20 +00:00
renovate[bot]
73d9f11a9c Update pre-commit dependencies (#12670) 2024-08-05 02:08:07 +00:00
renovate[bot]
d6c6db5a44 Update NPM Development dependencies (#12672) 2024-08-04 22:07:55 -04:00
renovate[bot]
56d985a972 Update Rust crate toml to v0.8.19 (#12669) 2024-08-04 22:07:44 -04:00
renovate[bot]
b3e0655cc9 Update Rust crate serde_json to v1.0.122 (#12668) 2024-08-04 22:07:35 -04:00
renovate[bot]
06baffec9e Update Rust crate clap to v4.5.13 (#12664) 2024-08-04 22:07:26 -04:00
Steve C
67a2ae800a [ruff] - add autofix zip-instead-of-pairwise (RUF007) (#12663)
## Summary

Adds autofix for `RUF007`

## Test Plan

`cargo test`, however I get errors for `test resolver::tests::symlink
... FAILED` which seems to not be my fault
2024-08-04 21:57:50 -04:00
InSync
7a2c75e2fc Replace ruff-lsp links in README.md with links to new documentation page (#12618)
Since `ruff-lsp` has been (semi-)deprecated for sometime, it wouldn't
make sense to mention it in the most prominent sections of the `README`.
Instead, they should point to the new <i>[Editor
Integrations](https://docs.astral.sh/ruff/editors/)</i> documentation
page.
2024-08-04 15:31:36 +05:30
DavideRagazzon
9ee44637ca Fix typo in configuration docs (#12655) 2024-08-04 09:43:51 +02:00
Charlie Marsh
733341ab39 Ignore DOC errors for stub functions (#12651)
## Summary

Closes https://github.com/astral-sh/ruff/issues/12650.
2024-08-03 08:13:21 -04:00
Micha Reiser
341a25eec1 Fix file watching on macOS if a module-search path is a symlink (#12634) 2024-08-03 07:24:07 +00:00
Charlie Marsh
38e178e914 Try both 'Raises' section styles when convention is unspecified (#12649)
## Summary

Closes https://github.com/astral-sh/ruff/issues/12647.
2024-08-02 21:04:46 -04:00
Alex Waygood
daccb3f4f3 [pydoclint] Deduplicate collected exceptions after traversing function bodies (#12642) 2024-08-02 23:17:06 +01:00
Charlie Marsh
c858afe03a [flake8-bugbear] Treat return as equivalent to break (B909) (#12646)
Closes https://github.com/astral-sh/ruff/issues/12640.
2024-08-02 18:14:17 -04:00
Alex Waygood
3c1c3199d0 [pydoclint] Teach rules to understand reraised exceptions as being explicitly raised (#12639)
## Summary

Fixes #12630.

DOC501 and DOC502 now understand functions with constructs like this to
be explicitly raising `TypeError` (which should be documented in a
function's docstring):

```py
try:
    foo():
except TypeError:
    ...
    raise
```

I made an exception for `Exception` and `BaseException`, however.
Constructs like this are reasonably common, and I don't think anybody
would say that it's worth putting in the docstring that it raises "some
kind of generic exception":

```py
try:
    foo()
except BaseException:
    do_some_logging()
    raise
```

## Test Plan

`cargo test -p ruff_linter --lib`
2024-08-02 22:47:22 +01:00
Ran Benita
fbfe2cb2f5 [flake8-async] Fix false positives with multiple async with items (ASYNC100) (#12643)
## Summary

Please see
https://github.com/astral-sh/ruff/pull/12605#discussion_r1699957443 for
a description of the issue.

They way I fixed it is to get the *last* timeout item in the `with`, and
if it's an `async with` and there are items after it, then don't trigger
the lint.

## Test Plan

Updated the fixture with some more cases.
2024-08-02 21:25:13 +00:00
Carl Meyer
1c311e4fdb [red-knot] update benchmark to run on tomllib (#12635)
Changes the red-knot benchmark to run on the stdlib "tomllib" library
(which is self-contained, four files, uses type annotations) instead of
on very small bits of handwritten code.

Also remove the `without_parse` benchmark: now that we are running on
real code that uses typeshed, we'd either have to pre-parse all of
typeshed (slow) or find some way to determine which typeshed modules
will be used by the benchmark (not feasible with reasonable complexity.)

## Test Plan

`cargo bench -p ruff_benchmark --bench red_knot`
2024-08-02 11:23:52 -07:00
Micha Reiser
12177a42e3 Set durabilities for low-durability fields on high-durability inputs (#12627) 2024-08-02 19:42:34 +02:00
Micha Reiser
dfb08856eb Fix file watcher stop data race (#12626) 2024-08-02 19:02:49 +02:00
Auguste Lalande
94d817e1a5 [pydoclint] Add docstring-missing-yields amd docstring-extraneous-yields (DOC402, DOC403) (#12538) 2024-08-02 17:55:42 +01:00
ember91
9296bd4e3f Fix a typo (#12633)
Co-authored-by: Emil Berg <emil.berg@ericsson.com>
2024-08-02 16:39:27 +01:00
894 changed files with 12498 additions and 2478 deletions

View File

@@ -8,7 +8,7 @@
semanticCommits: "disabled",
separateMajorMinor: false,
prHourlyLimit: 10,
enabledManagers: ["github-actions", "pre-commit", "cargo", "pep621", "npm"],
enabledManagers: ["github-actions", "pre-commit", "cargo", "pep621", "pip_requirements", "npm"],
cargo: {
// See https://docs.renovatebot.com/configuration-options/#rangestrategy
rangeStrategy: "update-lockfile",
@@ -16,6 +16,9 @@
pep621: {
fileMatch: ["^(python|scripts)/.*pyproject\\.toml$"],
},
pip_requirements: {
fileMatch: ["^docs/requirements.*\\.txt$"],
},
npm: {
fileMatch: ["^playground/.*package\\.json$"],
},
@@ -48,6 +51,14 @@
matchManagers: ["cargo"],
enabled: false,
},
{
// `mkdocs-material` requires a manual update to keep the version in sync
// with `mkdocs-material-insider`.
// See: https://squidfunk.github.io/mkdocs-material/insiders/upgrade/
matchManagers: ["pip_requirements"],
matchPackagePatterns: ["mkdocs-material"],
enabled: false,
},
{
groupName: "pre-commit dependencies",
matchManagers: ["pre-commit"],

View File

@@ -111,7 +111,7 @@ jobs:
- name: "Clippy"
run: cargo clippy --workspace --all-targets --all-features --locked -- -D warnings
- name: "Clippy (wasm)"
run: cargo clippy -p ruff_wasm --target wasm32-unknown-unknown --all-features --locked -- -D warnings
run: cargo clippy -p ruff_wasm -p red_knot_wasm --target wasm32-unknown-unknown --all-features --locked -- -D warnings
cargo-test-linux:
name: "cargo test (linux)"
@@ -191,10 +191,14 @@ jobs:
cache-dependency-path: playground/package-lock.json
- uses: jetli/wasm-pack-action@v0.4.0
- uses: Swatinem/rust-cache@v2
- name: "Run wasm-pack"
- name: "Test ruff_wasm"
run: |
cd crates/ruff_wasm
wasm-pack test --node
- name: "Test red_knot_wasm"
run: |
cd crates/red_knot_wasm
wasm-pack test --node
cargo-build-release:
name: "cargo build (release)"
@@ -616,7 +620,7 @@ jobs:
- uses: Swatinem/rust-cache@v2
- name: "Build benchmarks"
run: cargo codspeed build -p ruff_benchmark
run: cargo codspeed build --features codspeed -p ruff_benchmark
- name: "Run benchmarks"
uses: CodSpeedHQ/action@v3

View File

@@ -37,13 +37,13 @@ jobs:
- name: Sync typeshed
id: sync
run: |
rm -rf ruff/crates/red_knot_module_resolver/vendor/typeshed
mkdir ruff/crates/red_knot_module_resolver/vendor/typeshed
cp typeshed/README.md ruff/crates/red_knot_module_resolver/vendor/typeshed
cp typeshed/LICENSE ruff/crates/red_knot_module_resolver/vendor/typeshed
cp -r typeshed/stdlib ruff/crates/red_knot_module_resolver/vendor/typeshed/stdlib
rm -rf ruff/crates/red_knot_module_resolver/vendor/typeshed/stdlib/@tests
git -C typeshed rev-parse HEAD > ruff/crates/red_knot_module_resolver/vendor/typeshed/source_commit.txt
rm -rf ruff/crates/red_knot_python_semantic/vendor/typeshed
mkdir ruff/crates/red_knot_python_semantic/vendor/typeshed
cp typeshed/README.md ruff/crates/red_knot_python_semantic/vendor/typeshed
cp typeshed/LICENSE ruff/crates/red_knot_python_semantic/vendor/typeshed
cp -r typeshed/stdlib ruff/crates/red_knot_python_semantic/vendor/typeshed/stdlib
rm -rf ruff/crates/red_knot_python_semantic/vendor/typeshed/stdlib/@tests
git -C typeshed rev-parse HEAD > ruff/crates/red_knot_python_semantic/vendor/typeshed/source_commit.txt
- name: Commit the changes
id: commit
if: ${{ steps.sync.outcome == 'success' }}

8
.gitignore vendored
View File

@@ -21,6 +21,14 @@ flamegraph.svg
# `CARGO_TARGET_DIR=target-llvm-lines RUSTFLAGS="-Csymbol-mangling-version=v0" cargo llvm-lines -p ruff --lib`
/target*
# samply profiles
profile.json
# tracing-flame traces
tracing.folded
tracing-flamechart.svg
tracing-flamegraph.svg
###
# Rust.gitignore
###

View File

@@ -2,15 +2,15 @@ fail_fast: true
exclude: |
(?x)^(
crates/red_knot_module_resolver/vendor/.*|
crates/red_knot_python_semantic/vendor/.*|
crates/red_knot_workspace/resources/.*|
crates/ruff_linter/resources/.*|
crates/ruff_linter/src/rules/.*/snapshots/.*|
crates/ruff/resources/.*|
crates/ruff_python_formatter/resources/.*|
crates/ruff_python_formatter/tests/snapshots/.*|
crates/ruff_python_resolver/resources/.*|
crates/ruff_python_resolver/tests/snapshots/.*|
crates/red_knot_workspace/resources/.*
crates/ruff_python_resolver/tests/snapshots/.*
)$
repos:
@@ -43,7 +43,7 @@ repos:
)$
- repo: https://github.com/crate-ci/typos
rev: v1.23.5
rev: v1.23.6
hooks:
- id: typos
@@ -57,7 +57,7 @@ repos:
pass_filenames: false # This makes it a lot faster
- repo: https://github.com/astral-sh/ruff-pre-commit
rev: v0.5.5
rev: v0.5.6
hooks:
- id: ruff-format
- id: ruff

View File

@@ -1,5 +1,43 @@
# Changelog
## 0.5.7
### Preview features
- \[`flake8-comprehensions`\] Account for list and set comprehensions in `unnecessary-literal-within-tuple-call` (`C409`) ([#12657](https://github.com/astral-sh/ruff/pull/12657))
- \[`flake8-pyi`\] Add autofix for `future-annotations-in-stub` (`PYI044`) ([#12676](https://github.com/astral-sh/ruff/pull/12676))
- \[`flake8-return`\] Avoid syntax error when auto-fixing `RET505` with mixed indentation (space and tabs) ([#12740](https://github.com/astral-sh/ruff/pull/12740))
- \[`pydoclint`\] Add `docstring-missing-yields` (`DOC402`) and `docstring-extraneous-yields` (`DOC403`) ([#12538](https://github.com/astral-sh/ruff/pull/12538))
- \[`pydoclint`\] Avoid `DOC201` if docstring begins with "Return", "Returns", "Yield", or "Yields" ([#12675](https://github.com/astral-sh/ruff/pull/12675))
- \[`pydoclint`\] Deduplicate collected exceptions after traversing function bodies (`DOC501`) ([#12642](https://github.com/astral-sh/ruff/pull/12642))
- \[`pydoclint`\] Ignore `DOC` errors for stub functions ([#12651](https://github.com/astral-sh/ruff/pull/12651))
- \[`pydoclint`\] Teach rules to understand reraised exceptions as being explicitly raised (`DOC501`, `DOC502`) ([#12639](https://github.com/astral-sh/ruff/pull/12639))
- \[`ruff`\] Implement `incorrectly-parenthesized-tuple-in-subscript` (`RUF031`) ([#12480](https://github.com/astral-sh/ruff/pull/12480))
- \[`ruff`\] Mark `RUF023` fix as unsafe if `__slots__` is not a set and the binding is used elsewhere ([#12692](https://github.com/astral-sh/ruff/pull/12692))
### Rule changes
- \[`refurb`\] Add autofix for `implicit-cwd` (`FURB177`) ([#12708](https://github.com/astral-sh/ruff/pull/12708))
- \[`ruff`\] Add autofix for `zip-instead-of-pairwise` (`RUF007`) ([#12663](https://github.com/astral-sh/ruff/pull/12663))
- \[`tryceratops`\] Add `BaseException` to `raise-vanilla-class` rule (`TRY002`) ([#12620](https://github.com/astral-sh/ruff/pull/12620))
### Server
- Ignore non-file workspace URL; Ruff will display a warning notification in this case ([#12725](https://github.com/astral-sh/ruff/pull/12725))
### CLI
- Fix cache invalidation for nested `pyproject.toml` files ([#12727](https://github.com/astral-sh/ruff/pull/12727))
### Bug fixes
- \[`flake8-async`\] Fix false positives with multiple `async with` items (`ASYNC100`) ([#12643](https://github.com/astral-sh/ruff/pull/12643))
- \[`flake8-bandit`\] Avoid false-positives for list concatenations in SQL construction (`S608`) ([#12720](https://github.com/astral-sh/ruff/pull/12720))
- \[`flake8-bugbear`\] Treat `return` as equivalent to `break` (`B909`) ([#12646](https://github.com/astral-sh/ruff/pull/12646))
- \[`flake8-comprehensions`\] Set comprehensions not a violation for `sum` in `unnecessary-comprehension-in-call` (`C419`) ([#12691](https://github.com/astral-sh/ruff/pull/12691))
- \[`flake8-simplify`\] Parenthesize conditions based on precedence when merging if arms (`SIM114`) ([#12737](https://github.com/astral-sh/ruff/pull/12737))
- \[`pydoclint`\] Try both 'Raises' section styles when convention is unspecified (`DOC501`) ([#12649](https://github.com/astral-sh/ruff/pull/12649))
## 0.5.6
Ruff 0.5.6 automatically enables linting and formatting of notebooks in *preview mode*.

272
Cargo.lock generated
View File

@@ -133,6 +133,12 @@ version = "1.0.86"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b3d1d046238990b9cf5bcde22a3fb3584ee5cf65fb2765f454ed428c7a0063da"
[[package]]
name = "append-only-vec"
version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "74d9f7083455f1a474276ccd32374958d2cb591024aac45101c7623b10271347"
[[package]]
name = "arc-swap"
version = "1.7.1"
@@ -188,18 +194,6 @@ version = "2.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de"
[[package]]
name = "boomphf"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "617e2d952880a00583ddb9237ac3965732e8df6a92a8e7bcc054100ec467ec3b"
dependencies = [
"crossbeam-utils",
"log",
"rayon",
"wyhash",
]
[[package]]
name = "bstr"
version = "1.10.0"
@@ -326,9 +320,9 @@ dependencies = [
[[package]]
name = "clap"
version = "4.5.11"
version = "4.5.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "35723e6a11662c2afb578bcf0b88bf6ea8e21282a953428f240574fcc3a2b5b3"
checksum = "0fbb260a053428790f3de475e304ff84cdbc4face759ea7a3e64c1edd938a7fc"
dependencies = [
"clap_builder",
"clap_derive",
@@ -336,9 +330,9 @@ dependencies = [
[[package]]
name = "clap_builder"
version = "4.5.11"
version = "4.5.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "49eb96cbfa7cfa35017b7cd548c75b14c3118c98b423041d70562665e07fb0fa"
checksum = "64b17d7ea74e9f833c7dbf2cbe4fb12ff26783eda4782a8975b72f895c9b4d99"
dependencies = [
"anstream",
"anstyle",
@@ -379,9 +373,9 @@ dependencies = [
[[package]]
name = "clap_derive"
version = "4.5.11"
version = "4.5.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5d029b67f89d30bbb547c89fd5161293c0aec155fc691d7924b64550662db93e"
checksum = "501d359d5f3dcaf6ecdeee48833ae73ec6e42723a1e52419c79abf9507eec0a0"
dependencies = [
"heck",
"proc-macro2",
@@ -1053,9 +1047,9 @@ dependencies = [
[[package]]
name = "indexmap"
version = "2.2.6"
version = "2.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "168fb715dda47215e360912c096649d23d58bf392ac62f73919e831745e40f26"
checksum = "de3fc2e30ba82dd1b3911c8de1ffc143c74a914a14e99514d7637e3099df5ea0"
dependencies = [
"equivalent",
"hashbrown",
@@ -1206,9 +1200,9 @@ checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b"
[[package]]
name = "jobserver"
version = "0.1.31"
version = "0.1.32"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d2b099aaa34a9751c5bf0878add70444e1ed2dd73f347be99003d4577277de6e"
checksum = "48d1dbcbbeb6a7fec7e059840aa538bd62aaccf972c7346c4d9d2059312853d0"
dependencies = [
"libc",
]
@@ -1488,11 +1482,11 @@ dependencies = [
[[package]]
name = "nu-ansi-term"
version = "0.50.0"
version = "0.50.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dd2800e1520bdc966782168a627aa5d1ad92e33b984bf7c7615d31280c83ff14"
checksum = "d4a28e057d01f97e61255210fcff094d74ed0466038633e95017f5beb68e4399"
dependencies = [
"windows-sys 0.48.0",
"windows-sys 0.52.0",
]
[[package]]
@@ -1530,87 +1524,13 @@ checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d"
[[package]]
name = "ordermap"
version = "0.5.0"
version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ab5a8e22be64dfa1123429350872e7be33594dbf5ae5212c90c5890e71966d1d"
checksum = "8c81974681ab4f0cc9fe49cad56f821d1cc67a08cd2caa9b5d58b0adaa5dd36d"
dependencies = [
"indexmap",
]
[[package]]
name = "orx-concurrent-ordered-bag"
version = "2.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9aa866e2be4aa03927eddb481e7c479d5109fe3121324fb7db6d97f91adf9876"
dependencies = [
"orx-fixed-vec",
"orx-pinned-concurrent-col",
"orx-pinned-vec",
"orx-pseudo-default",
"orx-split-vec",
]
[[package]]
name = "orx-concurrent-vec"
version = "2.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c5912426ffb660f8b61e8f0812a1d07400803cd5513969d2c7af4d69602ba8a1"
dependencies = [
"orx-concurrent-ordered-bag",
"orx-fixed-vec",
"orx-pinned-concurrent-col",
"orx-pinned-vec",
"orx-pseudo-default",
"orx-split-vec",
]
[[package]]
name = "orx-fixed-vec"
version = "3.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7f69466c7c1fc2e1f00b58e39059b78c438b9fad144d1937ef177ecfc413e997"
dependencies = [
"orx-pinned-vec",
"orx-pseudo-default",
]
[[package]]
name = "orx-pinned-concurrent-col"
version = "2.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fdbcb1fa05dc1676f1c9cf19f443b3d2d2ca5835911477d22fa77cad8b79208d"
dependencies = [
"orx-fixed-vec",
"orx-pinned-vec",
"orx-pseudo-default",
"orx-split-vec",
]
[[package]]
name = "orx-pinned-vec"
version = "3.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c1071baf586de45722668234bddf56c52c1ece6a6153d16541bbb0505f0ac055"
dependencies = [
"orx-pseudo-default",
]
[[package]]
name = "orx-pseudo-default"
version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a2f627c439e723fa78e410a0faba89047a8a47d0dc013da5c0e05806e8a6cddb"
[[package]]
name = "orx-split-vec"
version = "3.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "52b9dbfa8c7069ae73a890870d3aa9097a897d616751d3d0278f2b42d5214730"
dependencies = [
"orx-pinned-vec",
"orx-pseudo-default",
]
[[package]]
name = "os_str_bytes"
version = "7.0.0"
@@ -1726,9 +1646,9 @@ dependencies = [
[[package]]
name = "pep440_rs"
version = "0.6.0"
version = "0.6.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ca0a570e7ec9171250cac57614e901f62408094b54b3798bb920d3cf0d4a0e09"
checksum = "466eada3179c2e069ca897b99006cbb33f816290eaeec62464eea907e22ae385"
dependencies = [
"once_cell",
"serde",
@@ -1940,34 +1860,46 @@ name = "red_knot"
version = "0.0.0"
dependencies = [
"anyhow",
"chrono",
"clap",
"colored",
"countme",
"crossbeam",
"ctrlc",
"filetime",
"rayon",
"red_knot_module_resolver",
"red_knot_python_semantic",
"red_knot_server",
"red_knot_workspace",
"ruff_db",
"salsa",
"tempfile",
"tracing",
"tracing-flame",
"tracing-subscriber",
"tracing-tree",
]
[[package]]
name = "red_knot_module_resolver"
name = "red_knot_python_semantic"
version = "0.0.0"
dependencies = [
"anyhow",
"bitflags 2.6.0",
"camino",
"compact_str",
"countme",
"hashbrown",
"insta",
"once_cell",
"ordermap",
"path-slash",
"ruff_db",
"ruff_index",
"ruff_python_ast",
"ruff_python_parser",
"ruff_python_stdlib",
"ruff_text_size",
"rustc-hash 2.0.0",
"salsa",
"tempfile",
@@ -1977,23 +1909,45 @@ dependencies = [
]
[[package]]
name = "red_knot_python_semantic"
name = "red_knot_server"
version = "0.0.0"
dependencies = [
"anyhow",
"bitflags 2.6.0",
"countme",
"hashbrown",
"ordermap",
"red_knot_module_resolver",
"crossbeam",
"jod-thread",
"libc",
"lsp-server",
"lsp-types",
"red_knot_python_semantic",
"red_knot_workspace",
"ruff_db",
"ruff_index",
"ruff_linter",
"ruff_notebook",
"ruff_python_ast",
"ruff_python_parser",
"ruff_source_file",
"ruff_text_size",
"rustc-hash 2.0.0",
"salsa",
"serde",
"serde_json",
"shellexpand",
"tracing",
"tracing-subscriber",
]
[[package]]
name = "red_knot_wasm"
version = "0.0.0"
dependencies = [
"console_error_panic_hook",
"console_log",
"js-sys",
"log",
"red_knot_python_semantic",
"red_knot_workspace",
"ruff_db",
"ruff_notebook",
"wasm-bindgen",
"wasm-bindgen-test",
]
[[package]]
@@ -2003,12 +1957,13 @@ dependencies = [
"anyhow",
"crossbeam",
"notify",
"red_knot_module_resolver",
"red_knot_python_semantic",
"ruff_cache",
"ruff_db",
"ruff_python_ast",
"rustc-hash 2.0.0",
"salsa",
"thiserror",
"tracing",
]
@@ -2034,9 +1989,9 @@ dependencies = [
[[package]]
name = "regex"
version = "1.10.5"
version = "1.10.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b91213439dad192326a0d7c6ee3955910425f441d7038e0d6933b0aec5c4517f"
checksum = "4219d74c6b67a3654a9fbebc4b419e22126d13d2f3c4a07ee0cb61ff79a79619"
dependencies = [
"aho-corasick",
"memchr",
@@ -2093,7 +2048,7 @@ dependencies = [
[[package]]
name = "ruff"
version = "0.5.6"
version = "0.5.7"
dependencies = [
"anyhow",
"argfile",
@@ -2148,8 +2103,10 @@ name = "ruff_benchmark"
version = "0.0.0"
dependencies = [
"codspeed-criterion-compat",
"criterion",
"mimalloc",
"once_cell",
"red_knot_python_semantic",
"red_knot_workspace",
"ruff_db",
"ruff_linter",
@@ -2200,6 +2157,9 @@ dependencies = [
"salsa",
"tempfile",
"tracing",
"tracing-subscriber",
"tracing-tree",
"web-time",
"zip",
]
@@ -2279,7 +2239,7 @@ dependencies = [
[[package]]
name = "ruff_linter"
version = "0.5.6"
version = "0.5.7"
dependencies = [
"aho-corasick",
"annotate-snippets 0.9.2",
@@ -2303,7 +2263,7 @@ dependencies = [
"once_cell",
"path-absolutize",
"pathdiff",
"pep440_rs 0.6.0",
"pep440_rs 0.6.6",
"pyproject-toml",
"quick-junit",
"regex",
@@ -2599,7 +2559,7 @@ dependencies = [
[[package]]
name = "ruff_wasm"
version = "0.5.6"
version = "0.5.7"
dependencies = [
"console_error_panic_hook",
"console_log",
@@ -2638,7 +2598,7 @@ dependencies = [
"matchit",
"path-absolutize",
"path-slash",
"pep440_rs 0.6.0",
"pep440_rs 0.6.6",
"regex",
"ruff_cache",
"ruff_formatter",
@@ -2739,15 +2699,15 @@ checksum = "e86697c916019a8588c99b5fac3cead74ec0b4b819707a682fd4d23fa0ce1ba1"
[[package]]
name = "salsa"
version = "0.18.0"
source = "git+https://github.com/MichaReiser/salsa.git?rev=0cae5c52a3240172ef0be5c9d19e63448c53397c#0cae5c52a3240172ef0be5c9d19e63448c53397c"
source = "git+https://github.com/MichaReiser/salsa.git?tag=red-knot-0.0.1#ece083e15b79f155f9e4368ec1318cec9a08d88b"
dependencies = [
"append-only-vec",
"arc-swap",
"boomphf",
"crossbeam",
"dashmap 6.0.1",
"hashlink",
"indexmap",
"orx-concurrent-vec",
"lazy_static",
"parking_lot",
"rustc-hash 2.0.0",
"salsa-macro-rules",
@@ -2759,12 +2719,12 @@ dependencies = [
[[package]]
name = "salsa-macro-rules"
version = "0.1.0"
source = "git+https://github.com/MichaReiser/salsa.git?rev=0cae5c52a3240172ef0be5c9d19e63448c53397c#0cae5c52a3240172ef0be5c9d19e63448c53397c"
source = "git+https://github.com/MichaReiser/salsa.git?tag=red-knot-0.0.1#ece083e15b79f155f9e4368ec1318cec9a08d88b"
[[package]]
name = "salsa-macros"
version = "0.18.0"
source = "git+https://github.com/MichaReiser/salsa.git?rev=0cae5c52a3240172ef0be5c9d19e63448c53397c#0cae5c52a3240172ef0be5c9d19e63448c53397c"
source = "git+https://github.com/MichaReiser/salsa.git?tag=red-knot-0.0.1#ece083e15b79f155f9e4368ec1318cec9a08d88b"
dependencies = [
"heck",
"proc-macro2",
@@ -2868,9 +2828,9 @@ dependencies = [
[[package]]
name = "serde_json"
version = "1.0.121"
version = "1.0.122"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4ab380d7d9f22ef3f21ad3e6c1ebe8e4fc7a2000ccba2e4d71fc96f15b2cb609"
checksum = "784b6203951c57ff748476b126ccb5e8e2959a5c19e5c617ab1956be3dbc68da"
dependencies = [
"itoa",
"memchr",
@@ -3051,12 +3011,13 @@ dependencies = [
[[package]]
name = "tempfile"
version = "3.10.1"
version = "3.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "85b77fafb263dd9d05cbeac119526425676db3784113aa9295c88498cbf8bff1"
checksum = "b8fcd239983515c23a32fb82099f97d0b11b8c72f654ed659363a95c3dad7a53"
dependencies = [
"cfg-if",
"fastrand",
"once_cell",
"rustix",
"windows-sys 0.52.0",
]
@@ -3194,9 +3155,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
[[package]]
name = "toml"
version = "0.8.16"
version = "0.8.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "81967dd0dd2c1ab0bc3468bd7caecc32b8a4aa47d0c8c695d8c2b2108168d62c"
checksum = "a1ed1f98e3fdc28d6d910e6737ae6ab1a93bf1985935a1193e68f93eeb68d24e"
dependencies = [
"serde",
"serde_spanned",
@@ -3206,18 +3167,18 @@ dependencies = [
[[package]]
name = "toml_datetime"
version = "0.6.7"
version = "0.6.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f8fb9f64314842840f1d940ac544da178732128f1c78c21772e876579e0da1db"
checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41"
dependencies = [
"serde",
]
[[package]]
name = "toml_edit"
version = "0.22.17"
version = "0.22.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8d9f8729f5aea9562aac1cc0441f5d6de3cff1ee0c5d67293eeca5eb36ee7c16"
checksum = "583c44c02ad26b0c3f3066fe629275e50627026c51ac2e595cca4c230ce1ce1d"
dependencies = [
"indexmap",
"serde",
@@ -3259,6 +3220,17 @@ dependencies = [
"valuable",
]
[[package]]
name = "tracing-flame"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0bae117ee14789185e129aaee5d93750abe67fdc5a9a62650452bfe4e122a3a9"
dependencies = [
"lazy_static",
"tracing",
"tracing-subscriber",
]
[[package]]
name = "tracing-indicatif"
version = "0.3.6"
@@ -3306,7 +3278,7 @@ version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f459ca79f1b0d5f71c54ddfde6debfc59c8b6eeb46808ae492077f739dc7b49c"
dependencies = [
"nu-ansi-term 0.50.0",
"nu-ansi-term 0.50.1",
"tracing-core",
"tracing-log",
"tracing-subscriber",
@@ -3640,6 +3612,16 @@ dependencies = [
"wasm-bindgen",
]
[[package]]
name = "web-time"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb"
dependencies = [
"js-sys",
"wasm-bindgen",
]
[[package]]
name = "webpki-roots"
version = "0.26.1"
@@ -3851,9 +3833,9 @@ checksum = "bec47e5bfd1bff0eeaf6d8b485cc1074891a197ab4225d504cb7a1ab88b02bf0"
[[package]]
name = "winnow"
version = "0.6.6"
version = "0.6.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f0c976aaaa0e1f90dbb21e9587cdaf1d9679a1cde8875c0d6bd83ab96a208352"
checksum = "68a9bda4691f099d435ad181000724da8e5899daa10713c2d432552b9ccd3a6f"
dependencies = [
"memchr",
]
@@ -3864,15 +3846,6 @@ version = "0.0.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d135d17ab770252ad95e9a872d365cf3090e3be864a34ab46f48555993efc904"
[[package]]
name = "wyhash"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "baf6e163c25e3fac820b4b453185ea2dea3b6a3e0a721d4d23d75bd33734c295"
dependencies = [
"rand_core",
]
[[package]]
name = "yansi"
version = "0.5.1"
@@ -3923,6 +3896,7 @@ dependencies = [
"byteorder",
"crc32fast",
"crossbeam-utils",
"flate2",
"zstd",
]

View File

@@ -35,8 +35,8 @@ ruff_source_file = { path = "crates/ruff_source_file" }
ruff_text_size = { path = "crates/ruff_text_size" }
ruff_workspace = { path = "crates/ruff_workspace" }
red_knot_module_resolver = { path = "crates/red_knot_module_resolver" }
red_knot_python_semantic = { path = "crates/red_knot_python_semantic" }
red_knot_server = { path = "crates/red_knot_server" }
red_knot_workspace = { path = "crates/red_knot_workspace" }
aho-corasick = { version = "1.1.3" }
@@ -58,6 +58,7 @@ console_error_panic_hook = { version = "0.1.7" }
console_log = { version = "1.0.0" }
countme = { version = "3.0.1" }
compact_str = "0.8.0"
criterion = { version = "0.5.1", default-features = false }
crossbeam = { version = "0.8.4" }
dashmap = { version = "6.0.1" }
drop_bomb = { version = "0.1.5" }
@@ -107,7 +108,7 @@ rand = { version = "0.8.5" }
rayon = { version = "1.10.0" }
regex = { version = "1.10.2" }
rustc-hash = { version = "2.0.0" }
salsa = { git = "https://github.com/MichaReiser/salsa.git", rev = "0cae5c52a3240172ef0be5c9d19e63448c53397c" }
salsa = { git = "https://github.com/MichaReiser/salsa.git", tag = "red-knot-0.0.1" }
schemars = { version = "0.8.16" }
seahash = { version = "4.1.0" }
serde = { version = "1.0.197", features = ["derive"] }
@@ -130,8 +131,9 @@ thiserror = { version = "1.0.58" }
tikv-jemallocator = { version = "0.6.0" }
toml = { version = "0.8.11" }
tracing = { version = "0.1.40" }
tracing-flame = { version = "0.2.0" }
tracing-indicatif = { version = "0.3.6" }
tracing-subscriber = { version = "0.3.18", features = ["env-filter"] }
tracing-subscriber = { version = "0.3.18", default-features = false, features = ["env-filter", "fmt"] }
tracing-tree = { version = "0.4.0" }
typed-arena = { version = "2.0.2" }
unic-ucd-category = { version = "0.9" }
@@ -151,12 +153,12 @@ walkdir = { version = "2.3.2" }
wasm-bindgen = { version = "0.2.92" }
wasm-bindgen-test = { version = "0.3.42" }
wild = { version = "2" }
zip = { version = "0.6.6", default-features = false, features = ["zstd"] }
zip = { version = "0.6.6", default-features = false }
[workspace.lints.rust]
unsafe_code = "warn"
unreachable_pub = "warn"
unexpected_cfgs = { level = "warn", check-cfg = ['cfg(fuzzing)'] }
unexpected_cfgs = { level = "warn", check-cfg = ["cfg(fuzzing)", "cfg(codspeed)"] }
[workspace.lints.clippy]
pedantic = { level = "warn", priority = -2 }

View File

@@ -29,14 +29,14 @@ An extremely fast Python linter and code formatter, written in Rust.
- 🐍 Installable via `pip`
- 🛠️ `pyproject.toml` support
- 🤝 Python 3.13 compatibility
- ⚖️ Drop-in parity with [Flake8](https://docs.astral.sh/ruff/faq/#how-does-ruff-compare-to-flake8), isort, and Black
- ⚖️ Drop-in parity with [Flake8](https://docs.astral.sh/ruff/faq/#how-does-ruffs-linter-compare-to-flake8), isort, and [Black](https://docs.astral.sh/ruff/faq/#how-does-ruffs-formatter-compare-to-black)
- 📦 Built-in caching, to avoid re-analyzing unchanged files
- 🔧 Fix support, for automatic error correction (e.g., automatically remove unused imports)
- 📏 Over [800 built-in rules](https://docs.astral.sh/ruff/rules/), with native re-implementations
of popular Flake8 plugins, like flake8-bugbear
- ⌨️ First-party [editor integrations](https://docs.astral.sh/ruff/integrations/) for
[VS Code](https://github.com/astral-sh/ruff-vscode) and [more](https://github.com/astral-sh/ruff-lsp)
- 🌎 Monorepo-friendly, with [hierarchical and cascading configuration](https://docs.astral.sh/ruff/configuration/#pyprojecttoml-discovery)
[VS Code](https://github.com/astral-sh/ruff-vscode) and [more](https://docs.astral.sh/ruff/editors/setup)
- 🌎 Monorepo-friendly, with [hierarchical and cascading configuration](https://docs.astral.sh/ruff/configuration/#config-file-discovery)
Ruff aims to be orders of magnitude faster than alternative tools while integrating more
functionality behind a single, common interface.
@@ -136,8 +136,8 @@ curl -LsSf https://astral.sh/ruff/install.sh | sh
powershell -c "irm https://astral.sh/ruff/install.ps1 | iex"
# For a specific version.
curl -LsSf https://astral.sh/ruff/0.5.6/install.sh | sh
powershell -c "irm https://astral.sh/ruff/0.5.6/install.ps1 | iex"
curl -LsSf https://astral.sh/ruff/0.5.7/install.sh | sh
powershell -c "irm https://astral.sh/ruff/0.5.7/install.ps1 | iex"
```
You can also install Ruff via [Homebrew](https://formulae.brew.sh/formula/ruff), [Conda](https://anaconda.org/conda-forge/ruff),
@@ -170,7 +170,7 @@ Ruff can also be used as a [pre-commit](https://pre-commit.com/) hook via [`ruff
```yaml
- repo: https://github.com/astral-sh/ruff-pre-commit
# Ruff version.
rev: v0.5.6
rev: v0.5.7
hooks:
# Run the linter.
- id: ruff
@@ -179,8 +179,7 @@ Ruff can also be used as a [pre-commit](https://pre-commit.com/) hook via [`ruff
- id: ruff-format
```
Ruff can also be used as a [VS Code extension](https://github.com/astral-sh/ruff-vscode) or
alongside any other editor through the [Ruff LSP](https://github.com/astral-sh/ruff-lsp).
Ruff can also be used as a [VS Code extension](https://github.com/astral-sh/ruff-vscode) or with [various other editors](https://docs.astral.sh/ruff/editors/setup).
Ruff can also be used as a [GitHub Action](https://github.com/features/actions) via
[`ruff-action`](https://github.com/chartboost/ruff-action):

View File

@@ -1,6 +1,6 @@
[files]
# https://github.com/crate-ci/typos/issues/868
extend-exclude = ["crates/red_knot_module_resolver/vendor/**/*", "**/resources/**/*", "**/snapshots/**/*"]
extend-exclude = ["crates/red_knot_python_semantic/vendor/**/*", "**/resources/**/*", "**/snapshots/**/*"]
[default.extend-words]
"arange" = "arange" # e.g. `numpy.arange`

View File

@@ -12,26 +12,29 @@ license.workspace = true
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
red_knot_module_resolver = { workspace = true }
red_knot_python_semantic = { workspace = true }
red_knot_workspace = { workspace = true }
red_knot_server = { workspace = true }
ruff_db = { workspace = true, features = ["os", "cache"] }
anyhow = { workspace = true }
chrono = { workspace = true }
clap = { workspace = true, features = ["wrap_help"] }
colored = { workspace = true }
countme = { workspace = true, features = ["enable"] }
crossbeam = { workspace = true }
ctrlc = { version = "3.4.4" }
rayon = { workspace = true }
salsa = { workspace = true }
tracing = { workspace = true }
tracing-subscriber = { workspace = true }
tracing = { workspace = true, features = ["release_max_level_debug"] }
tracing-subscriber = { workspace = true, features = ["env-filter", "fmt"] }
tracing-flame = { workspace = true }
tracing-tree = { workspace = true }
[dev-dependencies]
filetime = { workspace = true }
tempfile = { workspace = true }
[lints]
workspace = true

Binary file not shown.

After

Width:  |  Height:  |  Size: 40 KiB

View File

@@ -0,0 +1,123 @@
# Tracing
Traces are a useful tool to narrow down the location of a bug or, at least, to understand why the compiler is doing a particular thing.
Note, tracing messages with severity `debug` or greater are user-facing. They should be phrased accordingly.
Tracing spans are only shown when using `-vvv`.
## Verbosity levels
The CLI supports different verbosity levels.
- default: Only show errors and warnings.
- `-v` activates `info!`: Show generally useful information such as paths of configuration files, detected platform, etc., but it's not a lot of messages, it's something you'll activate in CI by default. cargo build e.g. shows you which packages are fresh.
- `-vv` activates `debug!` and timestamps: This should be enough information to get to the bottom of bug reports. When you're processing many packages or files, you'll get pages and pages of output, but each line is link to a specific action or state change.
- `-vvv` activates `trace!` (only in debug builds) and shows tracing-spans: At this level, you're logging everything. Most of this is wasted, it's really slow, we dump e.g. the entire resolution graph. Only useful to developers, and you almost certainly want to use `RED_KNOT_LOG` to filter it down to the area your investigating.
## `RED_KNOT_LOG`
By default, the CLI shows messages from the `ruff` and `red_knot` crates. Tracing messages from other crates are not shown.
The `RED_KNOT_LOG` environment variable allows you to customize which messages are shown by specifying one
or more [filter directives](https://docs.rs/tracing-subscriber/latest/tracing_subscriber/filter/struct.EnvFilter.html#directives).
### Examples
#### Show all debug messages
Shows debug messages from all crates.
```bash
RED_KNOT_LOG=debug
```
#### Show salsa query execution messages
Show the salsa `execute: my_query` messages in addition to all red knot messages.
```bash
RED_KNOT_LOG=ruff=trace,red_knot=trace,salsa=info
```
#### Show typing traces
Only show traces for the `red_knot_python_semantic::types` module.
```bash
RED_KNOT_LOG="red_knot_python_semantic::types"
```
Note: Ensure that you use `-vvv` to see tracing spans.
#### Show messages for a single file
Shows all messages that are inside of a span for a specific file.
```bash
RED_KNOT_LOG=red_knot[{file=/home/micha/astral/test/x.py}]=trace
```
**Note**: Tracing still shows all spans because tracing can't know at the time of entering the span
whether one if its children has the file `x.py`.
**Note**: Salsa currently logs the entire memoized values. In our case, the source text and parsed AST.
This very quickly leads to extremely long outputs.
## Tracing and Salsa
Be mindful about using `tracing` in Salsa queries, especially when using `warn` or `error` because it isn't guaranteed
that the query will execute after restoring from a persistent cache. In which case the user won't see the message.
For example, don't use `tracing` to show the user a message when generating a lint violation failed
because the message would only be shown when linting the file the first time, but not on subsequent analysis
runs or when restoring from a persistent cache. This can be confusing for users because they
don't understand why a specific lint violation isn't raised. Instead, change your
query to return the failure as part of the query's result or use a Salsa accumulator.
## Tracing in tests
You can use `ruff_db::testing::setup_logging` or `ruff_db::testing::setup_logging_with_filter` to set up logging in tests.
```rust
use ruff_db::testing::setup_logging;
#[test]
fn test() {
let _logging = setup_logging();
tracing::info!("This message will be printed to stderr");
}
```
Note: Most test runners capture stderr and only show its output when a test fails.
Note also that `setup_logging` only sets up logging for the current thread because [`set_global_default`](https://docs.rs/tracing/latest/tracing/subscriber/fn.set_global_default.html) can only be
called **once**.
## Release builds
`trace!` events are removed in release builds.
## Profiling
Red Knot generates a folded stack trace to the current directory named `tracing.folded` when setting the environment variable `RED_KNOT_LOG_PROFILE` to `1` or `true`.
```bash
RED_KNOT_LOG_PROFILE=1 red_knot -- --current-directory=../test -vvv
```
You can convert the textual representation into a visual one using `inferno`.
```shell
cargo install inferno
```
```shell
# flamegraph
cat tracing.folded | inferno-flamegraph > tracing-flamegraph.svg
# flamechart
cat tracing.folded | inferno-flamegraph --flamechart > tracing-flamechart.svg
```
![Example flamegraph](./tracing-flamegraph.png)
See [`tracing-flame`](https://crates.io/crates/tracing-flame) for more details.

View File

@@ -1,2 +0,0 @@
pub(crate) mod target_version;
pub(crate) mod verbosity;

View File

@@ -1,34 +0,0 @@
/// Enumeration of all supported Python versions
///
/// TODO: unify with the `PythonVersion` enum in the linter/formatter crates?
#[derive(Copy, Clone, Hash, Debug, PartialEq, Eq, PartialOrd, Ord, Default, clap::ValueEnum)]
pub enum TargetVersion {
Py37,
#[default]
Py38,
Py39,
Py310,
Py311,
Py312,
Py313,
}
impl std::fmt::Display for TargetVersion {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
ruff_db::program::TargetVersion::from(*self).fmt(f)
}
}
impl From<TargetVersion> for ruff_db::program::TargetVersion {
fn from(value: TargetVersion) -> Self {
match value {
TargetVersion::Py37 => Self::Py37,
TargetVersion::Py38 => Self::Py38,
TargetVersion::Py39 => Self::Py39,
TargetVersion::Py310 => Self::Py310,
TargetVersion::Py311 => Self::Py311,
TargetVersion::Py312 => Self::Py312,
TargetVersion::Py313 => Self::Py313,
}
}
}

View File

@@ -1,34 +0,0 @@
#[derive(Debug, Copy, Clone, Eq, PartialEq, Ord, PartialOrd)]
pub(crate) enum VerbosityLevel {
Info,
Debug,
Trace,
}
/// Logging flags to `#[command(flatten)]` into your CLI
#[derive(clap::Args, Debug, Clone, Default)]
#[command(about = None, long_about = None)]
pub(crate) struct Verbosity {
#[arg(
long,
short = 'v',
help = "Use verbose output (or `-vv` and `-vvv` for more verbose output)",
action = clap::ArgAction::Count,
global = true,
)]
verbose: u8,
}
impl Verbosity {
/// Returns the verbosity level based on the number of `-v` flags.
///
/// Returns `None` if the user did not specify any verbosity flags.
pub(crate) fn level(&self) -> Option<VerbosityLevel> {
match self.verbose {
0 => None,
1 => Some(VerbosityLevel::Info),
2 => Some(VerbosityLevel::Debug),
_ => Some(VerbosityLevel::Trace),
}
}
}

View File

@@ -0,0 +1,254 @@
//! Sets up logging for Red Knot
use anyhow::Context;
use colored::Colorize;
use std::fmt;
use std::fs::File;
use std::io::BufWriter;
use tracing::log::LevelFilter;
use tracing::{Event, Subscriber};
use tracing_subscriber::fmt::format::Writer;
use tracing_subscriber::fmt::{FmtContext, FormatEvent, FormatFields};
use tracing_subscriber::registry::LookupSpan;
use tracing_subscriber::EnvFilter;
/// Logging flags to `#[command(flatten)]` into your CLI
#[derive(clap::Args, Debug, Clone, Default)]
#[command(about = None, long_about = None)]
pub(crate) struct Verbosity {
#[arg(
long,
short = 'v',
help = "Use verbose output (or `-vv` and `-vvv` for more verbose output)",
action = clap::ArgAction::Count,
global = true,
)]
verbose: u8,
}
impl Verbosity {
/// Returns the verbosity level based on the number of `-v` flags.
///
/// Returns `None` if the user did not specify any verbosity flags.
pub(crate) fn level(&self) -> VerbosityLevel {
match self.verbose {
0 => VerbosityLevel::Default,
1 => VerbosityLevel::Verbose,
2 => VerbosityLevel::ExtraVerbose,
_ => VerbosityLevel::Trace,
}
}
}
#[derive(Debug, Copy, Clone, Eq, PartialEq, Ord, PartialOrd)]
pub(crate) enum VerbosityLevel {
/// Default output level. Only shows Ruff and Red Knot events up to the [`WARN`](tracing::Level::WARN).
Default,
/// Enables verbose output. Emits Ruff and Red Knot events up to the [`INFO`](tracing::Level::INFO).
/// Corresponds to `-v`.
Verbose,
/// Enables a more verbose tracing format and emits Ruff and Red Knot events up to [`DEBUG`](tracing::Level::DEBUG).
/// Corresponds to `-vv`
ExtraVerbose,
/// Enables all tracing events and uses a tree-like output format. Corresponds to `-vvv`.
Trace,
}
impl VerbosityLevel {
const fn level_filter(self) -> LevelFilter {
match self {
VerbosityLevel::Default => LevelFilter::Warn,
VerbosityLevel::Verbose => LevelFilter::Info,
VerbosityLevel::ExtraVerbose => LevelFilter::Debug,
VerbosityLevel::Trace => LevelFilter::Trace,
}
}
pub(crate) const fn is_trace(self) -> bool {
matches!(self, VerbosityLevel::Trace)
}
pub(crate) const fn is_extra_verbose(self) -> bool {
matches!(self, VerbosityLevel::ExtraVerbose)
}
}
pub(crate) fn setup_tracing(level: VerbosityLevel) -> anyhow::Result<TracingGuard> {
use tracing_subscriber::prelude::*;
// The `RED_KNOT_LOG` environment variable overrides the default log level.
let filter = if let Ok(log_env_variable) = std::env::var("RED_KNOT_LOG") {
EnvFilter::builder()
.parse(log_env_variable)
.context("Failed to parse directives specified in RED_KNOT_LOG environment variable.")?
} else {
match level {
VerbosityLevel::Default => {
// Show warning traces
EnvFilter::default().add_directive(tracing::level_filters::LevelFilter::WARN.into())
}
level => {
let level_filter = level.level_filter();
// Show info|debug|trace events, but allow `RED_KNOT_LOG` to override
let filter = EnvFilter::default().add_directive(
format!("red_knot={level_filter}")
.parse()
.expect("Hardcoded directive to be valid"),
);
filter.add_directive(
format!("ruff={level_filter}")
.parse()
.expect("Hardcoded directive to be valid"),
)
}
}
};
let (profiling_layer, guard) = setup_profile();
let registry = tracing_subscriber::registry()
.with(filter)
.with(profiling_layer);
if level.is_trace() {
let subscriber = registry.with(
tracing_tree::HierarchicalLayer::default()
.with_indent_lines(true)
.with_indent_amount(2)
.with_bracketed_fields(true)
.with_thread_ids(true)
.with_targets(true)
.with_writer(std::io::stderr)
.with_timer(tracing_tree::time::Uptime::default()),
);
subscriber.init();
} else {
let subscriber = registry.with(
tracing_subscriber::fmt::layer()
.event_format(RedKnotFormat {
display_level: true,
display_timestamp: level.is_extra_verbose(),
show_spans: false,
})
.with_writer(std::io::stderr),
);
subscriber.init();
}
Ok(TracingGuard {
_flame_guard: guard,
})
}
#[allow(clippy::type_complexity)]
fn setup_profile<S>() -> (
Option<tracing_flame::FlameLayer<S, BufWriter<File>>>,
Option<tracing_flame::FlushGuard<BufWriter<File>>>,
)
where
S: Subscriber + for<'span> LookupSpan<'span>,
{
if let Ok("1" | "true") = std::env::var("RED_KNOT_LOG_PROFILE").as_deref() {
let (layer, guard) = tracing_flame::FlameLayer::with_file("tracing.folded")
.expect("Flame layer to be created");
(Some(layer), Some(guard))
} else {
(None, None)
}
}
pub(crate) struct TracingGuard {
_flame_guard: Option<tracing_flame::FlushGuard<BufWriter<File>>>,
}
struct RedKnotFormat {
display_timestamp: bool,
display_level: bool,
show_spans: bool,
}
/// See <https://docs.rs/tracing-subscriber/0.3.18/src/tracing_subscriber/fmt/format/mod.rs.html#1026-1156>
impl<S, N> FormatEvent<S, N> for RedKnotFormat
where
S: Subscriber + for<'a> LookupSpan<'a>,
N: for<'a> FormatFields<'a> + 'static,
{
fn format_event(
&self,
ctx: &FmtContext<'_, S, N>,
mut writer: Writer<'_>,
event: &Event<'_>,
) -> fmt::Result {
let meta = event.metadata();
let ansi = writer.has_ansi_escapes();
if self.display_timestamp {
let timestamp = chrono::Local::now()
.format("%Y-%m-%d %H:%M:%S.%f")
.to_string();
if ansi {
write!(writer, "{} ", timestamp.dimmed())?;
} else {
write!(
writer,
"{} ",
chrono::Local::now().format("%Y-%m-%d %H:%M:%S.%f")
)?;
}
}
if self.display_level {
let level = meta.level();
// Same colors as tracing
if ansi {
let formatted_level = level.to_string();
match *level {
tracing::Level::TRACE => {
write!(writer, "{} ", formatted_level.purple().bold())?;
}
tracing::Level::DEBUG => write!(writer, "{} ", formatted_level.blue().bold())?,
tracing::Level::INFO => write!(writer, "{} ", formatted_level.green().bold())?,
tracing::Level::WARN => write!(writer, "{} ", formatted_level.yellow().bold())?,
tracing::Level::ERROR => write!(writer, "{} ", level.to_string().red().bold())?,
}
} else {
write!(writer, "{level} ")?;
}
}
if self.show_spans {
let span = event.parent();
let mut seen = false;
let span = span
.and_then(|id| ctx.span(id))
.or_else(|| ctx.lookup_current());
let scope = span.into_iter().flat_map(|span| span.scope().from_root());
for span in scope {
seen = true;
if ansi {
write!(writer, "{}:", span.metadata().name().bold())?;
} else {
write!(writer, "{}:", span.metadata().name())?;
}
}
if seen {
writer.write_char(' ')?;
}
}
ctx.field_format().format_fields(writer.by_ref(), event)?;
writeln!(writer)
}
}

View File

@@ -1,34 +1,39 @@
use std::process::{ExitCode, Termination};
use std::sync::Mutex;
use anyhow::{anyhow, Context};
use clap::Parser;
use colored::Colorize;
use crossbeam::channel as crossbeam_channel;
use tracing::subscriber::Interest;
use tracing::{Level, Metadata};
use tracing_subscriber::filter::LevelFilter;
use tracing_subscriber::layer::{Context, Filter, SubscriberExt};
use tracing_subscriber::{Layer, Registry};
use tracing_tree::time::Uptime;
use salsa::plumbing::ZalsaDatabase;
use red_knot_python_semantic::{ProgramSettings, SearchPathSettings};
use red_knot_server::run_server;
use red_knot_workspace::db::RootDatabase;
use red_knot_workspace::site_packages::VirtualEnvironment;
use red_knot_workspace::watch;
use red_knot_workspace::watch::WorkspaceWatcher;
use red_knot_workspace::workspace::WorkspaceMetadata;
use ruff_db::program::{ProgramSettings, SearchPathSettings};
use ruff_db::system::{OsSystem, System, SystemPathBuf};
use ruff_db::system::{OsSystem, System, SystemPath, SystemPathBuf};
use target_version::TargetVersion;
use cli::target_version::TargetVersion;
use cli::verbosity::{Verbosity, VerbosityLevel};
use crate::logging::{setup_tracing, Verbosity};
mod cli;
mod logging;
mod target_version;
mod verbosity;
#[derive(Debug, Parser)]
#[command(
author,
name = "red-knot",
about = "An experimental multifile analysis backend for Ruff"
about = "An extremely fast Python type checker."
)]
#[command(version)]
struct Args {
#[command(subcommand)]
pub(crate) command: Option<Command>,
#[arg(
long,
help = "Changes the current working directory.",
@@ -37,6 +42,17 @@ struct Args {
)]
current_directory: Option<SystemPathBuf>,
#[arg(
long,
help = "Path to the virtual environment the project uses",
long_help = "\
Path to the virtual environment the project uses. \
If provided, red-knot will use the `site-packages` directory of this virtual environment \
to resolve type information for the project's third-party dependencies.",
value_name = "PATH"
)]
venv_path: Option<SystemPathBuf>,
#[arg(
long,
value_name = "DIRECTORY",
@@ -51,7 +67,12 @@ struct Args {
)]
extra_search_path: Vec<SystemPathBuf>,
#[arg(long, help = "Python version to assume when resolving types", default_value_t = TargetVersion::default(), value_name="VERSION")]
#[arg(
long,
help = "Python version to assume when resolving types",
default_value_t = TargetVersion::default(),
value_name="VERSION")
]
target_version: TargetVersion,
#[clap(flatten)]
@@ -65,54 +86,107 @@ struct Args {
watch: bool,
}
#[allow(
clippy::print_stdout,
clippy::unnecessary_wraps,
clippy::print_stderr,
clippy::dbg_macro
)]
pub fn main() -> anyhow::Result<()> {
#[derive(Debug, clap::Subcommand)]
pub enum Command {
/// Start the language server
Server,
}
#[allow(clippy::print_stdout, clippy::unnecessary_wraps, clippy::print_stderr)]
pub fn main() -> ExitStatus {
run().unwrap_or_else(|error| {
use std::io::Write;
// Use `writeln` instead of `eprintln` to avoid panicking when the stderr pipe is broken.
let mut stderr = std::io::stderr().lock();
// This communicates that this isn't a linter error but Red Knot itself hard-errored for
// some reason (e.g. failed to resolve the configuration)
writeln!(stderr, "{}", "Red Knot failed".red().bold()).ok();
// Currently we generally only see one error, but e.g. with io errors when resolving
// the configuration it is help to chain errors ("resolving configuration failed" ->
// "failed to read file: subdir/pyproject.toml")
for cause in error.chain() {
writeln!(stderr, " {} {cause}", "Cause:".bold()).ok();
}
ExitStatus::Error
})
}
fn run() -> anyhow::Result<ExitStatus> {
let Args {
command,
current_directory,
custom_typeshed_dir,
extra_search_path: extra_paths,
venv_path,
target_version,
verbosity,
watch,
} = Args::parse_from(std::env::args().collect::<Vec<_>>());
let verbosity = verbosity.level();
countme::enable(verbosity == Some(VerbosityLevel::Trace));
setup_tracing(verbosity);
if matches!(command, Some(Command::Server)) {
return run_server().map(|()| ExitStatus::Success);
}
let cwd = if let Some(cwd) = current_directory {
let canonicalized = cwd.as_utf8_path().canonicalize_utf8().unwrap();
SystemPathBuf::from_utf8_path_buf(canonicalized)
} else {
let cwd = std::env::current_dir().unwrap();
SystemPathBuf::from_path_buf(cwd).unwrap()
let verbosity = verbosity.level();
countme::enable(verbosity.is_trace());
let _guard = setup_tracing(verbosity)?;
// The base path to which all CLI arguments are relative to.
let cli_base_path = {
let cwd = std::env::current_dir().context("Failed to get the current working directory")?;
SystemPathBuf::from_path_buf(cwd)
.map_err(|path| {
anyhow!(
"The current working directory '{}' contains non-unicode characters. Red Knot only supports unicode paths.",
path.display()
)
})?
};
let cwd = current_directory
.map(|cwd| {
if cwd.as_std_path().is_dir() {
Ok(SystemPath::absolute(&cwd, &cli_base_path))
} else {
Err(anyhow!(
"Provided current-directory path '{cwd}' is not a directory."
))
}
})
.transpose()?
.unwrap_or_else(|| cli_base_path.clone());
let system = OsSystem::new(cwd.clone());
let workspace_metadata =
WorkspaceMetadata::from_path(system.current_directory(), &system).unwrap();
let workspace_metadata = WorkspaceMetadata::from_path(system.current_directory(), &system)?;
// TODO: Verify the remaining search path settings eagerly.
let site_packages = venv_path
.map(|path| {
VirtualEnvironment::new(path, &OsSystem::new(cli_base_path))
.and_then(|venv| venv.site_packages_directories(&system))
})
.transpose()?
.unwrap_or_default();
// TODO: Respect the settings from the workspace metadata. when resolving the program settings.
let program_settings = ProgramSettings {
target_version: target_version.into(),
search_paths: SearchPathSettings {
extra_paths,
workspace_root: workspace_metadata.root().to_path_buf(),
src_root: workspace_metadata.root().to_path_buf(),
custom_typeshed: custom_typeshed_dir,
site_packages: vec![],
site_packages,
},
};
// TODO: Use the `program_settings` to compute the key for the database's persistent
// cache and load the cache if it exists.
let db = RootDatabase::new(workspace_metadata, program_settings, system);
let mut db = RootDatabase::new(workspace_metadata, program_settings, system);
let (main_loop, main_loop_cancellation_token) = MainLoop::new(verbosity);
let (main_loop, main_loop_cancellation_token) = MainLoop::new();
// Listen to Ctrl+C and abort the watch mode.
let main_loop_cancellation_token = Mutex::new(Some(main_loop_cancellation_token));
@@ -124,16 +198,35 @@ pub fn main() -> anyhow::Result<()> {
}
})?;
let mut db = salsa::Handle::new(db);
if watch {
main_loop.watch(&mut db)?;
let exit_status = if watch {
main_loop.watch(&mut db)?
} else {
main_loop.run(&mut db);
main_loop.run(&mut db)
};
tracing::trace!("Counts for entire CLI run:\n{}", countme::get_all());
std::mem::forget(db);
Ok(())
Ok(exit_status)
}
#[derive(Copy, Clone)]
pub enum ExitStatus {
/// Checking was successful and there were no errors.
Success = 0,
/// Checking was successful but there were errors.
Failure = 1,
/// Checking failed.
Error = 2,
}
impl Termination for ExitStatus {
fn report(self) -> ExitCode {
ExitCode::from(self as u8)
}
}
struct MainLoop {
@@ -145,12 +238,10 @@ struct MainLoop {
/// The file system watcher, if running in watch mode.
watcher: Option<WorkspaceWatcher>,
verbosity: Option<VerbosityLevel>,
}
impl MainLoop {
fn new(verbosity: Option<VerbosityLevel>) -> (Self, MainLoopCancellationToken) {
fn new() -> (Self, MainLoopCancellationToken) {
let (sender, receiver) = crossbeam_channel::bounded(10);
(
@@ -158,35 +249,45 @@ impl MainLoop {
sender: sender.clone(),
receiver,
watcher: None,
verbosity,
},
MainLoopCancellationToken { sender },
)
}
fn watch(mut self, db: &mut salsa::Handle<RootDatabase>) -> anyhow::Result<()> {
fn watch(mut self, db: &mut RootDatabase) -> anyhow::Result<ExitStatus> {
tracing::debug!("Starting watch mode");
let sender = self.sender.clone();
let watcher = watch::directory_watcher(move |event| {
sender.send(MainLoopMessage::ApplyChanges(event)).unwrap();
})?;
self.watcher = Some(WorkspaceWatcher::new(watcher, db));
self.run(db);
Ok(())
Ok(ExitStatus::Success)
}
#[allow(clippy::print_stderr)]
fn run(mut self, db: &mut salsa::Handle<RootDatabase>) {
// Schedule the first check.
fn run(mut self, db: &mut RootDatabase) -> ExitStatus {
self.sender.send(MainLoopMessage::CheckWorkspace).unwrap();
let mut revision = 0usize;
let result = self.main_loop(db);
tracing::debug!("Exiting main loop");
result
}
fn main_loop(&mut self, db: &mut RootDatabase) -> ExitStatus {
// Schedule the first check.
tracing::debug!("Starting main loop");
let mut revision = 0u64;
while let Ok(message) = self.receiver.recv() {
tracing::trace!("Main Loop: Tick");
match message {
MainLoopMessage::CheckWorkspace => {
let db = db.clone();
let db = db.snapshot();
let sender = self.sender.clone();
// Spawn a new task that checks the workspace. This needs to be done in a separate thread
@@ -196,7 +297,7 @@ impl MainLoop {
// Send the result back to the main loop for printing.
sender
.send(MainLoopMessage::CheckCompleted { result, revision })
.ok();
.unwrap();
}
});
}
@@ -205,43 +306,50 @@ impl MainLoop {
result,
revision: check_revision,
} => {
let has_diagnostics = !result.is_empty();
if check_revision == revision {
eprintln!("{}", result.join("\n"));
if self.verbosity == Some(VerbosityLevel::Trace) {
eprintln!("{}", countme::get_all());
for diagnostic in result {
tracing::error!("{}", diagnostic);
}
} else {
tracing::debug!(
"Discarding check result for outdated revision: current: {revision}, result revision: {check_revision}"
);
}
if self.watcher.is_none() {
return self.exit();
return if has_diagnostics {
ExitStatus::Failure
} else {
ExitStatus::Success
};
}
tracing::trace!("Counts after last check:\n{}", countme::get_all());
}
MainLoopMessage::ApplyChanges(changes) => {
revision += 1;
// Automatically cancels any pending queries and waits for them to complete.
db.get_mut().apply_changes(changes);
db.apply_changes(changes);
if let Some(watcher) = self.watcher.as_mut() {
watcher.update(db);
}
self.sender.send(MainLoopMessage::CheckWorkspace).unwrap();
}
MainLoopMessage::Exit => {
return self.exit();
// Cancel any pending queries and wait for them to complete.
// TODO: Don't use Salsa internal APIs
// [Zulip-Thread](https://salsa.zulipchat.com/#narrow/stream/333573-salsa-3.2E0/topic/Expose.20an.20API.20to.20cancel.20other.20queries)
let _ = db.zalsa_mut();
return ExitStatus::Success;
}
}
tracing::debug!("Waiting for next main loop message.");
}
self.exit();
}
#[allow(clippy::print_stderr, clippy::unused_self)]
fn exit(self) {
if self.verbosity == Some(VerbosityLevel::Trace) {
eprintln!("Exit");
eprintln!("{}", countme::get_all());
}
ExitStatus::Success
}
}
@@ -260,70 +368,7 @@ impl MainLoopCancellationToken {
#[derive(Debug)]
enum MainLoopMessage {
CheckWorkspace,
CheckCompleted {
result: Vec<String>,
revision: usize,
},
CheckCompleted { result: Vec<String>, revision: u64 },
ApplyChanges(Vec<watch::ChangeEvent>),
Exit,
}
fn setup_tracing(verbosity: Option<VerbosityLevel>) {
let trace_level = match verbosity {
None => Level::WARN,
Some(VerbosityLevel::Info) => Level::INFO,
Some(VerbosityLevel::Debug) => Level::DEBUG,
Some(VerbosityLevel::Trace) => Level::TRACE,
};
let subscriber = Registry::default().with(
tracing_tree::HierarchicalLayer::default()
.with_indent_lines(true)
.with_indent_amount(2)
.with_bracketed_fields(true)
.with_thread_ids(true)
.with_targets(true)
.with_writer(|| Box::new(std::io::stderr()))
.with_timer(Uptime::default())
.with_filter(LoggingFilter { trace_level }),
);
tracing::subscriber::set_global_default(subscriber).unwrap();
}
struct LoggingFilter {
trace_level: Level,
}
impl LoggingFilter {
fn is_enabled(&self, meta: &Metadata<'_>) -> bool {
let filter = if meta.target().starts_with("red_knot") || meta.target().starts_with("ruff") {
self.trace_level
} else if meta.target().starts_with("salsa") && self.trace_level <= Level::INFO {
// Salsa emits very verbose query traces with level info. Let's not show these to the user.
Level::WARN
} else {
Level::INFO
};
meta.level() <= &filter
}
}
impl<S> Filter<S> for LoggingFilter {
fn enabled(&self, meta: &Metadata<'_>, _cx: &Context<'_, S>) -> bool {
self.is_enabled(meta)
}
fn callsite_enabled(&self, meta: &'static Metadata<'static>) -> Interest {
if self.is_enabled(meta) {
Interest::always()
} else {
Interest::never()
}
}
fn max_level_hint(&self) -> Option<LevelFilter> {
Some(LevelFilter::from_level(self.trace_level))
}
}

View File

@@ -0,0 +1,48 @@
/// Enumeration of all supported Python versions
///
/// TODO: unify with the `PythonVersion` enum in the linter/formatter crates?
#[derive(Copy, Clone, Hash, Debug, PartialEq, Eq, PartialOrd, Ord, Default, clap::ValueEnum)]
pub enum TargetVersion {
Py37,
#[default]
Py38,
Py39,
Py310,
Py311,
Py312,
Py313,
}
impl TargetVersion {
const fn as_str(self) -> &'static str {
match self {
Self::Py37 => "py37",
Self::Py38 => "py38",
Self::Py39 => "py39",
Self::Py310 => "py310",
Self::Py311 => "py311",
Self::Py312 => "py312",
Self::Py313 => "py313",
}
}
}
impl std::fmt::Display for TargetVersion {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str(self.as_str())
}
}
impl From<TargetVersion> for red_knot_python_semantic::PythonVersion {
fn from(value: TargetVersion) -> Self {
match value {
TargetVersion::Py37 => Self::PY37,
TargetVersion::Py38 => Self::PY38,
TargetVersion::Py39 => Self::PY39,
TargetVersion::Py310 => Self::PY310,
TargetVersion::Py311 => Self::PY311,
TargetVersion::Py312 => Self::PY312,
TargetVersion::Py313 => Self::PY313,
}
}
}

View File

@@ -0,0 +1 @@

View File

@@ -6,13 +6,14 @@ use std::time::Duration;
use anyhow::{anyhow, Context};
use salsa::Setter;
use red_knot_module_resolver::{resolve_module, ModuleName};
use red_knot_python_semantic::{
resolve_module, ModuleName, Program, ProgramSettings, PythonVersion, SearchPathSettings,
};
use red_knot_workspace::db::RootDatabase;
use red_knot_workspace::watch;
use red_knot_workspace::watch::{directory_watcher, WorkspaceWatcher};
use red_knot_workspace::workspace::WorkspaceMetadata;
use ruff_db::files::{system_path_to_file, File, FileError};
use ruff_db::program::{Program, ProgramSettings, SearchPathSettings, TargetVersion};
use ruff_db::source::source_text;
use ruff_db::system::{OsSystem, SystemPath, SystemPathBuf};
use ruff_db::Upcast;
@@ -179,7 +180,7 @@ where
{
setup_with_search_paths(setup_files, |_root, workspace_path| SearchPathSettings {
extra_paths: vec![],
workspace_root: workspace_path.to_path_buf(),
src_root: workspace_path.to_path_buf(),
custom_typeshed: None,
site_packages: vec![],
})
@@ -233,7 +234,7 @@ where
}
let settings = ProgramSettings {
target_version: TargetVersion::default(),
target_version: PythonVersion::default(),
search_paths,
};
@@ -695,7 +696,7 @@ fn search_path() -> anyhow::Result<()> {
setup_with_search_paths([("bar.py", "import sub.a")], |root_path, workspace_path| {
SearchPathSettings {
extra_paths: vec![],
workspace_root: workspace_path.to_path_buf(),
src_root: workspace_path.to_path_buf(),
custom_typeshed: None,
site_packages: vec![root_path.join("site_packages")],
}
@@ -755,7 +756,7 @@ fn remove_search_path() -> anyhow::Result<()> {
setup_with_search_paths([("bar.py", "import sub.a")], |root_path, workspace_path| {
SearchPathSettings {
extra_paths: vec![],
workspace_root: workspace_path.to_path_buf(),
src_root: workspace_path.to_path_buf(),
custom_typeshed: None,
site_packages: vec![root_path.join("site_packages")],
}
@@ -1173,7 +1174,7 @@ mod unix {
},
|_root, workspace| SearchPathSettings {
extra_paths: vec![],
workspace_root: workspace.to_path_buf(),
src_root: workspace.to_path_buf(),
custom_typeshed: None,
site_packages: vec![workspace.join(".venv/lib/python3.12/site-packages")],
},
@@ -1184,8 +1185,9 @@ mod unix {
ModuleName::new_static("bar.baz").unwrap(),
)
.expect("Expected bar.baz to exist in site-packages.");
let baz_site_packages =
let baz_site_packages_path =
case.workspace_path(".venv/lib/python3.12/site-packages/bar/baz.py");
let baz_site_packages = case.system_file(&baz_site_packages_path).unwrap();
let baz_original = case.root_path().join("site-packages/bar/baz.py");
let baz_original_file = case.system_file(&baz_original).unwrap();
@@ -1195,12 +1197,12 @@ mod unix {
);
assert_eq!(
source_text(case.db(), baz.file()).as_str(),
source_text(case.db(), baz_site_packages).as_str(),
"def baz(): ..."
);
assert_eq!(
baz.file().path(case.db()).as_system_path(),
Some(&*baz_site_packages)
Some(&*baz_original)
);
// Write to the symlink target.
@@ -1212,7 +1214,7 @@ mod unix {
case.db_mut().apply_changes(changes);
assert_eq!(
source_text(case.db(), baz.file()).as_str(),
source_text(case.db(), baz_original_file).as_str(),
"def baz(): print('Version 2')"
);
@@ -1224,7 +1226,7 @@ mod unix {
// it doesn't seem worth doing considering that as prominent tools like PyCharm don't support it.
// Pyright does support it, thanks to chokidar.
assert_ne!(
source_text(case.db(), baz_original_file).as_str(),
source_text(case.db(), baz_site_packages).as_str(),
"def baz(): print('Version 2')"
);

View File

@@ -1,39 +0,0 @@
[package]
name = "red_knot_module_resolver"
version = "0.0.0"
publish = false
authors = { workspace = true }
edition = { workspace = true }
rust-version = { workspace = true }
homepage = { workspace = true }
documentation = { workspace = true }
repository = { workspace = true }
license = { workspace = true }
[dependencies]
ruff_db = { workspace = true }
ruff_python_stdlib = { workspace = true }
compact_str = { workspace = true }
camino = { workspace = true }
once_cell = { workspace = true }
rustc-hash = { workspace = true }
salsa = { workspace = true }
tracing = { workspace = true }
zip = { workspace = true }
[build-dependencies]
path-slash = { workspace = true }
walkdir = { workspace = true }
zip = { workspace = true }
[dev-dependencies]
ruff_db = { workspace = true, features = ["os"] }
anyhow = { workspace = true }
insta = { workspace = true }
tempfile = { workspace = true }
walkdir = { workspace = true }
[lints]
workspace = true

View File

@@ -1,105 +0,0 @@
use ruff_db::Upcast;
#[salsa::db]
pub trait Db: ruff_db::Db + Upcast<dyn ruff_db::Db> {}
#[cfg(test)]
pub(crate) mod tests {
use std::sync;
use ruff_db::files::Files;
use ruff_db::system::{DbWithTestSystem, TestSystem};
use ruff_db::vendored::VendoredFileSystem;
use crate::vendored_typeshed_stubs;
use super::*;
#[salsa::db]
pub(crate) struct TestDb {
storage: salsa::Storage<Self>,
system: TestSystem,
vendored: VendoredFileSystem,
files: Files,
events: sync::Arc<sync::Mutex<Vec<salsa::Event>>>,
}
impl TestDb {
pub(crate) fn new() -> Self {
Self {
storage: salsa::Storage::default(),
system: TestSystem::default(),
vendored: vendored_typeshed_stubs().clone(),
events: sync::Arc::default(),
files: Files::default(),
}
}
/// Takes the salsa events.
///
/// ## Panics
/// If there are any pending salsa snapshots.
pub(crate) fn take_salsa_events(&mut self) -> Vec<salsa::Event> {
let inner = sync::Arc::get_mut(&mut self.events).expect("no pending salsa snapshots");
let events = inner.get_mut().unwrap();
std::mem::take(&mut *events)
}
/// Clears the salsa events.
///
/// ## Panics
/// If there are any pending salsa snapshots.
pub(crate) fn clear_salsa_events(&mut self) {
self.take_salsa_events();
}
}
impl Upcast<dyn ruff_db::Db> for TestDb {
fn upcast(&self) -> &(dyn ruff_db::Db + 'static) {
self
}
fn upcast_mut(&mut self) -> &mut (dyn ruff_db::Db + 'static) {
self
}
}
#[salsa::db]
impl ruff_db::Db for TestDb {
fn vendored(&self) -> &VendoredFileSystem {
&self.vendored
}
fn system(&self) -> &dyn ruff_db::system::System {
&self.system
}
fn files(&self) -> &Files {
&self.files
}
}
#[salsa::db]
impl Db for TestDb {}
impl DbWithTestSystem for TestDb {
fn test_system(&self) -> &TestSystem {
&self.system
}
fn test_system_mut(&mut self) -> &mut TestSystem {
&mut self.system
}
}
#[salsa::db]
impl salsa::Database for TestDb {
fn salsa_event(&self, event: salsa::Event) {
self.attach(|_| {
tracing::trace!("event: {event:?}");
let mut events = self.events.lock().unwrap();
events.push(event);
});
}
}
}

View File

@@ -11,24 +11,38 @@ repository = { workspace = true }
license = { workspace = true }
[dependencies]
red_knot_module_resolver = { workspace = true }
ruff_db = { workspace = true }
ruff_index = { workspace = true }
ruff_python_ast = { workspace = true }
ruff_python_stdlib = { workspace = true }
ruff_text_size = { workspace = true }
bitflags = { workspace = true }
camino = { workspace = true }
compact_str = { workspace = true }
countme = { workspace = true }
once_cell = { workspace = true }
ordermap = { workspace = true }
salsa = { workspace = true }
tracing = { workspace = true }
rustc-hash = { workspace = true }
hashbrown = { workspace = true }
[build-dependencies]
path-slash = { workspace = true }
walkdir = { workspace = true }
zip = { workspace = true, features = ["zstd", "deflate"] }
[dev-dependencies]
anyhow = { workspace = true }
ruff_db = { workspace = true, features = ["os", "testing"]}
ruff_python_parser = { workspace = true }
anyhow = { workspace = true }
insta = { workspace = true }
tempfile = { workspace = true }
walkdir = { workspace = true }
zip = { workspace = true }
[lints]
workspace = true

View File

@@ -1,9 +1,9 @@
# Red Knot
A work-in-progress multifile module resolver for Ruff.
Semantic analysis for the red-knot project.
## Vendored types for the stdlib
This crate vendors [typeshed](https://github.com/python/typeshed)'s stubs for the standard library. The vendored stubs can be found in `crates/red_knot_module_resolver/vendor/typeshed`. The file `crates/red_knot_module_resolver/vendor/typeshed/source_commit.txt` tells you the typeshed commit that our vendored stdlib stubs currently correspond to.
This crate vendors [typeshed](https://github.com/python/typeshed)'s stubs for the standard library. The vendored stubs can be found in `crates/red_knot_python_semantic/vendor/typeshed`. The file `crates/red_knot_python_semantic/vendor/typeshed/source_commit.txt` tells you the typeshed commit that our vendored stdlib stubs currently correspond to.
The typeshed stubs are updated every two weeks via an automated PR using the `sync_typeshed.yaml` workflow in the `.github/workflows` directory. This workflow can also be triggered at any time via [workflow dispatch](https://docs.github.com/en/actions/using-workflows/manually-running-a-workflow#running-a-workflow).

View File

@@ -3,7 +3,7 @@
//!
//! This script should be automatically run at build time
//! whenever the script itself changes, or whenever any files
//! in `crates/red_knot_module_resolver/vendor/typeshed` change.
//! in `crates/red_knot_python_semantic/vendor/typeshed` change.
use std::fs::File;
use std::path::Path;
@@ -23,8 +23,21 @@ const TYPESHED_ZIP_LOCATION: &str = "/zipped_typeshed.zip";
fn zip_dir(directory_path: &str, writer: File) -> ZipResult<File> {
let mut zip = ZipWriter::new(writer);
// Use deflated compression for WASM builds because compiling `zstd-sys` requires clang
// [source](https://github.com/gyscos/zstd-rs/wiki/Compile-for-WASM) which complicates the build
// by a lot. Deflated compression is slower but it shouldn't matter much for the WASM use case
// (WASM itself is already slower than a native build for a specific platform).
// We can't use `#[cfg(...)]` here because the target-arch in a build script is the
// architecture of the system running the build script and not the architecture of the build-target.
// That's why we use the `TARGET` environment variable here.
let method = if std::env::var("TARGET").unwrap().contains("wasm32") {
CompressionMethod::Deflated
} else {
CompressionMethod::Zstd
};
let options = FileOptions::default()
.compression_method(CompressionMethod::Zstd)
.compression_method(method)
.unix_permissions(0o644);
for entry in walkdir::WalkDir::new(directory_path) {

View File

@@ -1,5 +1,5 @@
use red_knot_module_resolver::{resolve_module, ModuleName};
use crate::module_name::ModuleName;
use crate::module_resolver::resolve_module;
use crate::semantic_index::global_scope;
use crate::semantic_index::symbol::ScopeId;
use crate::Db;
@@ -11,6 +11,6 @@ use crate::Db;
pub(crate) fn builtins_scope(db: &dyn Db) -> Option<ScopeId<'_>> {
let builtins_name =
ModuleName::new_static("builtins").expect("Expected 'builtins' to be a valid module name");
let builtins_file = resolve_module(db.upcast(), builtins_name)?.file();
let builtins_file = resolve_module(db, builtins_name)?.file();
Some(global_scope(db, builtins_file))
}

View File

@@ -1,15 +1,14 @@
use red_knot_module_resolver::Db as ResolverDb;
use ruff_db::Upcast;
use ruff_db::{Db as SourceDb, Upcast};
/// Database giving access to semantic information about a Python program.
#[salsa::db]
pub trait Db: ResolverDb + Upcast<dyn ResolverDb> {}
pub trait Db: SourceDb + Upcast<dyn SourceDb> {}
#[cfg(test)]
pub(crate) mod tests {
use std::sync::Arc;
use red_knot_module_resolver::{vendored_typeshed_stubs, Db as ResolverDb};
use crate::module_resolver::vendored_typeshed_stubs;
use ruff_db::files::Files;
use ruff_db::system::{DbWithTestSystem, System, TestSystem};
use ruff_db::vendored::VendoredFileSystem;
@@ -91,29 +90,16 @@ pub(crate) mod tests {
}
}
impl Upcast<dyn ResolverDb> for TestDb {
fn upcast(&self) -> &(dyn ResolverDb + 'static) {
self
}
fn upcast_mut(&mut self) -> &mut (dyn ResolverDb + 'static) {
self
}
}
#[salsa::db]
impl red_knot_module_resolver::Db for TestDb {}
#[salsa::db]
impl Db for TestDb {}
#[salsa::db]
impl salsa::Database for TestDb {
fn salsa_event(&self, event: salsa::Event) {
self.attach(|_| {
tracing::trace!("event: {event:?}");
let mut events = self.events.lock().unwrap();
events.push(event);
});
fn salsa_event(&self, event: &dyn Fn() -> salsa::Event) {
let event = event();
tracing::trace!("event: {event:?}");
let mut events = self.events.lock().unwrap();
events.push(event);
}
}
}

View File

@@ -3,12 +3,20 @@ use std::hash::BuildHasherDefault;
use rustc_hash::FxHasher;
pub use db::Db;
pub use module_name::ModuleName;
pub use module_resolver::{resolve_module, system_module_search_paths, vendored_typeshed_stubs};
pub use program::{Program, ProgramSettings, SearchPathSettings};
pub use python_version::PythonVersion;
pub use semantic_model::{HasTy, SemanticModel};
pub mod ast_node_ref;
mod builtins;
mod db;
mod module_name;
mod module_resolver;
mod node_key;
mod program;
mod python_version;
pub mod semantic_index;
mod semantic_model;
pub mod types;

View File

@@ -42,7 +42,7 @@ impl ModuleName {
/// ## Examples
///
/// ```
/// use red_knot_module_resolver::ModuleName;
/// use red_knot_python_semantic::ModuleName;
///
/// assert_eq!(ModuleName::new_static("foo.bar").as_deref(), Some("foo.bar"));
/// assert_eq!(ModuleName::new_static(""), None);
@@ -68,7 +68,7 @@ impl ModuleName {
/// # Examples
///
/// ```
/// use red_knot_module_resolver::ModuleName;
/// use red_knot_python_semantic::ModuleName;
///
/// assert_eq!(ModuleName::new_static("foo.bar.baz").unwrap().components().collect::<Vec<_>>(), vec!["foo", "bar", "baz"]);
/// ```
@@ -82,7 +82,7 @@ impl ModuleName {
/// # Examples
///
/// ```
/// use red_knot_module_resolver::ModuleName;
/// use red_knot_python_semantic::ModuleName;
///
/// assert_eq!(ModuleName::new_static("foo.bar").unwrap().parent(), Some(ModuleName::new_static("foo").unwrap()));
/// assert_eq!(ModuleName::new_static("foo.bar.baz").unwrap().parent(), Some(ModuleName::new_static("foo.bar").unwrap()));
@@ -101,7 +101,7 @@ impl ModuleName {
/// # Examples
///
/// ```
/// use red_knot_module_resolver::ModuleName;
/// use red_knot_python_semantic::ModuleName;
///
/// assert!(ModuleName::new_static("foo.bar").unwrap().starts_with(&ModuleName::new_static("foo").unwrap()));
///
@@ -133,7 +133,7 @@ impl ModuleName {
/// # Examples
///
/// ```
/// use red_knot_module_resolver::ModuleName;
/// use red_knot_python_semantic::ModuleName;
///
/// assert_eq!(&*ModuleName::from_components(["a"]).unwrap(), "a");
/// assert_eq!(&*ModuleName::from_components(["a", "b"]).unwrap(), "a.b");

View File

@@ -1,19 +1,14 @@
use std::iter::FusedIterator;
pub use db::Db;
pub use module::{Module, ModuleKind};
pub use module_name::ModuleName;
pub(crate) use module::Module;
pub use resolver::resolve_module;
use ruff_db::system::SystemPath;
pub use typeshed::{
vendored_typeshed_stubs, TypeshedVersionsParseError, TypeshedVersionsParseErrorKind,
};
pub use typeshed::vendored_typeshed_stubs;
use crate::resolver::{module_resolution_settings, SearchPathIterator};
use crate::Db;
use resolver::{module_resolution_settings, SearchPathIterator};
mod db;
mod module;
mod module_name;
mod path;
mod resolver;
mod state;

View File

@@ -3,8 +3,8 @@ use std::sync::Arc;
use ruff_db::files::File;
use super::path::SearchPath;
use crate::module_name::ModuleName;
use crate::path::SearchPath;
/// Representation of a Python module.
#[derive(Clone, PartialEq, Eq)]

View File

@@ -11,8 +11,9 @@ use ruff_db::vendored::{VendoredPath, VendoredPathBuf};
use crate::db::Db;
use crate::module_name::ModuleName;
use crate::state::ResolverState;
use crate::typeshed::{TypeshedVersionsParseError, TypeshedVersionsQueryResult};
use super::state::ResolverState;
use super::typeshed::{TypeshedVersionsParseError, TypeshedVersionsQueryResult};
/// A path that points to a Python module.
///
@@ -382,22 +383,27 @@ enum SearchPathInner {
pub(crate) struct SearchPath(Arc<SearchPathInner>);
impl SearchPath {
fn directory_path(system: &dyn System, root: SystemPathBuf) -> SearchPathResult<SystemPathBuf> {
let canonicalized = system.canonicalize_path(&root).unwrap_or(root);
if system.is_directory(&canonicalized) {
Ok(canonicalized)
} else {
Err(SearchPathValidationError::NotADirectory(canonicalized))
}
}
/// Create a new "Extra" search path
pub(crate) fn extra(system: &dyn System, root: SystemPathBuf) -> SearchPathResult<Self> {
if system.is_directory(&root) {
Ok(Self(Arc::new(SearchPathInner::Extra(root))))
} else {
Err(SearchPathValidationError::NotADirectory(root))
}
Ok(Self(Arc::new(SearchPathInner::Extra(
Self::directory_path(system, root)?,
))))
}
/// Create a new first-party search path, pointing to the user code we were directly invoked on
pub(crate) fn first_party(system: &dyn System, root: SystemPathBuf) -> SearchPathResult<Self> {
if system.is_directory(&root) {
Ok(Self(Arc::new(SearchPathInner::FirstParty(root))))
} else {
Err(SearchPathValidationError::NotADirectory(root))
}
Ok(Self(Arc::new(SearchPathInner::FirstParty(
Self::directory_path(system, root)?,
))))
}
/// Create a new standard-library search path pointing to a custom directory on disk
@@ -408,12 +414,13 @@ impl SearchPath {
typeshed.to_path_buf(),
));
}
let stdlib = typeshed.join("stdlib");
if !system.is_directory(&stdlib) {
return Err(SearchPathValidationError::NoStdlibSubdirectory(
typeshed.to_path_buf(),
));
}
let stdlib =
Self::directory_path(system, typeshed.join("stdlib")).map_err(|err| match err {
SearchPathValidationError::NotADirectory(path) => {
SearchPathValidationError::NoStdlibSubdirectory(path)
}
err => err,
})?;
let typeshed_versions =
system_path_to_file(db.upcast(), stdlib.join("VERSIONS")).map_err(|err| match err {
FileError::NotFound => SearchPathValidationError::NoVersionsFile(typeshed),
@@ -421,7 +428,7 @@ impl SearchPath {
SearchPathValidationError::VersionsIsADirectory(typeshed)
}
})?;
crate::typeshed::parse_typeshed_versions(db, typeshed_versions)
super::typeshed::parse_typeshed_versions(db, typeshed_versions)
.as_ref()
.map_err(|validation_error| {
SearchPathValidationError::VersionsParseError(validation_error.clone())
@@ -440,24 +447,26 @@ impl SearchPath {
}
/// Create a new search path pointing to the `site-packages` directory on disk
///
/// TODO: the validation done here is somewhat redundant given that `site-packages`
/// are already validated at a higher level by the time we get here.
/// However, removing the validation here breaks some file-watching tests -- and
/// ultimately we'll probably want all search paths to be validated before a
/// `Program` is instantiated, so it doesn't seem like a huge priority right now.
pub(crate) fn site_packages(
system: &dyn System,
root: SystemPathBuf,
) -> SearchPathResult<Self> {
if system.is_directory(&root) {
Ok(Self(Arc::new(SearchPathInner::SitePackages(root))))
} else {
Err(SearchPathValidationError::NotADirectory(root))
}
Ok(Self(Arc::new(SearchPathInner::SitePackages(
Self::directory_path(system, root)?,
))))
}
/// Create a new search path pointing to an editable installation
pub(crate) fn editable(system: &dyn System, root: SystemPathBuf) -> SearchPathResult<Self> {
if system.is_directory(&root) {
Ok(Self(Arc::new(SearchPathInner::Editable(root))))
} else {
Err(SearchPathValidationError::NotADirectory(root))
}
Ok(Self(Arc::new(SearchPathInner::Editable(
Self::directory_path(system, root)?,
))))
}
#[must_use]
@@ -611,13 +620,13 @@ impl PartialEq<SearchPath> for VendoredPathBuf {
#[cfg(test)]
mod tests {
use ruff_db::program::TargetVersion;
use ruff_db::Db;
use crate::db::tests::TestDb;
use crate::testing::{FileSpec, MockedTypeshed, TestCase, TestCaseBuilder};
use super::*;
use crate::module_resolver::testing::{FileSpec, MockedTypeshed, TestCase, TestCaseBuilder};
use crate::python_version::PythonVersion;
impl ModulePath {
#[must_use]
@@ -857,7 +866,7 @@ mod tests {
fn typeshed_test_case(
typeshed: MockedTypeshed,
target_version: TargetVersion,
target_version: PythonVersion,
) -> (TestDb, SearchPath) {
let TestCase { db, stdlib, .. } = TestCaseBuilder::new()
.with_custom_typeshed(typeshed)
@@ -869,11 +878,11 @@ mod tests {
}
fn py38_typeshed_test_case(typeshed: MockedTypeshed) -> (TestDb, SearchPath) {
typeshed_test_case(typeshed, TargetVersion::Py38)
typeshed_test_case(typeshed, PythonVersion::PY38)
}
fn py39_typeshed_test_case(typeshed: MockedTypeshed) -> (TestDb, SearchPath) {
typeshed_test_case(typeshed, TargetVersion::Py39)
typeshed_test_case(typeshed, PythonVersion::PY39)
}
#[test]
@@ -889,7 +898,7 @@ mod tests {
};
let (db, stdlib_path) = py38_typeshed_test_case(TYPESHED);
let resolver = ResolverState::new(&db, TargetVersion::Py38);
let resolver = ResolverState::new(&db, PythonVersion::PY38);
let asyncio_regular_package = stdlib_path.join("asyncio");
assert!(asyncio_regular_package.is_directory(&resolver));
@@ -917,7 +926,7 @@ mod tests {
};
let (db, stdlib_path) = py38_typeshed_test_case(TYPESHED);
let resolver = ResolverState::new(&db, TargetVersion::Py38);
let resolver = ResolverState::new(&db, PythonVersion::PY38);
let xml_namespace_package = stdlib_path.join("xml");
assert!(xml_namespace_package.is_directory(&resolver));
@@ -939,7 +948,7 @@ mod tests {
};
let (db, stdlib_path) = py38_typeshed_test_case(TYPESHED);
let resolver = ResolverState::new(&db, TargetVersion::Py38);
let resolver = ResolverState::new(&db, PythonVersion::PY38);
let functools_module = stdlib_path.join("functools.pyi");
assert!(functools_module.to_file(&resolver).is_some());
@@ -955,7 +964,7 @@ mod tests {
};
let (db, stdlib_path) = py38_typeshed_test_case(TYPESHED);
let resolver = ResolverState::new(&db, TargetVersion::Py38);
let resolver = ResolverState::new(&db, PythonVersion::PY38);
let collections_regular_package = stdlib_path.join("collections");
assert_eq!(collections_regular_package.to_file(&resolver), None);
@@ -971,7 +980,7 @@ mod tests {
};
let (db, stdlib_path) = py38_typeshed_test_case(TYPESHED);
let resolver = ResolverState::new(&db, TargetVersion::Py38);
let resolver = ResolverState::new(&db, PythonVersion::PY38);
let importlib_namespace_package = stdlib_path.join("importlib");
assert_eq!(importlib_namespace_package.to_file(&resolver), None);
@@ -992,7 +1001,7 @@ mod tests {
};
let (db, stdlib_path) = py38_typeshed_test_case(TYPESHED);
let resolver = ResolverState::new(&db, TargetVersion::Py38);
let resolver = ResolverState::new(&db, PythonVersion::PY38);
let non_existent = stdlib_path.join("doesnt_even_exist");
assert_eq!(non_existent.to_file(&resolver), None);
@@ -1020,7 +1029,7 @@ mod tests {
};
let (db, stdlib_path) = py39_typeshed_test_case(TYPESHED);
let resolver = ResolverState::new(&db, TargetVersion::Py39);
let resolver = ResolverState::new(&db, PythonVersion::PY39);
// Since we've set the target version to Py39,
// `collections` should now exist as a directory, according to VERSIONS...
@@ -1049,7 +1058,7 @@ mod tests {
};
let (db, stdlib_path) = py39_typeshed_test_case(TYPESHED);
let resolver = ResolverState::new(&db, TargetVersion::Py39);
let resolver = ResolverState::new(&db, PythonVersion::PY39);
// The `importlib` directory now also exists
let importlib_namespace_package = stdlib_path.join("importlib");
@@ -1073,7 +1082,7 @@ mod tests {
};
let (db, stdlib_path) = py39_typeshed_test_case(TYPESHED);
let resolver = ResolverState::new(&db, TargetVersion::Py39);
let resolver = ResolverState::new(&db, PythonVersion::PY39);
// The `xml` package no longer exists on py39:
let xml_namespace_package = stdlib_path.join("xml");

View File

@@ -1,18 +1,18 @@
use std::borrow::Cow;
use std::iter::FusedIterator;
use once_cell::sync::Lazy;
use ruff_db::files::{File, FilePath, FileRootKind};
use ruff_db::program::{Program, SearchPathSettings, TargetVersion};
use ruff_db::system::{DirectoryEntry, System, SystemPath, SystemPathBuf};
use ruff_db::vendored::VendoredPath;
use rustc_hash::{FxBuildHasher, FxHashSet};
use ruff_db::files::{File, FilePath, FileRootKind};
use ruff_db::system::{DirectoryEntry, SystemPath, SystemPathBuf};
use ruff_db::vendored::VendoredPath;
use super::module::{Module, ModuleKind};
use super::path::{ModulePath, SearchPath, SearchPathValidationError};
use super::state::ResolverState;
use crate::db::Db;
use crate::module::{Module, ModuleKind};
use crate::module_name::ModuleName;
use crate::path::{ModulePath, SearchPath, SearchPathValidationError};
use crate::state::ResolverState;
use crate::{Program, PythonVersion, SearchPathSettings};
/// Resolves a module name to a module.
pub fn resolve_module(db: &dyn Db, module_name: ModuleName) -> Option<Module> {
@@ -33,10 +33,18 @@ pub(crate) fn resolve_module_query<'db>(
let name = module_name.name(db);
let _span = tracing::trace_span!("resolve_module", %name).entered();
let (search_path, module_file, kind) = resolve_name(db, name)?;
let Some((search_path, module_file, kind)) = resolve_name(db, name) else {
tracing::debug!("Module '{name}' not found in the search paths.");
return None;
};
let module = Module::new(name.clone(), kind, search_path, module_file);
tracing::debug!(
"Resolved module '{name}' to '{path}'.",
path = module_file.path(db)
);
Some(module)
}
@@ -124,17 +132,17 @@ fn try_resolve_module_resolution_settings(
let SearchPathSettings {
extra_paths,
workspace_root,
src_root,
custom_typeshed,
site_packages,
} = program.search_paths(db.upcast());
if let Some(custom_typeshed) = custom_typeshed {
tracing::info!("Custom typeshed directory: {custom_typeshed}");
if !extra_paths.is_empty() {
tracing::info!("Extra search paths: {extra_paths:?}");
}
if !extra_paths.is_empty() {
tracing::info!("extra search paths: {extra_paths:?}");
if let Some(custom_typeshed) = custom_typeshed {
tracing::info!("Custom typeshed directory: {custom_typeshed}");
}
let system = db.system();
@@ -143,23 +151,41 @@ fn try_resolve_module_resolution_settings(
let mut static_search_paths = vec![];
for path in extra_paths {
files.try_add_root(db.upcast(), path, FileRootKind::LibrarySearchPath);
static_search_paths.push(SearchPath::extra(system, path.clone())?);
}
static_search_paths.push(SearchPath::first_party(system, workspace_root.clone())?);
static_search_paths.push(if let Some(custom_typeshed) = custom_typeshed.as_ref() {
let search_path = SearchPath::extra(system, path.clone())?;
files.try_add_root(
db.upcast(),
custom_typeshed,
search_path.as_system_path().unwrap(),
FileRootKind::LibrarySearchPath,
);
SearchPath::custom_stdlib(db, custom_typeshed.clone())?
static_search_paths.push(search_path);
}
static_search_paths.push(SearchPath::first_party(system, src_root.clone())?);
static_search_paths.push(if let Some(custom_typeshed) = custom_typeshed.as_ref() {
let search_path = SearchPath::custom_stdlib(db, custom_typeshed.clone())?;
files.try_add_root(
db.upcast(),
search_path.as_system_path().unwrap(),
FileRootKind::LibrarySearchPath,
);
search_path
} else {
SearchPath::vendored_stdlib()
});
let mut site_packages_paths: Vec<_> = Vec::with_capacity(site_packages.len());
for path in site_packages {
let search_path = SearchPath::site_packages(system, path.to_path_buf())?;
files.try_add_root(
db.upcast(),
search_path.as_system_path().unwrap(),
FileRootKind::LibrarySearchPath,
);
site_packages_paths.push(search_path);
}
// TODO vendor typeshed's third-party stubs as well as the stdlib and fallback to them as a final step
let target_version = program.target_version(db.upcast());
@@ -185,7 +211,7 @@ fn try_resolve_module_resolution_settings(
Ok(ModuleResolutionSettings {
target_version,
static_search_paths,
site_packages_paths: site_packages.to_owned(),
site_packages_paths,
})
}
@@ -226,15 +252,19 @@ pub(crate) fn dynamic_resolution_paths(db: &dyn Db) -> Vec<SearchPath> {
let files = db.files();
let system = db.system();
for site_packages_dir in site_packages_paths {
for site_packages_search_path in site_packages_paths {
let site_packages_dir = site_packages_search_path
.as_system_path()
.expect("Expected site package path to be a system path");
if !existing_paths.insert(Cow::Borrowed(site_packages_dir)) {
continue;
}
let site_packages_root = files.try_add_root(
db.upcast(),
site_packages_dir,
FileRootKind::LibrarySearchPath,
);
let site_packages_root = files
.root(db.upcast(), site_packages_dir)
.expect("Site-package root to have been created.");
// This query needs to be re-executed each time a `.pth` file
// is added, modified or removed from the `site-packages` directory.
// However, we don't use Salsa queries to read the source text of `.pth` files;
@@ -242,8 +272,7 @@ pub(crate) fn dynamic_resolution_paths(db: &dyn Db) -> Vec<SearchPath> {
// site-package directory's revision.
site_packages_root.revision(db.upcast());
dynamic_paths
.push(SearchPath::site_packages(system, site_packages_dir.to_owned()).unwrap());
dynamic_paths.push(site_packages_search_path.clone());
// As well as modules installed directly into `site-packages`,
// the directory may also contain `.pth` files.
@@ -251,22 +280,34 @@ pub(crate) fn dynamic_resolution_paths(db: &dyn Db) -> Vec<SearchPath> {
// containing a (relative or absolute) path.
// Each of these paths may point to an editable install of a package,
// so should be considered an additional search path.
let Ok(pth_file_iterator) = PthFileIterator::new(db, site_packages_dir) else {
continue;
let pth_file_iterator = match PthFileIterator::new(db, site_packages_dir) {
Ok(iterator) => iterator,
Err(error) => {
tracing::warn!(
"Failed to search for editable installation in {site_packages_dir}: {error}"
);
continue;
}
};
// The Python documentation specifies that `.pth` files in `site-packages`
// are processed in alphabetical order, so collecting and then sorting is necessary.
// https://docs.python.org/3/library/site.html#module-site
let mut all_pth_files: Vec<PthFile> = pth_file_iterator.collect();
all_pth_files.sort_by(|a, b| a.path.cmp(&b.path));
all_pth_files.sort_unstable_by(|a, b| a.path.cmp(&b.path));
for pth_file in &all_pth_files {
for installation in pth_file.editable_installations() {
if existing_paths.insert(Cow::Owned(
installation.as_system_path().unwrap().to_path_buf(),
)) {
dynamic_paths.push(installation);
let installations = all_pth_files.iter().flat_map(PthFile::items);
for installation in installations {
if existing_paths.insert(Cow::Owned(installation.clone())) {
match SearchPath::editable(system, installation) {
Ok(search_path) => {
dynamic_paths.push(search_path);
}
Err(error) => {
tracing::debug!("Skipping editable installation: {error}");
}
}
}
}
@@ -312,7 +353,6 @@ impl<'db> FusedIterator for SearchPathIterator<'db> {}
/// One or more lines in a `.pth` file may be a (relative or absolute)
/// path that represents an editable installation of a package.
struct PthFile<'db> {
system: &'db dyn System,
path: SystemPathBuf,
contents: String,
site_packages: &'db SystemPath,
@@ -321,9 +361,8 @@ struct PthFile<'db> {
impl<'db> PthFile<'db> {
/// Yield paths in this `.pth` file that appear to represent editable installations,
/// and should therefore be added as module-resolution search paths.
fn editable_installations(&'db self) -> impl Iterator<Item = SearchPath> + 'db {
fn items(&'db self) -> impl Iterator<Item = SystemPathBuf> + 'db {
let PthFile {
system,
path: _,
contents,
site_packages,
@@ -342,8 +381,8 @@ impl<'db> PthFile<'db> {
{
return None;
}
let possible_editable_install = SystemPath::absolute(line, site_packages);
SearchPath::editable(*system, possible_editable_install).ok()
Some(SystemPath::absolute(line, site_packages))
})
}
}
@@ -392,12 +431,15 @@ impl<'db> Iterator for PthFileIterator<'db> {
continue;
}
let Ok(contents) = db.system().read_to_string(&path) else {
continue;
let contents = match system.read_to_string(&path) {
Ok(contents) => contents,
Err(error) => {
tracing::warn!("Failed to read .pth file '{path}': {error}");
continue;
}
};
return Some(PthFile {
system,
path,
contents,
site_packages,
@@ -409,7 +451,7 @@ impl<'db> Iterator for PthFileIterator<'db> {
/// Validated and normalized module-resolution settings.
#[derive(Clone, Debug, PartialEq, Eq)]
pub(crate) struct ModuleResolutionSettings {
target_version: TargetVersion,
target_version: PythonVersion,
/// Search paths that have been statically determined purely from reading Ruff's configuration settings.
/// These shouldn't ever change unless the config settings themselves change.
@@ -421,11 +463,11 @@ pub(crate) struct ModuleResolutionSettings {
/// That means we can't know where a second or third `site-packages` path should sit
/// in terms of module-resolution priority until we've discovered the editable installs
/// for the first `site-packages` path
site_packages_paths: Vec<SystemPathBuf>,
site_packages_paths: Vec<SearchPath>,
}
impl ModuleResolutionSettings {
fn target_version(&self) -> TargetVersion {
fn target_version(&self) -> PythonVersion {
self.target_version
}
@@ -447,60 +489,20 @@ struct ModuleNameIngredient<'db> {
pub(super) name: ModuleName,
}
/// Modules that are builtin to the Python interpreter itself.
///
/// When these module names are imported, standard module resolution is bypassed:
/// the module name always resolves to the stdlib module,
/// even if there's a module of the same name in the workspace root
/// (which would normally result in the stdlib module being overridden).
///
/// TODO(Alex): write a script to generate this list,
/// similar to what we do in `crates/ruff_python_stdlib/src/sys.rs`
static BUILTIN_MODULES: Lazy<FxHashSet<&str>> = Lazy::new(|| {
const BUILTIN_MODULE_NAMES: &[&str] = &[
"_abc",
"_ast",
"_codecs",
"_collections",
"_functools",
"_imp",
"_io",
"_locale",
"_operator",
"_signal",
"_sre",
"_stat",
"_string",
"_symtable",
"_thread",
"_tokenize",
"_tracemalloc",
"_typing",
"_warnings",
"_weakref",
"atexit",
"builtins",
"errno",
"faulthandler",
"gc",
"itertools",
"marshal",
"posix",
"pwd",
"sys",
"time",
];
BUILTIN_MODULE_NAMES.iter().copied().collect()
});
/// Given a module name and a list of search paths in which to lookup modules,
/// attempt to resolve the module name
fn resolve_name(db: &dyn Db, name: &ModuleName) -> Option<(SearchPath, File, ModuleKind)> {
let resolver_settings = module_resolution_settings(db);
let resolver_state = ResolverState::new(db, resolver_settings.target_version());
let is_builtin_module = BUILTIN_MODULES.contains(&name.as_str());
let target_version = resolver_settings.target_version();
let resolver_state = ResolverState::new(db, target_version);
let is_builtin_module =
ruff_python_stdlib::sys::is_builtin_module(target_version.minor, name.as_str());
for search_path in resolver_settings.search_paths(db) {
// When a builtin module is imported, standard module resolution is bypassed:
// the module name always resolves to the stdlib module,
// even if there's a module of the same name in the first-party root
// (which would normally result in the stdlib module being overridden).
if is_builtin_module && !search_path.is_standard_library() {
continue;
}
@@ -647,9 +649,9 @@ mod tests {
use ruff_db::Db;
use crate::db::tests::TestDb;
use crate::module::ModuleKind;
use crate::module_name::ModuleName;
use crate::testing::{FileSpec, MockedTypeshed, TestCase, TestCaseBuilder};
use crate::module_resolver::module::ModuleKind;
use crate::module_resolver::testing::{FileSpec, MockedTypeshed, TestCase, TestCaseBuilder};
use super::*;
@@ -704,7 +706,7 @@ mod tests {
let TestCase { db, stdlib, .. } = TestCaseBuilder::new()
.with_src_files(SRC)
.with_custom_typeshed(TYPESHED)
.with_target_version(TargetVersion::Py38)
.with_target_version(PythonVersion::PY38)
.build();
let builtins_module_name = ModuleName::new_static("builtins").unwrap();
@@ -722,7 +724,7 @@ mod tests {
let TestCase { db, stdlib, .. } = TestCaseBuilder::new()
.with_custom_typeshed(TYPESHED)
.with_target_version(TargetVersion::Py38)
.with_target_version(PythonVersion::PY38)
.build();
let functools_module_name = ModuleName::new_static("functools").unwrap();
@@ -775,7 +777,7 @@ mod tests {
let TestCase { db, stdlib, .. } = TestCaseBuilder::new()
.with_custom_typeshed(TYPESHED)
.with_target_version(TargetVersion::Py38)
.with_target_version(PythonVersion::PY38)
.build();
let existing_modules = create_module_names(&["asyncio", "functools", "xml.etree"]);
@@ -820,7 +822,7 @@ mod tests {
let TestCase { db, .. } = TestCaseBuilder::new()
.with_custom_typeshed(TYPESHED)
.with_target_version(TargetVersion::Py38)
.with_target_version(PythonVersion::PY38)
.build();
let nonexisting_modules = create_module_names(&[
@@ -864,7 +866,7 @@ mod tests {
let TestCase { db, stdlib, .. } = TestCaseBuilder::new()
.with_custom_typeshed(TYPESHED)
.with_target_version(TargetVersion::Py39)
.with_target_version(PythonVersion::PY39)
.build();
let existing_modules = create_module_names(&[
@@ -906,7 +908,7 @@ mod tests {
let TestCase { db, .. } = TestCaseBuilder::new()
.with_custom_typeshed(TYPESHED)
.with_target_version(TargetVersion::Py39)
.with_target_version(PythonVersion::PY39)
.build();
let nonexisting_modules = create_module_names(&["importlib", "xml", "xml.etree"]);
@@ -930,7 +932,7 @@ mod tests {
let TestCase { db, src, .. } = TestCaseBuilder::new()
.with_src_files(SRC)
.with_custom_typeshed(TYPESHED)
.with_target_version(TargetVersion::Py38)
.with_target_version(PythonVersion::PY38)
.build();
let functools_module_name = ModuleName::new_static("functools").unwrap();
@@ -954,7 +956,7 @@ mod tests {
fn stdlib_uses_vendored_typeshed_when_no_custom_typeshed_supplied() {
let TestCase { db, stdlib, .. } = TestCaseBuilder::new()
.with_vendored_typeshed()
.with_target_version(TargetVersion::default())
.with_target_version(PythonVersion::default())
.build();
let pydoc_data_topics_name = ModuleName::new_static("pydoc_data.topics").unwrap();
@@ -1168,14 +1170,21 @@ mod tests {
#[test]
#[cfg(target_family = "unix")]
fn symlink() -> anyhow::Result<()> {
use crate::db::tests::TestDb;
use ruff_db::program::Program;
use anyhow::Context;
use crate::program::Program;
use ruff_db::system::{OsSystem, SystemPath};
use crate::db::tests::TestDb;
let mut db = TestDb::new();
let temp_dir = tempfile::tempdir()?;
let root = SystemPath::from_std_path(temp_dir.path()).unwrap();
let root = temp_dir
.path()
.canonicalize()
.context("Failed to canonicalize temp dir")?;
let root = SystemPath::from_std_path(&root).unwrap();
db.use_system(OsSystem::new(root));
let src = root.join("src");
@@ -1195,12 +1204,12 @@ mod tests {
let search_paths = SearchPathSettings {
extra_paths: vec![],
workspace_root: src.clone(),
src_root: src.clone(),
custom_typeshed: Some(custom_typeshed.clone()),
site_packages: vec![site_packages],
};
Program::new(&db, TargetVersion::Py38, search_paths);
Program::new(&db, PythonVersion::PY38, search_paths);
let foo_module = resolve_module(&db, ModuleName::new_static("foo").unwrap()).unwrap();
let bar_module = resolve_module(&db, ModuleName::new_static("bar").unwrap()).unwrap();
@@ -1234,7 +1243,7 @@ mod tests {
fn deleting_an_unrelated_file_doesnt_change_module_resolution() {
let TestCase { mut db, src, .. } = TestCaseBuilder::new()
.with_src_files(&[("foo.py", "x = 1"), ("bar.py", "x = 2")])
.with_target_version(TargetVersion::Py38)
.with_target_version(PythonVersion::PY38)
.build();
let foo_module_name = ModuleName::new_static("foo").unwrap();
@@ -1322,7 +1331,7 @@ mod tests {
..
} = TestCaseBuilder::new()
.with_custom_typeshed(TYPESHED)
.with_target_version(TargetVersion::Py38)
.with_target_version(PythonVersion::PY38)
.build();
let functools_module_name = ModuleName::new_static("functools").unwrap();
@@ -1370,7 +1379,7 @@ mod tests {
..
} = TestCaseBuilder::new()
.with_custom_typeshed(TYPESHED)
.with_target_version(TargetVersion::Py38)
.with_target_version(PythonVersion::PY38)
.build();
let functools_module_name = ModuleName::new_static("functools").unwrap();
@@ -1410,7 +1419,7 @@ mod tests {
} = TestCaseBuilder::new()
.with_src_files(SRC)
.with_custom_typeshed(TYPESHED)
.with_target_version(TargetVersion::Py38)
.with_target_version(PythonVersion::PY38)
.build();
let functools_module_name = ModuleName::new_static("functools").unwrap();
@@ -1696,10 +1705,10 @@ not_a_directory
Program::new(
&db,
TargetVersion::default(),
PythonVersion::default(),
SearchPathSettings {
extra_paths: vec![],
workspace_root: SystemPathBuf::from("/src"),
src_root: SystemPathBuf::from("/src"),
custom_typeshed: None,
site_packages: vec![venv_site_packages, system_site_packages],
},

View File

@@ -1,17 +1,17 @@
use ruff_db::program::TargetVersion;
use ruff_db::vendored::VendoredFileSystem;
use super::typeshed::LazyTypeshedVersions;
use crate::db::Db;
use crate::typeshed::LazyTypeshedVersions;
use crate::python_version::PythonVersion;
pub(crate) struct ResolverState<'db> {
pub(crate) db: &'db dyn Db,
pub(crate) typeshed_versions: LazyTypeshedVersions<'db>,
pub(crate) target_version: TargetVersion,
pub(crate) target_version: PythonVersion,
}
impl<'db> ResolverState<'db> {
pub(crate) fn new(db: &'db dyn Db, target_version: TargetVersion) -> Self {
pub(crate) fn new(db: &'db dyn Db, target_version: PythonVersion) -> Self {
Self {
db,
typeshed_versions: LazyTypeshedVersions::new(),

View File

@@ -1,8 +1,9 @@
use ruff_db::program::{Program, SearchPathSettings, TargetVersion};
use ruff_db::system::{DbWithTestSystem, SystemPath, SystemPathBuf};
use ruff_db::vendored::VendoredPathBuf;
use crate::db::tests::TestDb;
use crate::program::{Program, SearchPathSettings};
use crate::python_version::PythonVersion;
/// A test case for the module resolver.
///
@@ -16,7 +17,7 @@ pub(crate) struct TestCase<T> {
// so this is a single directory instead of a `Vec` of directories,
// like it is in `ruff_db::Program`.
pub(crate) site_packages: SystemPathBuf,
pub(crate) target_version: TargetVersion,
pub(crate) target_version: PythonVersion,
}
/// A `(file_name, file_contents)` tuple
@@ -98,7 +99,7 @@ pub(crate) struct UnspecifiedTypeshed;
/// to `()`.
pub(crate) struct TestCaseBuilder<T> {
typeshed_option: T,
target_version: TargetVersion,
target_version: PythonVersion,
first_party_files: Vec<FileSpec>,
site_packages_files: Vec<FileSpec>,
}
@@ -117,7 +118,7 @@ impl<T> TestCaseBuilder<T> {
}
/// Specify the target Python version the module resolver should assume
pub(crate) fn with_target_version(mut self, target_version: TargetVersion) -> Self {
pub(crate) fn with_target_version(mut self, target_version: PythonVersion) -> Self {
self.target_version = target_version;
self
}
@@ -144,7 +145,7 @@ impl TestCaseBuilder<UnspecifiedTypeshed> {
pub(crate) fn new() -> TestCaseBuilder<UnspecifiedTypeshed> {
Self {
typeshed_option: UnspecifiedTypeshed,
target_version: TargetVersion::default(),
target_version: PythonVersion::default(),
first_party_files: vec![],
site_packages_files: vec![],
}
@@ -224,7 +225,7 @@ impl TestCaseBuilder<MockedTypeshed> {
target_version,
SearchPathSettings {
extra_paths: vec![],
workspace_root: src.clone(),
src_root: src.clone(),
custom_typeshed: Some(typeshed.clone()),
site_packages: vec![site_packages.clone()],
},
@@ -277,7 +278,7 @@ impl TestCaseBuilder<VendoredTypeshed> {
target_version,
SearchPathSettings {
extra_paths: vec![],
workspace_root: src.clone(),
src_root: src.clone(),
custom_typeshed: None,
site_packages: vec![site_packages.clone()],
},

View File

@@ -1,8 +1,8 @@
pub use self::vendored::vendored_typeshed_stubs;
pub(crate) use self::versions::{
parse_typeshed_versions, LazyTypeshedVersions, TypeshedVersionsQueryResult,
pub(super) use self::versions::{
parse_typeshed_versions, LazyTypeshedVersions, TypeshedVersionsParseError,
TypeshedVersionsQueryResult,
};
pub use self::versions::{TypeshedVersionsParseError, TypeshedVersionsParseErrorKind};
mod vendored;
mod versions;

View File

@@ -6,16 +6,15 @@ use std::ops::{RangeFrom, RangeInclusive};
use std::str::FromStr;
use once_cell::sync::Lazy;
use ruff_db::program::TargetVersion;
use ruff_db::system::SystemPath;
use rustc_hash::FxHashMap;
use ruff_db::files::{system_path_to_file, File};
use super::vendored::vendored_typeshed_stubs;
use crate::db::Db;
use crate::module_name::ModuleName;
use super::vendored::vendored_typeshed_stubs;
use crate::python_version::PythonVersion;
#[derive(Debug)]
pub(crate) struct LazyTypeshedVersions<'db>(OnceCell<&'db TypeshedVersions>);
@@ -44,7 +43,7 @@ impl<'db> LazyTypeshedVersions<'db> {
db: &'db dyn Db,
module: &ModuleName,
stdlib_root: Option<&SystemPath>,
target_version: TargetVersion,
target_version: PythonVersion,
) -> TypeshedVersionsQueryResult {
let versions = self.0.get_or_init(|| {
let versions_path = if let Some(system_path) = stdlib_root {
@@ -64,7 +63,7 @@ impl<'db> LazyTypeshedVersions<'db> {
// Unwrapping here is not correct...
parse_typeshed_versions(db, versions_file).as_ref().unwrap()
});
versions.query_module(module, PyVersion::from(target_version))
versions.query_module(module, target_version)
}
}
@@ -90,7 +89,7 @@ static VENDORED_VERSIONS: Lazy<TypeshedVersions> = Lazy::new(|| {
});
#[derive(Debug, PartialEq, Eq, Clone)]
pub struct TypeshedVersionsParseError {
pub(crate) struct TypeshedVersionsParseError {
line_number: Option<NonZeroU16>,
reason: TypeshedVersionsParseErrorKind,
}
@@ -123,7 +122,7 @@ impl std::error::Error for TypeshedVersionsParseError {
}
#[derive(Debug, PartialEq, Eq, Clone)]
pub enum TypeshedVersionsParseErrorKind {
pub(super) enum TypeshedVersionsParseErrorKind {
TooManyLines(NonZeroUsize),
UnexpectedNumberOfColons,
InvalidModuleName(String),
@@ -178,7 +177,7 @@ impl TypeshedVersions {
fn query_module(
&self,
module: &ModuleName,
target_version: PyVersion,
target_version: PythonVersion,
) -> TypeshedVersionsQueryResult {
if let Some(range) = self.exact(module) {
if range.contains(target_version) {
@@ -323,13 +322,13 @@ impl fmt::Display for TypeshedVersions {
#[derive(Debug, Clone, Eq, PartialEq, Hash)]
enum PyVersionRange {
AvailableFrom(RangeFrom<PyVersion>),
AvailableWithin(RangeInclusive<PyVersion>),
AvailableFrom(RangeFrom<PythonVersion>),
AvailableWithin(RangeInclusive<PythonVersion>),
}
impl PyVersionRange {
#[must_use]
fn contains(&self, version: PyVersion) -> bool {
fn contains(&self, version: PythonVersion) -> bool {
match self {
Self::AvailableFrom(inner) => inner.contains(&version),
Self::AvailableWithin(inner) => inner.contains(&version),
@@ -343,9 +342,14 @@ impl FromStr for PyVersionRange {
fn from_str(s: &str) -> Result<Self, Self::Err> {
let mut parts = s.split('-').map(str::trim);
match (parts.next(), parts.next(), parts.next()) {
(Some(lower), Some(""), None) => Ok(Self::AvailableFrom((lower.parse()?)..)),
(Some(lower), Some(""), None) => {
let lower = PythonVersion::from_versions_file_string(lower)?;
Ok(Self::AvailableFrom(lower..))
}
(Some(lower), Some(upper), None) => {
Ok(Self::AvailableWithin((lower.parse()?)..=(upper.parse()?)))
let lower = PythonVersion::from_versions_file_string(lower)?;
let upper = PythonVersion::from_versions_file_string(upper)?;
Ok(Self::AvailableWithin(lower..=upper))
}
_ => Err(TypeshedVersionsParseErrorKind::UnexpectedNumberOfHyphens),
}
@@ -363,74 +367,20 @@ impl fmt::Display for PyVersionRange {
}
}
#[derive(Debug, Clone, Copy, Eq, PartialEq, Ord, PartialOrd, Hash)]
struct PyVersion {
major: u8,
minor: u8,
}
impl FromStr for PyVersion {
type Err = TypeshedVersionsParseErrorKind;
fn from_str(s: &str) -> Result<Self, Self::Err> {
impl PythonVersion {
fn from_versions_file_string(s: &str) -> Result<Self, TypeshedVersionsParseErrorKind> {
let mut parts = s.split('.').map(str::trim);
let (Some(major), Some(minor), None) = (parts.next(), parts.next(), parts.next()) else {
return Err(TypeshedVersionsParseErrorKind::UnexpectedNumberOfPeriods(
s.to_string(),
));
};
let major = match u8::from_str(major) {
Ok(major) => major,
Err(err) => {
return Err(TypeshedVersionsParseErrorKind::IntegerParsingFailure {
version: s.to_string(),
err,
})
PythonVersion::try_from((major, minor)).map_err(|int_parse_error| {
TypeshedVersionsParseErrorKind::IntegerParsingFailure {
version: s.to_string(),
err: int_parse_error,
}
};
let minor = match u8::from_str(minor) {
Ok(minor) => minor,
Err(err) => {
return Err(TypeshedVersionsParseErrorKind::IntegerParsingFailure {
version: s.to_string(),
err,
})
}
};
Ok(Self { major, minor })
}
}
impl fmt::Display for PyVersion {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
let PyVersion { major, minor } = self;
write!(f, "{major}.{minor}")
}
}
impl From<TargetVersion> for PyVersion {
fn from(value: TargetVersion) -> Self {
match value {
TargetVersion::Py37 => PyVersion { major: 3, minor: 7 },
TargetVersion::Py38 => PyVersion { major: 3, minor: 8 },
TargetVersion::Py39 => PyVersion { major: 3, minor: 9 },
TargetVersion::Py310 => PyVersion {
major: 3,
minor: 10,
},
TargetVersion::Py311 => PyVersion {
major: 3,
minor: 11,
},
TargetVersion::Py312 => PyVersion {
major: 3,
minor: 12,
},
TargetVersion::Py313 => PyVersion {
major: 3,
minor: 13,
},
}
})
}
}
@@ -440,7 +390,6 @@ mod tests {
use std::path::Path;
use insta::assert_snapshot;
use ruff_db::program::TargetVersion;
use super::*;
@@ -478,34 +427,34 @@ mod tests {
assert!(versions.contains_exact(&asyncio));
assert_eq!(
versions.query_module(&asyncio, TargetVersion::Py310.into()),
versions.query_module(&asyncio, PythonVersion::PY310),
TypeshedVersionsQueryResult::Exists
);
assert!(versions.contains_exact(&asyncio_staggered));
assert_eq!(
versions.query_module(&asyncio_staggered, TargetVersion::Py38.into()),
versions.query_module(&asyncio_staggered, PythonVersion::PY38),
TypeshedVersionsQueryResult::Exists
);
assert_eq!(
versions.query_module(&asyncio_staggered, TargetVersion::Py37.into()),
versions.query_module(&asyncio_staggered, PythonVersion::PY37),
TypeshedVersionsQueryResult::DoesNotExist
);
assert!(versions.contains_exact(&audioop));
assert_eq!(
versions.query_module(&audioop, TargetVersion::Py312.into()),
versions.query_module(&audioop, PythonVersion::PY312),
TypeshedVersionsQueryResult::Exists
);
assert_eq!(
versions.query_module(&audioop, TargetVersion::Py313.into()),
versions.query_module(&audioop, PythonVersion::PY313),
TypeshedVersionsQueryResult::DoesNotExist
);
}
#[test]
fn typeshed_versions_consistent_with_vendored_stubs() {
const VERSIONS_DATA: &str = include_str!("../../vendor/typeshed/stdlib/VERSIONS");
const VERSIONS_DATA: &str = include_str!("../../../vendor/typeshed/stdlib/VERSIONS");
let vendored_typeshed_dir = Path::new("vendor/typeshed").canonicalize().unwrap();
let vendored_typeshed_versions = TypeshedVersions::from_str(VERSIONS_DATA).unwrap();
@@ -590,15 +539,15 @@ foo: 3.8- # trailing comment
assert!(parsed_versions.contains_exact(&bar));
assert_eq!(
parsed_versions.query_module(&bar, TargetVersion::Py37.into()),
parsed_versions.query_module(&bar, PythonVersion::PY37),
TypeshedVersionsQueryResult::Exists
);
assert_eq!(
parsed_versions.query_module(&bar, TargetVersion::Py310.into()),
parsed_versions.query_module(&bar, PythonVersion::PY310),
TypeshedVersionsQueryResult::Exists
);
assert_eq!(
parsed_versions.query_module(&bar, TargetVersion::Py311.into()),
parsed_versions.query_module(&bar, PythonVersion::PY311),
TypeshedVersionsQueryResult::DoesNotExist
);
}
@@ -610,15 +559,15 @@ foo: 3.8- # trailing comment
assert!(parsed_versions.contains_exact(&foo));
assert_eq!(
parsed_versions.query_module(&foo, TargetVersion::Py37.into()),
parsed_versions.query_module(&foo, PythonVersion::PY37),
TypeshedVersionsQueryResult::DoesNotExist
);
assert_eq!(
parsed_versions.query_module(&foo, TargetVersion::Py38.into()),
parsed_versions.query_module(&foo, PythonVersion::PY38),
TypeshedVersionsQueryResult::Exists
);
assert_eq!(
parsed_versions.query_module(&foo, TargetVersion::Py311.into()),
parsed_versions.query_module(&foo, PythonVersion::PY311),
TypeshedVersionsQueryResult::Exists
);
}
@@ -630,15 +579,15 @@ foo: 3.8- # trailing comment
assert!(parsed_versions.contains_exact(&bar_baz));
assert_eq!(
parsed_versions.query_module(&bar_baz, TargetVersion::Py37.into()),
parsed_versions.query_module(&bar_baz, PythonVersion::PY37),
TypeshedVersionsQueryResult::Exists
);
assert_eq!(
parsed_versions.query_module(&bar_baz, TargetVersion::Py39.into()),
parsed_versions.query_module(&bar_baz, PythonVersion::PY39),
TypeshedVersionsQueryResult::Exists
);
assert_eq!(
parsed_versions.query_module(&bar_baz, TargetVersion::Py310.into()),
parsed_versions.query_module(&bar_baz, PythonVersion::PY310),
TypeshedVersionsQueryResult::DoesNotExist
);
}
@@ -650,15 +599,15 @@ foo: 3.8- # trailing comment
assert!(!parsed_versions.contains_exact(&bar_eggs));
assert_eq!(
parsed_versions.query_module(&bar_eggs, TargetVersion::Py37.into()),
parsed_versions.query_module(&bar_eggs, PythonVersion::PY37),
TypeshedVersionsQueryResult::MaybeExists
);
assert_eq!(
parsed_versions.query_module(&bar_eggs, TargetVersion::Py310.into()),
parsed_versions.query_module(&bar_eggs, PythonVersion::PY310),
TypeshedVersionsQueryResult::MaybeExists
);
assert_eq!(
parsed_versions.query_module(&bar_eggs, TargetVersion::Py311.into()),
parsed_versions.query_module(&bar_eggs, PythonVersion::PY311),
TypeshedVersionsQueryResult::DoesNotExist
);
}
@@ -670,11 +619,11 @@ foo: 3.8- # trailing comment
assert!(!parsed_versions.contains_exact(&spam));
assert_eq!(
parsed_versions.query_module(&spam, TargetVersion::Py37.into()),
parsed_versions.query_module(&spam, PythonVersion::PY37),
TypeshedVersionsQueryResult::DoesNotExist
);
assert_eq!(
parsed_versions.query_module(&spam, TargetVersion::Py313.into()),
parsed_versions.query_module(&spam, PythonVersion::PY313),
TypeshedVersionsQueryResult::DoesNotExist
);
}

View File

@@ -1,9 +1,11 @@
use crate::{system::SystemPathBuf, Db};
use crate::python_version::PythonVersion;
use crate::Db;
use ruff_db::system::SystemPathBuf;
use salsa::Durability;
#[salsa::input(singleton)]
pub struct Program {
pub target_version: TargetVersion,
pub target_version: PythonVersion,
#[return_ref]
pub search_paths: SearchPathSettings,
@@ -19,53 +21,12 @@ impl Program {
#[derive(Debug, Eq, PartialEq)]
pub struct ProgramSettings {
pub target_version: TargetVersion,
pub target_version: PythonVersion,
pub search_paths: SearchPathSettings,
}
/// Enumeration of all supported Python versions
///
/// TODO: unify with the `PythonVersion` enum in the linter/formatter crates?
#[derive(Copy, Clone, Hash, PartialEq, Eq, PartialOrd, Ord, Default)]
pub enum TargetVersion {
Py37,
#[default]
Py38,
Py39,
Py310,
Py311,
Py312,
Py313,
}
impl TargetVersion {
const fn as_str(self) -> &'static str {
match self {
Self::Py37 => "py37",
Self::Py38 => "py38",
Self::Py39 => "py39",
Self::Py310 => "py310",
Self::Py311 => "py311",
Self::Py312 => "py312",
Self::Py313 => "py313",
}
}
}
impl std::fmt::Display for TargetVersion {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str(self.as_str())
}
}
impl std::fmt::Debug for TargetVersion {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
std::fmt::Display::fmt(self, f)
}
}
/// Configures the search paths for module resolution.
#[derive(Eq, PartialEq, Debug, Clone)]
#[derive(Eq, PartialEq, Debug, Clone, Default)]
pub struct SearchPathSettings {
/// List of user-provided paths that should take first priority in the module resolution.
/// Examples in other type checkers are mypy's MYPYPATH environment variable,
@@ -73,7 +34,7 @@ pub struct SearchPathSettings {
pub extra_paths: Vec<SystemPathBuf>,
/// The root of the workspace, used for finding first-party modules.
pub workspace_root: SystemPathBuf,
pub src_root: SystemPathBuf,
/// Optional path to a "custom typeshed" directory on disk for us to use for standard-library types.
/// If this is not provided, we will fallback to our vendored typeshed stubs for the stdlib,

View File

@@ -0,0 +1,62 @@
use std::fmt;
/// Representation of a Python version.
///
/// Unlike the `TargetVersion` enums in the CLI crates,
/// this does not necessarily represent a Python version that we actually support.
#[derive(Debug, Clone, Copy, Eq, PartialEq, Ord, PartialOrd, Hash)]
pub struct PythonVersion {
pub major: u8,
pub minor: u8,
}
impl PythonVersion {
pub const PY37: PythonVersion = PythonVersion { major: 3, minor: 7 };
pub const PY38: PythonVersion = PythonVersion { major: 3, minor: 8 };
pub const PY39: PythonVersion = PythonVersion { major: 3, minor: 9 };
pub const PY310: PythonVersion = PythonVersion {
major: 3,
minor: 10,
};
pub const PY311: PythonVersion = PythonVersion {
major: 3,
minor: 11,
};
pub const PY312: PythonVersion = PythonVersion {
major: 3,
minor: 12,
};
pub const PY313: PythonVersion = PythonVersion {
major: 3,
minor: 13,
};
pub fn free_threaded_build_available(self) -> bool {
self >= PythonVersion::PY313
}
}
impl Default for PythonVersion {
fn default() -> Self {
Self::PY38
}
}
impl TryFrom<(&str, &str)> for PythonVersion {
type Error = std::num::ParseIntError;
fn try_from(value: (&str, &str)) -> Result<Self, Self::Error> {
let (major, minor) = value;
Ok(Self {
major: major.parse()?,
minor: minor.parse()?,
})
}
}
impl fmt::Display for PythonVersion {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
let PythonVersion { major, minor } = self;
write!(f, "{major}.{minor}")
}
}

View File

@@ -34,7 +34,7 @@ type SymbolMap = hashbrown::HashMap<ScopedSymbolId, (), ()>;
/// Prefer using [`symbol_table`] when working with symbols from a single scope.
#[salsa::tracked(return_ref, no_eq)]
pub(crate) fn semantic_index(db: &dyn Db, file: File) -> SemanticIndex<'_> {
let _span = tracing::trace_span!("semantic_index", file=?file.path(db)).entered();
let _span = tracing::trace_span!("semantic_index", file = %file.path(db)).entered();
let parsed = parsed_module(db.upcast(), file);
@@ -50,7 +50,7 @@ pub(crate) fn semantic_index(db: &dyn Db, file: File) -> SemanticIndex<'_> {
pub(crate) fn symbol_table<'db>(db: &'db dyn Db, scope: ScopeId<'db>) -> Arc<SymbolTable> {
let file = scope.file(db);
let _span =
tracing::trace_span!("symbol_table", scope=?scope.as_id(), file=?file.path(db)).entered();
tracing::trace_span!("symbol_table", scope=?scope.as_id(), file=%file.path(db)).entered();
let index = semantic_index(db, file);
index.symbol_table(scope.file_scope_id(db))
@@ -65,7 +65,7 @@ pub(crate) fn symbol_table<'db>(db: &'db dyn Db, scope: ScopeId<'db>) -> Arc<Sym
pub(crate) fn use_def_map<'db>(db: &'db dyn Db, scope: ScopeId<'db>) -> Arc<UseDefMap<'db>> {
let file = scope.file(db);
let _span =
tracing::trace_span!("use_def_map", scope=?scope.as_id(), file=?file.path(db)).entered();
tracing::trace_span!("use_def_map", scope=?scope.as_id(), file=%file.path(db)).entered();
let index = semantic_index(db, file);
index.use_def_map(scope.file_scope_id(db))
@@ -74,7 +74,7 @@ pub(crate) fn use_def_map<'db>(db: &'db dyn Db, scope: ScopeId<'db>) -> Arc<UseD
/// Returns the module global scope of `file`.
#[salsa::tracked]
pub(crate) fn global_scope(db: &dyn Db, file: File) -> ScopeId<'_> {
let _span = tracing::trace_span!("global_scope", file=?file.path(db)).entered();
let _span = tracing::trace_span!("global_scope", file = %file.path(db)).entered();
FileScopeId::global().to_scope_id(db, file)
}

View File

@@ -1,8 +1,9 @@
use red_knot_module_resolver::{resolve_module, Module, ModuleName};
use ruff_db::files::File;
use ruff_python_ast as ast;
use ruff_python_ast::{Expr, ExpressionRef, StmtClassDef};
use crate::module_name::ModuleName;
use crate::module_resolver::{resolve_module, Module};
use crate::semantic_index::ast_ids::HasScopedAstId;
use crate::semantic_index::semantic_index;
use crate::types::{definition_ty, global_symbol_ty_by_name, infer_scope_types, Type};
@@ -25,7 +26,7 @@ impl<'db> SemanticModel<'db> {
}
pub fn resolve_module(&self, module_name: ModuleName) -> Option<Module> {
resolve_module(self.db.upcast(), module_name)
resolve_module(self.db, module_name)
}
pub fn global_symbol_ty(&self, module: &Module, symbol_name: &str) -> Type<'db> {
@@ -164,10 +165,11 @@ impl HasTy for ast::Alias {
mod tests {
use ruff_db::files::system_path_to_file;
use ruff_db::parsed::parsed_module;
use ruff_db::program::{Program, SearchPathSettings, TargetVersion};
use ruff_db::system::{DbWithTestSystem, SystemPathBuf};
use crate::db::tests::TestDb;
use crate::program::{Program, SearchPathSettings};
use crate::python_version::PythonVersion;
use crate::types::Type;
use crate::{HasTy, SemanticModel};
@@ -175,10 +177,10 @@ mod tests {
let db = TestDb::new();
Program::new(
&db,
TargetVersion::Py38,
PythonVersion::default(),
SearchPathSettings {
extra_paths: vec![],
workspace_root: SystemPathBuf::from("/src"),
src_root: SystemPathBuf::from("/src"),
site_packages: vec![],
custom_typeshed: None,
},

View File

@@ -130,6 +130,8 @@ pub enum Type<'db> {
Union(UnionType<'db>),
Intersection(IntersectionType<'db>),
IntLiteral(i64),
/// A boolean literal, either `True` or `False`.
BooleanLiteral(bool),
// TODO protocols, callable types, overloads, generics, type vars
}
@@ -175,6 +177,17 @@ impl<'db> Type<'db> {
// TODO raise error
Type::Unknown
}
Type::BooleanLiteral(_) => Type::Unknown,
}
}
#[must_use]
pub fn instance(&self) -> Type<'db> {
match self {
Type::Any => Type::Any,
Type::Unknown => Type::Unknown,
Type::Class(class) => Type::Instance(*class),
_ => Type::Unknown, // TODO type errors
}
}
}

View File

@@ -35,6 +35,9 @@ impl Display for DisplayType<'_> {
Type::Union(union) => union.display(self.db).fmt(f),
Type::Intersection(intersection) => intersection.display(self.db).fmt(f),
Type::IntLiteral(n) => write!(f, "Literal[{n}]"),
Type::BooleanLiteral(boolean) => {
write!(f, "Literal[{}]", if *boolean { "True" } else { "False" })
}
}
}
}

View File

@@ -24,13 +24,14 @@ use rustc_hash::FxHashMap;
use salsa;
use salsa::plumbing::AsId;
use red_knot_module_resolver::{resolve_module, ModuleName};
use ruff_db::files::File;
use ruff_db::parsed::parsed_module;
use ruff_python_ast as ast;
use ruff_python_ast::{ExprContext, TypeParams};
use crate::builtins::builtins_scope;
use crate::module_name::ModuleName;
use crate::module_resolver::resolve_module;
use crate::semantic_index::ast_ids::{HasScopedAstId, HasScopedUseId, ScopedExpressionId};
use crate::semantic_index::definition::{Definition, DefinitionKind, DefinitionNodeKey};
use crate::semantic_index::expression::Expression;
@@ -50,7 +51,7 @@ use crate::Db;
pub(crate) fn infer_scope_types<'db>(db: &'db dyn Db, scope: ScopeId<'db>) -> TypeInference<'db> {
let file = scope.file(db);
let _span =
tracing::trace_span!("infer_scope_types", scope=?scope.as_id(), file=?file.path(db))
tracing::trace_span!("infer_scope_types", scope=?scope.as_id(), file=%file.path(db))
.entered();
// Using the index here is fine because the code below depends on the AST anyway.
@@ -83,7 +84,7 @@ pub(crate) fn infer_definition_types<'db>(
let _span = tracing::trace_span!(
"infer_definition_types",
definition = ?definition.as_id(),
file = ?file.path(db)
file = %file.path(db)
)
.entered();
@@ -104,7 +105,7 @@ pub(crate) fn infer_expression_types<'db>(
) -> TypeInference<'db> {
let file = expression.file(db);
let _span =
tracing::trace_span!("infer_expression_types", expression=?expression.as_id(), file=?file.path(db))
tracing::trace_span!("infer_expression_types", expression=?expression.as_id(), file=%file.path(db))
.entered();
let index = semantic_index(db, file);
@@ -840,9 +841,7 @@ impl<'db> TypeInferenceBuilder<'db> {
}
fn module_ty_from_name(&self, name: &ast::Identifier) -> Type<'db> {
let module_name = ModuleName::new(&name.id);
let module =
module_name.and_then(|module_name| resolve_module(self.db.upcast(), module_name));
let module = ModuleName::new(&name.id).and_then(|name| resolve_module(self.db, name));
module
.map(|module| Type::Module(module.file()))
.unwrap_or(Type::Unbound)
@@ -932,24 +931,24 @@ impl<'db> TypeInferenceBuilder<'db> {
ast::Number::Int(n) => n
.as_i64()
.map(Type::IntLiteral)
.unwrap_or_else(|| builtins_symbol_ty_by_name(self.db, "int")),
// TODO builtins.float or builtins.complex
_ => Type::Unknown,
.unwrap_or_else(|| builtins_symbol_ty_by_name(self.db, "int").instance()),
ast::Number::Float(_) => builtins_symbol_ty_by_name(self.db, "float").instance(),
ast::Number::Complex { .. } => {
builtins_symbol_ty_by_name(self.db, "complex").instance()
}
}
}
#[allow(clippy::unused_self)]
fn infer_boolean_literal_expression(
&mut self,
_literal: &ast::ExprBooleanLiteral,
) -> Type<'db> {
// TODO builtins.bool and boolean Literal types
Type::Unknown
fn infer_boolean_literal_expression(&mut self, literal: &ast::ExprBooleanLiteral) -> Type<'db> {
let ast::ExprBooleanLiteral { range: _, value } = literal;
Type::BooleanLiteral(*value)
}
#[allow(clippy::unused_self)]
fn infer_string_literal_expression(&mut self, _literal: &ast::ExprStringLiteral) -> Type<'db> {
// TODO Literal[str] or builtins.str
// TODO Literal["..."] or str
Type::Unknown
}
@@ -997,7 +996,7 @@ impl<'db> TypeInferenceBuilder<'db> {
&mut self,
_literal: &ast::ExprEllipsisLiteral,
) -> Type<'db> {
// TODO builtins.Ellipsis
// TODO Ellipsis
Type::Unknown
}
@@ -1014,7 +1013,7 @@ impl<'db> TypeInferenceBuilder<'db> {
}
// TODO generic
builtins_symbol_ty_by_name(self.db, "tuple")
builtins_symbol_ty_by_name(self.db, "tuple").instance()
}
fn infer_list_expression(&mut self, list: &ast::ExprList) -> Type<'db> {
@@ -1029,7 +1028,7 @@ impl<'db> TypeInferenceBuilder<'db> {
}
// TODO generic
builtins_symbol_ty_by_name(self.db, "list")
builtins_symbol_ty_by_name(self.db, "list").instance()
}
fn infer_set_expression(&mut self, set: &ast::ExprSet) -> Type<'db> {
@@ -1040,7 +1039,7 @@ impl<'db> TypeInferenceBuilder<'db> {
}
// TODO generic
builtins_symbol_ty_by_name(self.db, "set")
builtins_symbol_ty_by_name(self.db, "set").instance()
}
fn infer_dict_expression(&mut self, dict: &ast::ExprDict) -> Type<'db> {
@@ -1052,7 +1051,7 @@ impl<'db> TypeInferenceBuilder<'db> {
}
// TODO generic
builtins_symbol_ty_by_name(self.db, "dict")
builtins_symbol_ty_by_name(self.db, "dict").instance()
}
fn infer_generator_expression(&mut self, generator: &ast::ExprGenerator) -> Type<'db> {
@@ -1352,22 +1351,26 @@ impl<'db> TypeInferenceBuilder<'db> {
match right_ty {
Type::IntLiteral(m) => {
match op {
ast::Operator::Add => n
.checked_add(m)
.map(Type::IntLiteral)
.unwrap_or_else(|| builtins_symbol_ty_by_name(self.db, "int")),
ast::Operator::Sub => n
.checked_sub(m)
.map(Type::IntLiteral)
.unwrap_or_else(|| builtins_symbol_ty_by_name(self.db, "int")),
ast::Operator::Mult => n
.checked_mul(m)
.map(Type::IntLiteral)
.unwrap_or_else(|| builtins_symbol_ty_by_name(self.db, "int")),
ast::Operator::Div => n
.checked_div(m)
.map(Type::IntLiteral)
.unwrap_or_else(|| builtins_symbol_ty_by_name(self.db, "int")),
ast::Operator::Add => {
n.checked_add(m).map(Type::IntLiteral).unwrap_or_else(|| {
builtins_symbol_ty_by_name(self.db, "int").instance()
})
}
ast::Operator::Sub => {
n.checked_sub(m).map(Type::IntLiteral).unwrap_or_else(|| {
builtins_symbol_ty_by_name(self.db, "int").instance()
})
}
ast::Operator::Mult => {
n.checked_mul(m).map(Type::IntLiteral).unwrap_or_else(|| {
builtins_symbol_ty_by_name(self.db, "int").instance()
})
}
ast::Operator::Div => {
n.checked_div(m).map(Type::IntLiteral).unwrap_or_else(|| {
builtins_symbol_ty_by_name(self.db, "int").instance()
})
}
ast::Operator::Mod => n
.checked_rem(m)
.map(Type::IntLiteral)
@@ -1441,7 +1444,7 @@ impl<'db> TypeInferenceBuilder<'db> {
self.infer_optional_expression(upper.as_deref());
self.infer_optional_expression(step.as_deref());
// TODO builtins.slice
// TODO slice
Type::Unknown
}
@@ -1493,13 +1496,14 @@ impl<'db> TypeInferenceBuilder<'db> {
mod tests {
use ruff_db::files::{system_path_to_file, File};
use ruff_db::parsed::parsed_module;
use ruff_db::program::{Program, SearchPathSettings, TargetVersion};
use ruff_db::system::{DbWithTestSystem, SystemPathBuf};
use ruff_db::testing::assert_function_query_was_not_run;
use ruff_python_ast::name::Name;
use crate::builtins::builtins_scope;
use crate::db::tests::TestDb;
use crate::program::{Program, SearchPathSettings};
use crate::python_version::PythonVersion;
use crate::semantic_index::definition::Definition;
use crate::semantic_index::symbol::FileScopeId;
use crate::semantic_index::{global_scope, semantic_index, symbol_table, use_def_map};
@@ -1511,10 +1515,10 @@ mod tests {
Program::new(
&db,
TargetVersion::Py38,
PythonVersion::default(),
SearchPathSettings {
extra_paths: Vec::new(),
workspace_root: SystemPathBuf::from("/src"),
src_root: SystemPathBuf::from("/src"),
site_packages: vec![],
custom_typeshed: None,
},
@@ -1528,10 +1532,10 @@ mod tests {
Program::new(
&db,
TargetVersion::Py38,
PythonVersion::default(),
SearchPathSettings {
extra_paths: Vec::new(),
workspace_root: SystemPathBuf::from("/src"),
src_root: SystemPathBuf::from("/src"),
site_packages: vec![],
custom_typeshed: Some(SystemPathBuf::from(typeshed)),
},
@@ -1639,12 +1643,35 @@ mod tests {
}
#[test]
fn resolve_literal() -> anyhow::Result<()> {
fn number_literal() -> anyhow::Result<()> {
let mut db = setup_db();
db.write_file("src/a.py", "x = 1")?;
db.write_dedented(
"src/a.py",
"
a = 1
b = 9223372036854775808
c = 1.45
d = 2j
",
)?;
assert_public_ty(&db, "src/a.py", "x", "Literal[1]");
assert_public_ty(&db, "src/a.py", "a", "Literal[1]");
assert_public_ty(&db, "src/a.py", "b", "int");
assert_public_ty(&db, "src/a.py", "c", "float");
assert_public_ty(&db, "src/a.py", "d", "complex");
Ok(())
}
#[test]
fn boolean_literal() -> anyhow::Result<()> {
let mut db = setup_db();
db.write_file("src/a.py", "x = True\ny = False")?;
assert_public_ty(&db, "src/a.py", "x", "Literal[True]");
assert_public_ty(&db, "src/a.py", "y", "Literal[False]");
Ok(())
}
@@ -2279,7 +2306,7 @@ mod tests {
",
)?;
assert_public_ty(&db, "/src/a.py", "x", "Literal[int]");
assert_public_ty(&db, "/src/a.py", "x", "int");
Ok(())
}
@@ -2296,7 +2323,7 @@ mod tests {
)?;
// TODO should be a generic type
assert_public_ty(&db, "/src/a.py", "x", "Literal[tuple]");
assert_public_ty(&db, "/src/a.py", "x", "tuple");
Ok(())
}
@@ -2313,7 +2340,7 @@ mod tests {
)?;
// TODO should be a generic type
assert_public_ty(&db, "/src/a.py", "x", "Literal[list]");
assert_public_ty(&db, "/src/a.py", "x", "list");
Ok(())
}
@@ -2330,7 +2357,7 @@ mod tests {
)?;
// TODO should be a generic type
assert_public_ty(&db, "/src/a.py", "x", "Literal[set]");
assert_public_ty(&db, "/src/a.py", "x", "set");
Ok(())
}
@@ -2347,7 +2374,7 @@ mod tests {
)?;
// TODO should be a generic type
assert_public_ty(&db, "/src/a.py", "x", "Literal[dict]");
assert_public_ty(&db, "/src/a.py", "x", "dict");
Ok(())
}

Some files were not shown because too many files have changed in this diff Show More