Compare commits
30 Commits
cjm/record
...
red-knot-f
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
b93d3e6f21 | ||
|
|
523235d6ea | ||
|
|
414990c022 | ||
|
|
4779dd1173 | ||
|
|
c5adbf17da | ||
|
|
c6dcf3502b | ||
|
|
1e585b8667 | ||
|
|
21d824abfd | ||
|
|
7e28c80354 | ||
|
|
bc03d376e8 | ||
|
|
eb6f562419 | ||
|
|
5561d445d7 | ||
|
|
c391c8b6cb | ||
|
|
ce030a467f | ||
|
|
04a922866a | ||
|
|
0ed7af35ec | ||
|
|
87929ad5f1 | ||
|
|
8a887daeb4 | ||
|
|
7317d734be | ||
|
|
c1a2a60182 | ||
|
|
8e056b3a93 | ||
|
|
616dd1873f | ||
|
|
acfb1a83c9 | ||
|
|
7c0e32f255 | ||
|
|
4b84c55e3a | ||
|
|
4c8d33ec45 | ||
|
|
113e259e6d | ||
|
|
3474e37836 | ||
|
|
dfe90a3b2b | ||
|
|
00d7c01cfc |
16
.github/workflows/ci.yaml
vendored
16
.github/workflows/ci.yaml
vendored
@@ -194,6 +194,22 @@ jobs:
|
||||
cd crates/ruff_wasm
|
||||
wasm-pack test --node
|
||||
|
||||
cargo-build-release:
|
||||
name: "cargo build (release)"
|
||||
runs-on: macos-latest
|
||||
needs: determine_changes
|
||||
if: ${{ needs.determine_changes.outputs.code == 'true' || github.ref == 'refs/heads/main' }}
|
||||
timeout-minutes: 20
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: "Install Rust toolchain"
|
||||
run: rustup show
|
||||
- name: "Install mold"
|
||||
uses: rui314/setup-mold@v1
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
- name: "Build"
|
||||
run: cargo build --release --locked
|
||||
|
||||
cargo-fuzz:
|
||||
name: "cargo fuzz"
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
72
.github/workflows/daily_fuzz.yaml
vendored
Normal file
72
.github/workflows/daily_fuzz.yaml
vendored
Normal file
@@ -0,0 +1,72 @@
|
||||
name: Daily parser fuzz
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
schedule:
|
||||
- cron: "0 0 * * *"
|
||||
pull_request:
|
||||
paths:
|
||||
- ".github/workflows/daily_fuzz.yaml"
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
CARGO_INCREMENTAL: 0
|
||||
CARGO_NET_RETRY: 10
|
||||
CARGO_TERM_COLOR: always
|
||||
RUSTUP_MAX_RETRIES: 10
|
||||
PACKAGE_NAME: ruff
|
||||
FORCE_COLOR: 1
|
||||
|
||||
jobs:
|
||||
fuzz:
|
||||
name: Fuzz
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 20
|
||||
# Don't run the cron job on forks:
|
||||
if: ${{ github.repository == 'astral-sh/ruff' || github.event_name != 'schedule' }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.12"
|
||||
- name: Install uv
|
||||
run: curl -LsSf https://astral.sh/uv/install.sh | sh
|
||||
- name: Install Python requirements
|
||||
run: uv pip install -r scripts/fuzz-parser/requirements.txt --system
|
||||
- name: "Install Rust toolchain"
|
||||
run: rustup show
|
||||
- name: "Install mold"
|
||||
uses: rui314/setup-mold@v1
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
- name: Build ruff
|
||||
# A debug build means the script runs slower once it gets started,
|
||||
# but this is outweighed by the fact that a release build takes *much* longer to compile in CI
|
||||
run: cargo build --locked
|
||||
- name: Fuzz
|
||||
run: python scripts/fuzz-parser/fuzz.py $(shuf -i 0-9999999999999999999 -n 1000) --test-executable target/debug/ruff
|
||||
|
||||
create-issue-on-failure:
|
||||
name: Create an issue if the daily fuzz surfaced any bugs
|
||||
runs-on: ubuntu-latest
|
||||
needs: fuzz
|
||||
if: ${{ github.repository == 'astral-sh/ruff' && always() && github.event_name == 'schedule' && needs.fuzz.result == 'failure' }}
|
||||
permissions:
|
||||
issues: write
|
||||
steps:
|
||||
- uses: actions/github-script@v7
|
||||
with:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
script: |
|
||||
await github.rest.issues.create({
|
||||
owner: "astral-sh",
|
||||
repo: "ruff",
|
||||
title: `Daily parser fuzz failed on ${new Date().toDateString()}`,
|
||||
body: "Runs listed here: https://github.com/astral-sh/ruff/actions/workflows/daily_fuzz.yml",
|
||||
labels: ["bug", "parser", "fuzzer"],
|
||||
})
|
||||
@@ -41,7 +41,7 @@ repos:
|
||||
)$
|
||||
|
||||
- repo: https://github.com/crate-ci/typos
|
||||
rev: v1.20.9
|
||||
rev: v1.20.10
|
||||
hooks:
|
||||
- id: typos
|
||||
|
||||
@@ -55,7 +55,7 @@ repos:
|
||||
pass_filenames: false # This makes it a lot faster
|
||||
|
||||
- repo: https://github.com/astral-sh/ruff-pre-commit
|
||||
rev: v0.4.1
|
||||
rev: v0.4.2
|
||||
hooks:
|
||||
- id: ruff-format
|
||||
- id: ruff
|
||||
|
||||
131
Cargo.lock
generated
131
Cargo.lock
generated
@@ -353,7 +353,7 @@ dependencies = [
|
||||
"heck 0.5.0",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.60",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -501,6 +501,19 @@ dependencies = [
|
||||
"itertools 0.10.5",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam"
|
||||
version = "0.8.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1137cd7e7fc0fb5d3c5a8678be38ec56e819125d8d7907411fe24ccb943faca8"
|
||||
dependencies = [
|
||||
"crossbeam-channel",
|
||||
"crossbeam-deque",
|
||||
"crossbeam-epoch",
|
||||
"crossbeam-queue",
|
||||
"crossbeam-utils",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-channel"
|
||||
version = "0.5.12"
|
||||
@@ -529,6 +542,15 @@ dependencies = [
|
||||
"crossbeam-utils",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-queue"
|
||||
version = "0.3.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "df0346b5d5e76ac2fe4e327c5fd1118d6be7c51dfb18f9b7922923f287471e35"
|
||||
dependencies = [
|
||||
"crossbeam-utils",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-utils"
|
||||
version = "0.8.19"
|
||||
@@ -572,7 +594,7 @@ dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"strsim 0.10.0",
|
||||
"syn 2.0.60",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -583,7 +605,7 @@ checksum = "a668eda54683121533a393014d8692171709ff57a7d61f187b6e782719f8933f"
|
||||
dependencies = [
|
||||
"darling_core",
|
||||
"quote",
|
||||
"syn 2.0.60",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -593,7 +615,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "978747c1d849a7d2ee5e8adc0159961c48fb7e5db2f06af6723b80123bb53856"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"hashbrown 0.14.3",
|
||||
"hashbrown 0.14.5",
|
||||
"lock_api",
|
||||
"once_cell",
|
||||
"parking_lot_core",
|
||||
@@ -838,9 +860,9 @@ checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"
|
||||
|
||||
[[package]]
|
||||
name = "hashbrown"
|
||||
version = "0.14.3"
|
||||
version = "0.14.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604"
|
||||
checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1"
|
||||
dependencies = [
|
||||
"ahash",
|
||||
"allocator-api2",
|
||||
@@ -967,7 +989,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "168fb715dda47215e360912c096649d23d58bf392ac62f73919e831745e40f26"
|
||||
dependencies = [
|
||||
"equivalent",
|
||||
"hashbrown 0.14.3",
|
||||
"hashbrown 0.14.5",
|
||||
"serde",
|
||||
]
|
||||
|
||||
@@ -1065,7 +1087,7 @@ dependencies = [
|
||||
"Inflector",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.60",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1212,7 +1234,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4a5011f2d59093de14a4a90e01b9d85dee9276e58a25f0107dcee167dd601be0"
|
||||
dependencies = [
|
||||
"quote",
|
||||
"syn 2.0.60",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1305,9 +1327,9 @@ checksum = "2532096657941c2fea9c289d370a250971c689d4f143798ff67113ec042024a5"
|
||||
|
||||
[[package]]
|
||||
name = "matchit"
|
||||
version = "0.8.1"
|
||||
version = "0.8.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "357db4d45704af452edb5861033c1c28db6f583d2e34cc6d40c6e096eb111499"
|
||||
checksum = "540f1c43aed89909c0cc0cc604e3bb2f7e7a341a3728a9e6cfe760e733cd11ed"
|
||||
|
||||
[[package]]
|
||||
name = "memchr"
|
||||
@@ -1486,9 +1508,9 @@ checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39"
|
||||
|
||||
[[package]]
|
||||
name = "parking_lot"
|
||||
version = "0.12.1"
|
||||
version = "0.12.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f"
|
||||
checksum = "7e4af0ca4f6caed20e900d564c242b8e5d4903fdacf31d3daf527b66fe6f42fb"
|
||||
dependencies = [
|
||||
"lock_api",
|
||||
"parking_lot_core",
|
||||
@@ -1668,7 +1690,7 @@ checksum = "52a40bc70c2c58040d2d8b167ba9a5ff59fc9dab7ad44771cfde3dcfde7a09c6"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.60",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1804,18 +1826,20 @@ version = "0.1.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"bitflags 2.5.0",
|
||||
"crossbeam-channel",
|
||||
"crossbeam",
|
||||
"ctrlc",
|
||||
"dashmap",
|
||||
"hashbrown 0.14.3",
|
||||
"hashbrown 0.14.5",
|
||||
"indexmap",
|
||||
"log",
|
||||
"notify",
|
||||
"parking_lot",
|
||||
"rayon",
|
||||
"ruff_formatter",
|
||||
"ruff_index",
|
||||
"ruff_notebook",
|
||||
"ruff_python_ast",
|
||||
"ruff_python_formatter",
|
||||
"ruff_python_parser",
|
||||
"ruff_python_trivia",
|
||||
"ruff_text_size",
|
||||
@@ -1911,7 +1935,7 @@ dependencies = [
|
||||
"pmutil",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.60",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -2161,7 +2185,7 @@ dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"ruff_python_trivia",
|
||||
"syn 2.0.60",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -2341,7 +2365,7 @@ name = "ruff_server"
|
||||
version = "0.2.2"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"crossbeam-channel",
|
||||
"crossbeam",
|
||||
"insta",
|
||||
"jod-thread",
|
||||
"libc",
|
||||
@@ -2544,9 +2568,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "schemars"
|
||||
version = "0.8.16"
|
||||
version = "0.8.17"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "45a28f4c49489add4ce10783f7911893516f15afe45d015608d41faca6bc4d29"
|
||||
checksum = "7f55c82c700538496bdc329bb4918a81f87cc8888811bd123cf325a0f2f8d309"
|
||||
dependencies = [
|
||||
"dyn-clone",
|
||||
"schemars_derive",
|
||||
@@ -2556,14 +2580,14 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "schemars_derive"
|
||||
version = "0.8.16"
|
||||
version = "0.8.17"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c767fd6fa65d9ccf9cf026122c1b555f2ef9a4f0cea69da4d7dbc3e258d30967"
|
||||
checksum = "83263746fe5e32097f06356968a077f96089739c927a61450efa069905eec108"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"serde_derive_internals",
|
||||
"syn 1.0.109",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -2586,9 +2610,9 @@ checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b"
|
||||
|
||||
[[package]]
|
||||
name = "serde"
|
||||
version = "1.0.198"
|
||||
version = "1.0.199"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9846a40c979031340571da2545a4e5b7c4163bdae79b301d5f86d03979451fcc"
|
||||
checksum = "0c9f6e76df036c77cd94996771fb40db98187f096dd0b9af39c6c6e452ba966a"
|
||||
dependencies = [
|
||||
"serde_derive",
|
||||
]
|
||||
@@ -2606,24 +2630,24 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "serde_derive"
|
||||
version = "1.0.198"
|
||||
version = "1.0.199"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e88edab869b01783ba905e7d0153f9fc1a6505a96e4ad3018011eedb838566d9"
|
||||
checksum = "11bd257a6541e141e42ca6d24ae26f7714887b47e89aa739099104c7e4d3b7fc"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.60",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_derive_internals"
|
||||
version = "0.26.0"
|
||||
version = "0.29.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "85bf8229e7920a9f636479437026331ce11aa132b4dde37d121944a44d6e5f3c"
|
||||
checksum = "330f01ce65a3a5fe59a60c82f3c9a024b573b8a6e875bd233fe5f934e71d54e3"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 1.0.109",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -2645,7 +2669,7 @@ checksum = "6c64451ba24fc7a6a2d60fc75dd9c83c90903b19028d4eff35e88fc1e86564e9"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.60",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -2668,9 +2692,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "serde_with"
|
||||
version = "3.8.0"
|
||||
version = "3.8.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2c85f8e96d1d6857f13768fcbd895fcb06225510022a2774ed8b5150581847b0"
|
||||
checksum = "0ad483d2ab0149d5a5ebcd9972a3852711e0153d863bf5a5d0391d28883c4a20"
|
||||
dependencies = [
|
||||
"serde",
|
||||
"serde_derive",
|
||||
@@ -2679,14 +2703,14 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "serde_with_macros"
|
||||
version = "3.8.0"
|
||||
version = "3.8.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c8b3a576c4eb2924262d5951a3b737ccaf16c931e39a2810c36f9a7e25575557"
|
||||
checksum = "65569b702f41443e8bc8bbb1c5779bd0450bbe723b56198980e80ec45780bce2"
|
||||
dependencies = [
|
||||
"darling",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.60",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -2798,7 +2822,7 @@ dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"rustversion",
|
||||
"syn 2.0.60",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -2807,17 +2831,6 @@ version = "2.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc"
|
||||
|
||||
[[package]]
|
||||
name = "syn"
|
||||
version = "1.0.109"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "syn"
|
||||
version = "2.0.60"
|
||||
@@ -2882,7 +2895,7 @@ dependencies = [
|
||||
"cfg-if",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.60",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -2893,7 +2906,7 @@ checksum = "5c89e72a01ed4c579669add59014b9a524d609c0c88c6a585ce37485879f6ffb"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.60",
|
||||
"syn",
|
||||
"test-case-core",
|
||||
]
|
||||
|
||||
@@ -2925,7 +2938,7 @@ checksum = "d1cd413b5d558b4c5bf3680e324a6fa5014e7b7c067a51e69dbdf47eb7148b66"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.60",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -3037,7 +3050,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.60",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -3273,7 +3286,7 @@ checksum = "9881bea7cbe687e36c9ab3b778c36cd0487402e270304e8b1296d5085303c1a2"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.60",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -3358,7 +3371,7 @@ dependencies = [
|
||||
"once_cell",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.60",
|
||||
"syn",
|
||||
"wasm-bindgen-shared",
|
||||
]
|
||||
|
||||
@@ -3392,7 +3405,7 @@ checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.60",
|
||||
"syn",
|
||||
"wasm-bindgen-backend",
|
||||
"wasm-bindgen-shared",
|
||||
]
|
||||
@@ -3425,7 +3438,7 @@ checksum = "b7f89739351a2e03cb94beb799d47fb2cac01759b40ec441f7de39b00cbf7ef0"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.60",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -3694,7 +3707,7 @@ checksum = "9ce1b18ccd8e73a9321186f97e46f9f04b778851177567b1975109d26a08d2a6"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.60",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
||||
@@ -30,7 +30,7 @@ console_error_panic_hook = { version = "0.1.7" }
|
||||
console_log = { version = "1.0.0" }
|
||||
countme = { version = "3.0.1" }
|
||||
criterion = { version = "0.5.1", default-features = false }
|
||||
crossbeam-channel = { version = "0.5.12" }
|
||||
crossbeam = { version = "0.8.4" }
|
||||
dashmap = { version = "5.5.3" }
|
||||
dirs = { version = "5.0.0" }
|
||||
drop_bomb = { version = "0.1.5" }
|
||||
|
||||
@@ -12,17 +12,19 @@ license.workspace = true
|
||||
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
||||
|
||||
[dependencies]
|
||||
ruff_python_parser = { path = "../ruff_python_parser" }
|
||||
ruff_python_ast = { path = "../ruff_python_ast" }
|
||||
ruff_python_trivia = { path = "../ruff_python_trivia" }
|
||||
ruff_text_size = { path = "../ruff_text_size" }
|
||||
ruff_formatter = { path = "../ruff_formatter" }
|
||||
ruff_index = { path = "../ruff_index" }
|
||||
ruff_notebook = { path = "../ruff_notebook" }
|
||||
ruff_python_ast = { path = "../ruff_python_ast" }
|
||||
ruff_python_formatter = { path = "../ruff_python_formatter" }
|
||||
ruff_python_parser = { path = "../ruff_python_parser" }
|
||||
ruff_python_trivia = { path = "../ruff_python_trivia" }
|
||||
ruff_text_size = { path = "../ruff_text_size" }
|
||||
|
||||
anyhow = { workspace = true }
|
||||
bitflags = { workspace = true }
|
||||
ctrlc = "3.4.4"
|
||||
crossbeam-channel = { workspace = true }
|
||||
crossbeam = { workspace = true }
|
||||
dashmap = { workspace = true }
|
||||
hashbrown = { workspace = true }
|
||||
indexmap = { workspace = true }
|
||||
|
||||
@@ -2,6 +2,7 @@ use std::fmt::Formatter;
|
||||
use std::hash::Hash;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
|
||||
use crate::db::QueryResult;
|
||||
use dashmap::mapref::entry::Entry;
|
||||
|
||||
use crate::FxDashMap;
|
||||
@@ -27,11 +28,11 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
pub fn get<F>(&self, key: &K, compute: F) -> V
|
||||
pub fn get<F>(&self, key: &K, compute: F) -> QueryResult<V>
|
||||
where
|
||||
F: FnOnce(&K) -> V,
|
||||
F: FnOnce(&K) -> QueryResult<V>,
|
||||
{
|
||||
match self.map.entry(key.clone()) {
|
||||
Ok(match self.map.entry(key.clone()) {
|
||||
Entry::Occupied(cached) => {
|
||||
self.statistics.hit();
|
||||
|
||||
@@ -40,11 +41,11 @@ where
|
||||
Entry::Vacant(vacant) => {
|
||||
self.statistics.miss();
|
||||
|
||||
let value = compute(key);
|
||||
let value = compute(key)?;
|
||||
vacant.insert(value.clone());
|
||||
value
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
pub fn set(&mut self, key: K, value: V) {
|
||||
@@ -117,23 +118,29 @@ pub type CacheStatistics = DebugStatistics;
|
||||
#[cfg(not(debug_assertions))]
|
||||
pub type CacheStatistics = ReleaseStatistics;
|
||||
|
||||
pub trait StatisticsRecorder {
|
||||
fn hit(&self);
|
||||
fn miss(&self);
|
||||
fn to_statistics(&self) -> Option<Statistics>;
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
pub struct DebugStatistics {
|
||||
hits: AtomicUsize,
|
||||
misses: AtomicUsize,
|
||||
}
|
||||
|
||||
impl DebugStatistics {
|
||||
impl StatisticsRecorder for DebugStatistics {
|
||||
// TODO figure out appropriate Ordering
|
||||
pub fn hit(&self) {
|
||||
fn hit(&self) {
|
||||
self.hits.fetch_add(1, Ordering::SeqCst);
|
||||
}
|
||||
|
||||
pub fn miss(&self) {
|
||||
fn miss(&self) {
|
||||
self.misses.fetch_add(1, Ordering::SeqCst);
|
||||
}
|
||||
|
||||
pub fn to_statistics(&self) -> Option<Statistics> {
|
||||
fn to_statistics(&self) -> Option<Statistics> {
|
||||
let hits = self.hits.load(Ordering::SeqCst);
|
||||
let misses = self.misses.load(Ordering::SeqCst);
|
||||
|
||||
@@ -144,15 +151,15 @@ impl DebugStatistics {
|
||||
#[derive(Debug, Default)]
|
||||
pub struct ReleaseStatistics;
|
||||
|
||||
impl ReleaseStatistics {
|
||||
impl StatisticsRecorder for ReleaseStatistics {
|
||||
#[inline]
|
||||
pub const fn hit(&self) {}
|
||||
fn hit(&self) {}
|
||||
|
||||
#[inline]
|
||||
pub const fn miss(&self) {}
|
||||
fn miss(&self) {}
|
||||
|
||||
#[inline]
|
||||
pub const fn to_statistics(&self) -> Option<Statistics> {
|
||||
fn to_statistics(&self) -> Option<Statistics> {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,35 +1,25 @@
|
||||
use std::sync::{Arc, Condvar, Mutex};
|
||||
use std::sync::atomic::AtomicBool;
|
||||
use std::sync::Arc;
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct CancellationTokenSource {
|
||||
signal: Arc<(Mutex<bool>, Condvar)>,
|
||||
signal: Arc<AtomicBool>,
|
||||
}
|
||||
|
||||
impl CancellationTokenSource {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
signal: Arc::new((Mutex::new(false), Condvar::default())),
|
||||
signal: Arc::new(AtomicBool::new(false)),
|
||||
}
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "trace")]
|
||||
#[tracing::instrument(level = "trace", skip_all)]
|
||||
pub fn cancel(&self) {
|
||||
let (cancelled, condvar) = &*self.signal;
|
||||
|
||||
let mut cancelled = cancelled.lock().unwrap();
|
||||
|
||||
if *cancelled {
|
||||
return;
|
||||
}
|
||||
|
||||
*cancelled = true;
|
||||
condvar.notify_all();
|
||||
self.signal.store(true, std::sync::atomic::Ordering::SeqCst);
|
||||
}
|
||||
|
||||
pub fn is_cancelled(&self) -> bool {
|
||||
let (cancelled, _) = &*self.signal;
|
||||
|
||||
*cancelled.lock().unwrap()
|
||||
self.signal.load(std::sync::atomic::Ordering::SeqCst)
|
||||
}
|
||||
|
||||
pub fn token(&self) -> CancellationToken {
|
||||
@@ -41,26 +31,12 @@ impl CancellationTokenSource {
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct CancellationToken {
|
||||
signal: Arc<(Mutex<bool>, Condvar)>,
|
||||
signal: Arc<AtomicBool>,
|
||||
}
|
||||
|
||||
impl CancellationToken {
|
||||
/// Returns `true` if cancellation has been requested.
|
||||
pub fn is_cancelled(&self) -> bool {
|
||||
let (cancelled, _) = &*self.signal;
|
||||
|
||||
*cancelled.lock().unwrap()
|
||||
}
|
||||
|
||||
pub fn wait(&self) {
|
||||
let (bool, condvar) = &*self.signal;
|
||||
|
||||
let lock = condvar
|
||||
.wait_while(bool.lock().unwrap(), |bool| !*bool)
|
||||
.unwrap();
|
||||
|
||||
debug_assert!(*lock);
|
||||
|
||||
drop(lock);
|
||||
self.signal.load(std::sync::atomic::Ordering::SeqCst)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,53 +1,144 @@
|
||||
use std::path::Path;
|
||||
use std::sync::Arc;
|
||||
|
||||
pub use jars::{HasJar, HasJars};
|
||||
pub use query::{QueryError, QueryResult};
|
||||
pub use runtime::DbRuntime;
|
||||
pub use storage::JarsStorage;
|
||||
|
||||
use crate::files::FileId;
|
||||
use crate::lint::{Diagnostics, LintSyntaxStorage};
|
||||
use crate::lint::{Diagnostics, LintSemanticStorage, LintSyntaxStorage};
|
||||
use crate::module::{Module, ModuleData, ModuleName, ModuleResolver, ModuleSearchPath};
|
||||
use crate::parse::{Parsed, ParsedStorage};
|
||||
use crate::source::{Source, SourceStorage};
|
||||
use crate::symbols::{SymbolId, SymbolTable, SymbolTablesStorage};
|
||||
use crate::types::{Type, TypeStore};
|
||||
|
||||
pub trait SourceDb {
|
||||
mod jars;
|
||||
mod query;
|
||||
mod runtime;
|
||||
mod storage;
|
||||
|
||||
pub trait Database {
|
||||
/// Returns a reference to the runtime of the current worker.
|
||||
fn runtime(&self) -> &DbRuntime;
|
||||
|
||||
/// Returns a mutable reference to the runtime. Only one worker can hold a mutable reference to the runtime.
|
||||
fn runtime_mut(&mut self) -> &mut DbRuntime;
|
||||
|
||||
/// Returns `Ok` if the queries have not been cancelled and `Err(QueryError::Cancelled)` otherwise.
|
||||
fn cancelled(&self) -> QueryResult<()> {
|
||||
self.runtime().cancelled()
|
||||
}
|
||||
|
||||
/// Returns `true` if the queries have been cancelled.
|
||||
fn is_cancelled(&self) -> bool {
|
||||
self.runtime().is_cancelled()
|
||||
}
|
||||
}
|
||||
|
||||
/// Database that supports running queries from multiple threads.
|
||||
pub trait ParallelDatabase: Database + Send {
|
||||
/// Creates a snapshot of the database state that can be used to query the database in another thread.
|
||||
///
|
||||
/// The snapshot is a read-only view of the database but query results are shared between threads.
|
||||
/// All queries will be automatically cancelled when applying any mutations (calling [`HasJars::jars_mut`])
|
||||
/// to the database (not the snapshot, because they're readonly).
|
||||
///
|
||||
/// ## Creating a snapshot
|
||||
///
|
||||
/// Creating a snapshot of the database's jars is cheap but creating a snapshot of
|
||||
/// other state stored on the database might require deep-cloning data. That's why you should
|
||||
/// avoid creating snapshots in a hot function (e.g. don't create a snapshot for each file, instead
|
||||
/// create a snapshot when scheduling the check of an entire program).
|
||||
///
|
||||
/// ## Salsa compatibility
|
||||
/// Salsa prohibits creating a snapshot while running a local query (it's fine if other workers run a query) [[source](https://github.com/salsa-rs/salsa/issues/80)].
|
||||
/// We should avoid creating snapshots while running a query because we might want to adopt Salsa in the future (if we can figure out persistent caching).
|
||||
/// Unfortunately, the infrastructure doesn't provide an automated way of knowing when a query is run, that's
|
||||
/// why we have to "enforce" this constraint manually.
|
||||
fn snapshot(&self) -> Snapshot<Self>;
|
||||
}
|
||||
|
||||
/// Readonly snapshot of a database.
|
||||
///
|
||||
/// ## Dead locks
|
||||
/// A snapshot should always be dropped as soon as it is no longer necessary to run queries.
|
||||
/// Storing the snapshot without running a query or periodically checking if cancellation was requested
|
||||
/// can lead to deadlocks because mutating the [`Database`] requires cancels all pending queries
|
||||
/// and waiting for all [`Snapshot`]s to be dropped.
|
||||
#[derive(Debug)]
|
||||
pub struct Snapshot<DB: ?Sized>
|
||||
where
|
||||
DB: ParallelDatabase,
|
||||
{
|
||||
db: DB,
|
||||
}
|
||||
|
||||
impl<DB> Snapshot<DB>
|
||||
where
|
||||
DB: ParallelDatabase,
|
||||
{
|
||||
pub fn new(db: DB) -> Self {
|
||||
Snapshot { db }
|
||||
}
|
||||
}
|
||||
|
||||
impl<DB> std::ops::Deref for Snapshot<DB>
|
||||
where
|
||||
DB: ParallelDatabase,
|
||||
{
|
||||
type Target = DB;
|
||||
|
||||
fn deref(&self) -> &DB {
|
||||
&self.db
|
||||
}
|
||||
}
|
||||
|
||||
// Red knot specific databases code.
|
||||
|
||||
pub trait SourceDb: Database {
|
||||
// queries
|
||||
fn file_id(&self, path: &std::path::Path) -> FileId;
|
||||
|
||||
fn file_path(&self, file_id: FileId) -> Arc<std::path::Path>;
|
||||
|
||||
fn source(&self, file_id: FileId) -> Source;
|
||||
fn source(&self, file_id: FileId) -> QueryResult<Source>;
|
||||
|
||||
fn parse(&self, file_id: FileId) -> Parsed;
|
||||
|
||||
fn lint_syntax(&self, file_id: FileId) -> Diagnostics;
|
||||
fn parse(&self, file_id: FileId) -> QueryResult<Parsed>;
|
||||
}
|
||||
|
||||
pub trait SemanticDb: SourceDb {
|
||||
// queries
|
||||
fn resolve_module(&self, name: ModuleName) -> Option<Module>;
|
||||
fn resolve_module(&self, name: ModuleName) -> QueryResult<Option<Module>>;
|
||||
|
||||
fn file_to_module(&self, file_id: FileId) -> Option<Module>;
|
||||
fn file_to_module(&self, file_id: FileId) -> QueryResult<Option<Module>>;
|
||||
|
||||
fn path_to_module(&self, path: &Path) -> Option<Module>;
|
||||
fn path_to_module(&self, path: &Path) -> QueryResult<Option<Module>>;
|
||||
|
||||
fn symbol_table(&self, file_id: FileId) -> Arc<SymbolTable>;
|
||||
fn symbol_table(&self, file_id: FileId) -> QueryResult<Arc<SymbolTable>>;
|
||||
|
||||
fn infer_symbol_type(&self, file_id: FileId, symbol_id: SymbolId) -> QueryResult<Type>;
|
||||
|
||||
// mutations
|
||||
|
||||
fn add_module(&mut self, path: &Path) -> Option<(Module, Vec<Arc<ModuleData>>)>;
|
||||
|
||||
fn set_module_search_paths(&mut self, paths: Vec<ModuleSearchPath>);
|
||||
|
||||
fn infer_symbol_type(&mut self, file_id: FileId, symbol_id: SymbolId) -> Type;
|
||||
}
|
||||
|
||||
pub trait Db: SemanticDb {}
|
||||
pub trait LintDb: SemanticDb {
|
||||
fn lint_syntax(&self, file_id: FileId) -> QueryResult<Diagnostics>;
|
||||
|
||||
fn lint_semantic(&self, file_id: FileId) -> QueryResult<Diagnostics>;
|
||||
}
|
||||
|
||||
pub trait Db: LintDb {}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
pub struct SourceJar {
|
||||
pub sources: SourceStorage,
|
||||
pub parsed: ParsedStorage,
|
||||
pub lint_syntax: LintSyntaxStorage,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
@@ -57,31 +148,23 @@ pub struct SemanticJar {
|
||||
pub type_store: TypeStore,
|
||||
}
|
||||
|
||||
/// Gives access to a specific jar in the database.
|
||||
///
|
||||
/// Nope, the terminology isn't borrowed from Java but from Salsa <https://salsa-rs.github.io/salsa/>,
|
||||
/// which is an analogy to storing the salsa in different jars.
|
||||
///
|
||||
/// The basic idea is that each crate can define its own jar and the jars can be combined to a single
|
||||
/// database in the top level crate. Each crate also defines its own `Database` trait. The combination of
|
||||
/// `Database` trait and the jar allows to write queries in isolation without having to know how they get composed at the upper levels.
|
||||
///
|
||||
/// Salsa further defines a `HasIngredient` trait which slices the jar to a specific storage (e.g. a specific cache).
|
||||
/// We don't need this just jet because we write our queries by hand. We may want a similar trait if we decide
|
||||
/// to use a macro to generate the queries.
|
||||
pub trait HasJar<T> {
|
||||
/// Gives a read-only reference to the jar.
|
||||
fn jar(&self) -> &T;
|
||||
|
||||
/// Gives a mutable reference to the jar.
|
||||
fn jar_mut(&mut self) -> &mut T;
|
||||
#[derive(Debug, Default)]
|
||||
pub struct LintJar {
|
||||
pub lint_syntax: LintSyntaxStorage,
|
||||
pub lint_semantic: LintSemanticStorage,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) mod tests {
|
||||
use crate::db::{HasJar, SourceDb, SourceJar};
|
||||
use std::path::Path;
|
||||
use std::sync::Arc;
|
||||
|
||||
use crate::db::{
|
||||
Database, DbRuntime, HasJar, HasJars, JarsStorage, LintDb, LintJar, QueryResult, SourceDb,
|
||||
SourceJar,
|
||||
};
|
||||
use crate::files::{FileId, Files};
|
||||
use crate::lint::{lint_syntax, Diagnostics};
|
||||
use crate::lint::{lint_semantic, lint_syntax, Diagnostics};
|
||||
use crate::module::{
|
||||
add_module, file_to_module, path_to_module, resolve_module, set_module_search_paths,
|
||||
Module, ModuleData, ModuleName, ModuleSearchPath,
|
||||
@@ -90,8 +173,6 @@ pub(crate) mod tests {
|
||||
use crate::source::{source_text, Source};
|
||||
use crate::symbols::{symbol_table, SymbolId, SymbolTable};
|
||||
use crate::types::{infer_symbol_type, Type};
|
||||
use std::path::Path;
|
||||
use std::sync::Arc;
|
||||
|
||||
use super::{SemanticDb, SemanticJar};
|
||||
|
||||
@@ -100,27 +181,36 @@ pub(crate) mod tests {
|
||||
#[derive(Debug, Default)]
|
||||
pub(crate) struct TestDb {
|
||||
files: Files,
|
||||
source: SourceJar,
|
||||
semantic: SemanticJar,
|
||||
jars: JarsStorage<Self>,
|
||||
}
|
||||
|
||||
impl HasJar<SourceJar> for TestDb {
|
||||
fn jar(&self) -> &SourceJar {
|
||||
&self.source
|
||||
fn jar(&self) -> QueryResult<&SourceJar> {
|
||||
Ok(&self.jars()?.0)
|
||||
}
|
||||
|
||||
fn jar_mut(&mut self) -> &mut SourceJar {
|
||||
&mut self.source
|
||||
&mut self.jars_mut().0
|
||||
}
|
||||
}
|
||||
|
||||
impl HasJar<SemanticJar> for TestDb {
|
||||
fn jar(&self) -> &SemanticJar {
|
||||
&self.semantic
|
||||
fn jar(&self) -> QueryResult<&SemanticJar> {
|
||||
Ok(&self.jars()?.1)
|
||||
}
|
||||
|
||||
fn jar_mut(&mut self) -> &mut SemanticJar {
|
||||
&mut self.semantic
|
||||
&mut self.jars_mut().1
|
||||
}
|
||||
}
|
||||
|
||||
impl HasJar<LintJar> for TestDb {
|
||||
fn jar(&self) -> QueryResult<&LintJar> {
|
||||
Ok(&self.jars()?.2)
|
||||
}
|
||||
|
||||
fn jar_mut(&mut self) -> &mut LintJar {
|
||||
&mut self.jars_mut().2
|
||||
}
|
||||
}
|
||||
|
||||
@@ -133,36 +223,36 @@ pub(crate) mod tests {
|
||||
self.files.path(file_id)
|
||||
}
|
||||
|
||||
fn source(&self, file_id: FileId) -> Source {
|
||||
fn source(&self, file_id: FileId) -> QueryResult<Source> {
|
||||
source_text(self, file_id)
|
||||
}
|
||||
|
||||
fn parse(&self, file_id: FileId) -> Parsed {
|
||||
fn parse(&self, file_id: FileId) -> QueryResult<Parsed> {
|
||||
parse(self, file_id)
|
||||
}
|
||||
|
||||
fn lint_syntax(&self, file_id: FileId) -> Diagnostics {
|
||||
lint_syntax(self, file_id)
|
||||
}
|
||||
}
|
||||
|
||||
impl SemanticDb for TestDb {
|
||||
fn resolve_module(&self, name: ModuleName) -> Option<Module> {
|
||||
fn resolve_module(&self, name: ModuleName) -> QueryResult<Option<Module>> {
|
||||
resolve_module(self, name)
|
||||
}
|
||||
|
||||
fn file_to_module(&self, file_id: FileId) -> Option<Module> {
|
||||
fn file_to_module(&self, file_id: FileId) -> QueryResult<Option<Module>> {
|
||||
file_to_module(self, file_id)
|
||||
}
|
||||
|
||||
fn path_to_module(&self, path: &Path) -> Option<Module> {
|
||||
fn path_to_module(&self, path: &Path) -> QueryResult<Option<Module>> {
|
||||
path_to_module(self, path)
|
||||
}
|
||||
|
||||
fn symbol_table(&self, file_id: FileId) -> Arc<SymbolTable> {
|
||||
fn symbol_table(&self, file_id: FileId) -> QueryResult<Arc<SymbolTable>> {
|
||||
symbol_table(self, file_id)
|
||||
}
|
||||
|
||||
fn infer_symbol_type(&self, file_id: FileId, symbol_id: SymbolId) -> QueryResult<Type> {
|
||||
infer_symbol_type(self, file_id, symbol_id)
|
||||
}
|
||||
|
||||
fn add_module(&mut self, path: &Path) -> Option<(Module, Vec<Arc<ModuleData>>)> {
|
||||
add_module(self, path)
|
||||
}
|
||||
@@ -170,9 +260,37 @@ pub(crate) mod tests {
|
||||
fn set_module_search_paths(&mut self, paths: Vec<ModuleSearchPath>) {
|
||||
set_module_search_paths(self, paths);
|
||||
}
|
||||
}
|
||||
|
||||
fn infer_symbol_type(&mut self, file_id: FileId, symbol_id: SymbolId) -> Type {
|
||||
infer_symbol_type(self, file_id, symbol_id)
|
||||
impl LintDb for TestDb {
|
||||
fn lint_syntax(&self, file_id: FileId) -> QueryResult<Diagnostics> {
|
||||
lint_syntax(self, file_id)
|
||||
}
|
||||
|
||||
fn lint_semantic(&self, file_id: FileId) -> QueryResult<Diagnostics> {
|
||||
lint_semantic(self, file_id)
|
||||
}
|
||||
}
|
||||
|
||||
impl HasJars for TestDb {
|
||||
type Jars = (SourceJar, SemanticJar, LintJar);
|
||||
|
||||
fn jars(&self) -> QueryResult<&Self::Jars> {
|
||||
self.jars.jars()
|
||||
}
|
||||
|
||||
fn jars_mut(&mut self) -> &mut Self::Jars {
|
||||
self.jars.jars_mut()
|
||||
}
|
||||
}
|
||||
|
||||
impl Database for TestDb {
|
||||
fn runtime(&self) -> &DbRuntime {
|
||||
self.jars.runtime()
|
||||
}
|
||||
|
||||
fn runtime_mut(&mut self) -> &mut DbRuntime {
|
||||
self.jars.runtime_mut()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
37
crates/red_knot/src/db/jars.rs
Normal file
37
crates/red_knot/src/db/jars.rs
Normal file
@@ -0,0 +1,37 @@
|
||||
use crate::db::query::QueryResult;
|
||||
|
||||
/// Gives access to a specific jar in the database.
|
||||
///
|
||||
/// Nope, the terminology isn't borrowed from Java but from Salsa <https://salsa-rs.github.io/salsa/>,
|
||||
/// which is an analogy to storing the salsa in different jars.
|
||||
///
|
||||
/// The basic idea is that each crate can define its own jar and the jars can be combined to a single
|
||||
/// database in the top level crate. Each crate also defines its own `Database` trait. The combination of
|
||||
/// `Database` trait and the jar allows to write queries in isolation without having to know how they get composed at the upper levels.
|
||||
///
|
||||
/// Salsa further defines a `HasIngredient` trait which slices the jar to a specific storage (e.g. a specific cache).
|
||||
/// We don't need this just jet because we write our queries by hand. We may want a similar trait if we decide
|
||||
/// to use a macro to generate the queries.
|
||||
pub trait HasJar<T> {
|
||||
/// Gives a read-only reference to the jar.
|
||||
fn jar(&self) -> QueryResult<&T>;
|
||||
|
||||
/// Gives a mutable reference to the jar.
|
||||
fn jar_mut(&mut self) -> &mut T;
|
||||
}
|
||||
|
||||
/// Gives access to the jars in a database.
|
||||
pub trait HasJars {
|
||||
/// A type storing the jars.
|
||||
///
|
||||
/// Most commonly, this is a tuple where each jar is a tuple element.
|
||||
type Jars: Default;
|
||||
|
||||
/// Gives access to the underlying jars but tests if the queries have been cancelled.
|
||||
///
|
||||
/// Returns `Err(QueryError::Cancelled)` if the queries have been cancelled.
|
||||
fn jars(&self) -> QueryResult<&Self::Jars>;
|
||||
|
||||
/// Gives mutable access to the underlying jars.
|
||||
fn jars_mut(&mut self) -> &mut Self::Jars;
|
||||
}
|
||||
20
crates/red_knot/src/db/query.rs
Normal file
20
crates/red_knot/src/db/query.rs
Normal file
@@ -0,0 +1,20 @@
|
||||
use std::fmt::{Display, Formatter};
|
||||
|
||||
/// Reason why a db query operation failed.
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub enum QueryError {
|
||||
/// The query was cancelled because the DB was mutated or the query was cancelled by the host (e.g. on a file change or when pressing CTRL+C).
|
||||
Cancelled,
|
||||
}
|
||||
|
||||
impl Display for QueryError {
|
||||
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
|
||||
match self {
|
||||
QueryError::Cancelled => f.write_str("query was cancelled"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for QueryError {}
|
||||
|
||||
pub type QueryResult<T> = Result<T, QueryError>;
|
||||
41
crates/red_knot/src/db/runtime.rs
Normal file
41
crates/red_knot/src/db/runtime.rs
Normal file
@@ -0,0 +1,41 @@
|
||||
use crate::cancellation::CancellationTokenSource;
|
||||
use crate::db::{QueryError, QueryResult};
|
||||
|
||||
/// Holds the jar agnostic state of the database.
|
||||
#[derive(Debug, Default)]
|
||||
pub struct DbRuntime {
|
||||
/// The cancellation token source used to signal other works that the queries should be aborted and
|
||||
/// exit at the next possible point.
|
||||
cancellation_token: CancellationTokenSource,
|
||||
}
|
||||
|
||||
impl DbRuntime {
|
||||
pub(super) fn snapshot(&self) -> Self {
|
||||
Self {
|
||||
cancellation_token: self.cancellation_token.clone(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Cancels the pending queries of other workers. The current worker cannot have any pending
|
||||
/// queries because we're holding a mutable reference to the runtime.
|
||||
pub(super) fn cancel_other_workers(&mut self) {
|
||||
self.cancellation_token.cancel();
|
||||
// Set a new cancellation token so that we're in a non-cancelled state again when running the next
|
||||
// query.
|
||||
self.cancellation_token = CancellationTokenSource::default();
|
||||
}
|
||||
|
||||
/// Returns `Ok` if the queries have not been cancelled and `Err(QueryError::Cancelled)` otherwise.
|
||||
pub(super) fn cancelled(&self) -> QueryResult<()> {
|
||||
if self.cancellation_token.is_cancelled() {
|
||||
Err(QueryError::Cancelled)
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns `true` if the queries have been cancelled.
|
||||
pub(super) fn is_cancelled(&self) -> bool {
|
||||
self.cancellation_token.is_cancelled()
|
||||
}
|
||||
}
|
||||
117
crates/red_knot/src/db/storage.rs
Normal file
117
crates/red_knot/src/db/storage.rs
Normal file
@@ -0,0 +1,117 @@
|
||||
use std::fmt::Formatter;
|
||||
use std::sync::Arc;
|
||||
|
||||
use crossbeam::sync::WaitGroup;
|
||||
|
||||
use crate::db::query::QueryResult;
|
||||
use crate::db::runtime::DbRuntime;
|
||||
use crate::db::{HasJars, ParallelDatabase};
|
||||
|
||||
/// Stores the jars of a database and the state for each worker.
|
||||
///
|
||||
/// Today, all state is shared across all workers, but it may be desired to store data per worker in the future.
|
||||
pub struct JarsStorage<T>
|
||||
where
|
||||
T: HasJars + Sized,
|
||||
{
|
||||
// It's important that `jars_wait_group` is declared after `jars` to ensure that `jars` is dropped first.
|
||||
// See https://doc.rust-lang.org/reference/destructors.html
|
||||
/// Stores the jars of the database.
|
||||
jars: Arc<T::Jars>,
|
||||
|
||||
/// Used to count the references to `jars`. Allows implementing `jars_mut` without requiring to clone `jars`.
|
||||
jars_wait_group: WaitGroup,
|
||||
|
||||
/// The data agnostic state.
|
||||
runtime: DbRuntime,
|
||||
}
|
||||
|
||||
impl<Db> JarsStorage<Db>
|
||||
where
|
||||
Db: HasJars,
|
||||
{
|
||||
pub(super) fn new() -> Self {
|
||||
Self {
|
||||
jars: Arc::new(Db::Jars::default()),
|
||||
jars_wait_group: WaitGroup::default(),
|
||||
runtime: DbRuntime::default(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Creates a snapshot of the jars.
|
||||
///
|
||||
/// Creating the snapshot is cheap because it doesn't clone the jars, it only increments a ref counter.
|
||||
#[must_use]
|
||||
pub fn snapshot(&self) -> JarsStorage<Db>
|
||||
where
|
||||
Db: ParallelDatabase,
|
||||
{
|
||||
Self {
|
||||
jars: self.jars.clone(),
|
||||
jars_wait_group: self.jars_wait_group.clone(),
|
||||
runtime: self.runtime.snapshot(),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn jars(&self) -> QueryResult<&Db::Jars> {
|
||||
self.runtime.cancelled()?;
|
||||
Ok(&self.jars)
|
||||
}
|
||||
|
||||
/// Returns a mutable reference to the jars without cloning their content.
|
||||
///
|
||||
/// The method cancels any pending queries of other works and waits for them to complete so that
|
||||
/// this instance is the only instance holding a reference to the jars.
|
||||
pub(crate) fn jars_mut(&mut self) -> &mut Db::Jars {
|
||||
// We have a mutable ref here, so no more workers can be spawned between calling this function and taking the mut ref below.
|
||||
self.cancel_other_workers();
|
||||
|
||||
// Now all other references to `self.jars` should have been released. We can now safely return a mutable reference
|
||||
// to the Arc's content.
|
||||
let jars =
|
||||
Arc::get_mut(&mut self.jars).expect("All references to jars should have been released");
|
||||
|
||||
jars
|
||||
}
|
||||
|
||||
pub(crate) fn runtime(&self) -> &DbRuntime {
|
||||
&self.runtime
|
||||
}
|
||||
|
||||
pub(crate) fn runtime_mut(&mut self) -> &mut DbRuntime {
|
||||
// Note: This method may need to use a similar trick to `jars_mut` if `DbRuntime` is ever to store data that is shared between workers.
|
||||
&mut self.runtime
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "trace", skip(self))]
|
||||
fn cancel_other_workers(&mut self) {
|
||||
self.runtime.cancel_other_workers();
|
||||
|
||||
// Wait for all other works to complete.
|
||||
let existing_wait = std::mem::take(&mut self.jars_wait_group);
|
||||
existing_wait.wait();
|
||||
}
|
||||
}
|
||||
|
||||
impl<Db> Default for JarsStorage<Db>
|
||||
where
|
||||
Db: HasJars,
|
||||
{
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
impl<T> std::fmt::Debug for JarsStorage<T>
|
||||
where
|
||||
T: HasJars,
|
||||
<T as HasJars>::Jars: std::fmt::Debug,
|
||||
{
|
||||
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
|
||||
f.debug_struct("SharedStorage")
|
||||
.field("jars", &self.jars)
|
||||
.field("jars_wait_group", &self.jars_wait_group)
|
||||
.field("runtime", &self.runtime)
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
135
crates/red_knot/src/format.rs
Normal file
135
crates/red_knot/src/format.rs
Normal file
@@ -0,0 +1,135 @@
|
||||
use std::ops::{Deref, DerefMut};
|
||||
|
||||
use ruff_formatter::PrintedRange;
|
||||
use ruff_python_formatter::{FormatModuleError, PyFormatOptions};
|
||||
use ruff_text_size::TextRange;
|
||||
|
||||
use crate::cache::KeyValueCache;
|
||||
use crate::db::{HasJar, QueryError, SourceDb};
|
||||
use crate::files::FileId;
|
||||
use crate::lint::Diagnostics;
|
||||
use crate::FxDashSet;
|
||||
|
||||
pub(crate) trait FormatDb: SourceDb {
|
||||
/// Formats a file and returns its formatted content or an indicator that it is unchanged.
|
||||
fn format_file(&self, file_id: FileId) -> Result<FormattedFile, FormatError>;
|
||||
|
||||
/// Formats a range in a file.
|
||||
fn format_file_range(
|
||||
&self,
|
||||
file_id: FileId,
|
||||
range: TextRange,
|
||||
) -> Result<PrintedRange, FormatError>;
|
||||
|
||||
fn check_file_formatted(&self, file_id: FileId) -> Result<Diagnostics, FormatError>;
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "trace", skip(db))]
|
||||
pub(crate) fn format_file<Db>(db: &Db, file_id: FileId) -> Result<FormattedFile, FormatError>
|
||||
where
|
||||
Db: FormatDb + HasJar<FormatJar>,
|
||||
{
|
||||
let formatted = &db.jar()?.formatted;
|
||||
|
||||
if formatted.contains(&file_id) {
|
||||
return Ok(FormattedFile::Unchanged);
|
||||
}
|
||||
|
||||
let source = db.source(file_id)?;
|
||||
|
||||
// TODO use the `format_module` method here to re-use the AST.
|
||||
let printed =
|
||||
ruff_python_formatter::format_module_source(source.text(), PyFormatOptions::default())?;
|
||||
|
||||
Ok(if printed.as_code() == source.text() {
|
||||
formatted.insert(file_id);
|
||||
FormattedFile::Unchanged
|
||||
} else {
|
||||
FormattedFile::Formatted(printed.into_code())
|
||||
})
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "trace", skip(db))]
|
||||
pub(crate) fn format_file_range<Db: FormatDb + HasJar<FormatJar>>(
|
||||
db: &Db,
|
||||
file_id: FileId,
|
||||
range: TextRange,
|
||||
) -> Result<PrintedRange, FormatError> {
|
||||
let formatted = &db.jar()?.formatted;
|
||||
let source = db.source(file_id)?;
|
||||
|
||||
if formatted.contains(&file_id) {
|
||||
return Ok(PrintedRange::new(source.text()[range].into(), range));
|
||||
}
|
||||
|
||||
// TODO use the `format_module` method here to re-use the AST.
|
||||
|
||||
let result =
|
||||
ruff_python_formatter::format_range(source.text(), range, PyFormatOptions::default())?;
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
/// Checks if the file is correctly formatted. It creates a diagnostic for formatting issues.
|
||||
#[tracing::instrument(level = "trace", skip(db))]
|
||||
pub(crate) fn check_formatted<Db>(db: &Db, file_id: FileId) -> Result<Diagnostics, FormatError>
|
||||
where
|
||||
Db: FormatDb + HasJar<FormatJar>,
|
||||
{
|
||||
Ok(if db.format_file(file_id)?.is_unchanged() {
|
||||
Diagnostics::Empty
|
||||
} else {
|
||||
Diagnostics::from(vec!["File is not formatted".to_string()])
|
||||
})
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub(crate) enum FormatError {
|
||||
Format(FormatModuleError),
|
||||
Query(QueryError),
|
||||
}
|
||||
|
||||
impl From<FormatModuleError> for FormatError {
|
||||
fn from(value: FormatModuleError) -> Self {
|
||||
Self::Format(value)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<QueryError> for FormatError {
|
||||
fn from(value: QueryError) -> Self {
|
||||
Self::Query(value)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Eq, PartialEq, Debug)]
|
||||
pub(crate) enum FormattedFile {
|
||||
Formatted(String),
|
||||
Unchanged,
|
||||
}
|
||||
|
||||
impl FormattedFile {
|
||||
pub(crate) const fn is_unchanged(&self) -> bool {
|
||||
matches!(self, FormattedFile::Unchanged)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
pub struct FormatJar {
|
||||
pub formatted: FxDashSet<FileId>,
|
||||
}
|
||||
|
||||
#[derive(Default, Debug)]
|
||||
pub(crate) struct FormattedStorage(KeyValueCache<FileId, ()>);
|
||||
|
||||
impl Deref for FormattedStorage {
|
||||
type Target = KeyValueCache<FileId, ()>;
|
||||
|
||||
fn deref(&self) -> &Self::Target {
|
||||
&self.0
|
||||
}
|
||||
}
|
||||
|
||||
impl DerefMut for FormattedStorage {
|
||||
fn deref_mut(&mut self) -> &mut Self::Target {
|
||||
&mut self.0
|
||||
}
|
||||
}
|
||||
@@ -1,3 +1,4 @@
|
||||
use std::fmt::Formatter;
|
||||
use std::hash::BuildHasherDefault;
|
||||
use std::ops::Deref;
|
||||
use std::path::{Path, PathBuf};
|
||||
@@ -11,6 +12,7 @@ pub mod cache;
|
||||
pub mod cancellation;
|
||||
pub mod db;
|
||||
pub mod files;
|
||||
mod format;
|
||||
pub mod hir;
|
||||
pub mod lint;
|
||||
pub mod module;
|
||||
@@ -26,7 +28,7 @@ pub(crate) type FxDashMap<K, V> = dashmap::DashMap<K, V, BuildHasherDefault<FxHa
|
||||
pub(crate) type FxDashSet<V> = dashmap::DashSet<V, BuildHasherDefault<FxHasher>>;
|
||||
pub(crate) type FxIndexSet<V> = indexmap::set::IndexSet<V, BuildHasherDefault<FxHasher>>;
|
||||
|
||||
#[derive(Debug)]
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Workspace {
|
||||
/// TODO this should be a resolved path. We should probably use a newtype wrapper that guarantees that
|
||||
/// PATH is a UTF-8 path and is normalized.
|
||||
@@ -100,3 +102,9 @@ where
|
||||
Self(value.into())
|
||||
}
|
||||
}
|
||||
|
||||
impl std::fmt::Display for Name {
|
||||
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
|
||||
f.write_str(self.as_str())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,27 +1,42 @@
|
||||
use std::cell::RefCell;
|
||||
use std::ops::{Deref, DerefMut};
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use ruff_python_ast::visitor::Visitor;
|
||||
use ruff_python_ast::StringLiteral;
|
||||
use ruff_python_ast::{ModModule, StringLiteral};
|
||||
|
||||
use crate::cache::KeyValueCache;
|
||||
use crate::db::{HasJar, SourceDb, SourceJar};
|
||||
use crate::db::{HasJar, LintDb, LintJar, QueryResult, SemanticDb};
|
||||
use crate::files::FileId;
|
||||
use crate::parse::Parsed;
|
||||
use crate::source::Source;
|
||||
use crate::symbols::{Definition, SymbolId, SymbolTable};
|
||||
use crate::types::Type;
|
||||
|
||||
#[tracing::instrument(level = "debug", skip(db))]
|
||||
pub(crate) fn lint_syntax<Db>(db: &Db, file_id: FileId) -> Diagnostics
|
||||
pub(crate) fn lint_syntax<Db>(db: &Db, file_id: FileId) -> QueryResult<Diagnostics>
|
||||
where
|
||||
Db: SourceDb + HasJar<SourceJar>,
|
||||
Db: LintDb + HasJar<LintJar>,
|
||||
{
|
||||
let storage = &db.jar().lint_syntax;
|
||||
let storage = &db.jar()?.lint_syntax;
|
||||
|
||||
#[allow(clippy::print_stdout)]
|
||||
if std::env::var("RED_KNOT_SLOW_LINT").is_ok() {
|
||||
for i in 0..10 {
|
||||
db.cancelled()?;
|
||||
println!("RED_KNOT_SLOW_LINT is set, sleeping for {i}/10 seconds");
|
||||
std::thread::sleep(Duration::from_secs(1));
|
||||
}
|
||||
}
|
||||
|
||||
storage.get(&file_id, |file_id| {
|
||||
let mut diagnostics = Vec::new();
|
||||
|
||||
let source = db.source(*file_id);
|
||||
let source = db.source(*file_id)?;
|
||||
lint_lines(source.text(), &mut diagnostics);
|
||||
|
||||
let parsed = db.parse(*file_id);
|
||||
let parsed = db.parse(*file_id)?;
|
||||
|
||||
if parsed.errors().is_empty() {
|
||||
let ast = parsed.ast();
|
||||
@@ -36,11 +51,11 @@ where
|
||||
diagnostics.extend(parsed.errors().iter().map(std::string::ToString::to_string));
|
||||
}
|
||||
|
||||
Diagnostics::from(diagnostics)
|
||||
Ok(Diagnostics::from(diagnostics))
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn lint_lines(source: &str, diagnostics: &mut Vec<String>) {
|
||||
fn lint_lines(source: &str, diagnostics: &mut Vec<String>) {
|
||||
for (line_number, line) in source.lines().enumerate() {
|
||||
if line.len() < 88 {
|
||||
continue;
|
||||
@@ -57,6 +72,113 @@ pub(crate) fn lint_lines(source: &str, diagnostics: &mut Vec<String>) {
|
||||
}
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "debug", skip(db))]
|
||||
pub(crate) fn lint_semantic<Db>(db: &Db, file_id: FileId) -> QueryResult<Diagnostics>
|
||||
where
|
||||
Db: LintDb + HasJar<LintJar>,
|
||||
{
|
||||
let storage = &db.jar()?.lint_semantic;
|
||||
|
||||
storage.get(&file_id, |file_id| {
|
||||
let source = db.source(*file_id)?;
|
||||
let parsed = db.parse(*file_id)?;
|
||||
let symbols = db.symbol_table(*file_id)?;
|
||||
|
||||
let context = SemanticLintContext {
|
||||
file_id: *file_id,
|
||||
source,
|
||||
parsed,
|
||||
symbols,
|
||||
db,
|
||||
diagnostics: RefCell::new(Vec::new()),
|
||||
};
|
||||
|
||||
lint_unresolved_imports(&context)?;
|
||||
|
||||
Ok(Diagnostics::from(context.diagnostics.take()))
|
||||
})
|
||||
}
|
||||
|
||||
fn lint_unresolved_imports(context: &SemanticLintContext) -> QueryResult<()> {
|
||||
// TODO: Consider iterating over the dependencies (imports) only instead of all definitions.
|
||||
for (symbol, definition) in context.symbols().all_definitions() {
|
||||
match definition {
|
||||
Definition::Import(import) => {
|
||||
let ty = context.infer_symbol_type(symbol)?;
|
||||
|
||||
if ty.is_unknown() {
|
||||
context.push_diagnostic(format!("Unresolved module {}", import.module));
|
||||
}
|
||||
}
|
||||
Definition::ImportFrom(import) => {
|
||||
let ty = context.infer_symbol_type(symbol)?;
|
||||
|
||||
if ty.is_unknown() {
|
||||
let module_name = import.module().map(Deref::deref).unwrap_or_default();
|
||||
let message = if import.level() > 0 {
|
||||
format!(
|
||||
"Unresolved relative import '{}' from {}{}",
|
||||
import.name(),
|
||||
".".repeat(import.level() as usize),
|
||||
module_name
|
||||
)
|
||||
} else {
|
||||
format!(
|
||||
"Unresolved import '{}' from '{}'",
|
||||
import.name(),
|
||||
module_name
|
||||
)
|
||||
};
|
||||
|
||||
context.push_diagnostic(message);
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub struct SemanticLintContext<'a> {
|
||||
file_id: FileId,
|
||||
source: Source,
|
||||
parsed: Parsed,
|
||||
symbols: Arc<SymbolTable>,
|
||||
db: &'a dyn SemanticDb,
|
||||
diagnostics: RefCell<Vec<String>>,
|
||||
}
|
||||
|
||||
impl<'a> SemanticLintContext<'a> {
|
||||
pub fn source_text(&self) -> &str {
|
||||
self.source.text()
|
||||
}
|
||||
|
||||
pub fn file_id(&self) -> FileId {
|
||||
self.file_id
|
||||
}
|
||||
|
||||
pub fn ast(&self) -> &ModModule {
|
||||
self.parsed.ast()
|
||||
}
|
||||
|
||||
pub fn symbols(&self) -> &SymbolTable {
|
||||
&self.symbols
|
||||
}
|
||||
|
||||
pub fn infer_symbol_type(&self, symbol_id: SymbolId) -> QueryResult<Type> {
|
||||
self.db.infer_symbol_type(self.file_id, symbol_id)
|
||||
}
|
||||
|
||||
pub fn push_diagnostic(&self, diagnostic: String) {
|
||||
self.diagnostics.borrow_mut().push(diagnostic);
|
||||
}
|
||||
|
||||
pub fn extend_diagnostics(&mut self, diagnostics: impl IntoIterator<Item = String>) {
|
||||
self.diagnostics.get_mut().extend(diagnostics);
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct SyntaxLintVisitor<'a> {
|
||||
diagnostics: Vec<String>,
|
||||
@@ -123,3 +245,20 @@ impl DerefMut for LintSyntaxStorage {
|
||||
&mut self.0
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default, Debug)]
|
||||
pub struct LintSemanticStorage(KeyValueCache<FileId, Diagnostics>);
|
||||
|
||||
impl Deref for LintSemanticStorage {
|
||||
type Target = KeyValueCache<FileId, Diagnostics>;
|
||||
|
||||
fn deref(&self) -> &Self::Target {
|
||||
&self.0
|
||||
}
|
||||
}
|
||||
|
||||
impl DerefMut for LintSemanticStorage {
|
||||
fn deref_mut(&mut self) -> &mut Self::Target {
|
||||
&mut self.0
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,6 +4,7 @@ use std::collections::hash_map::Entry;
|
||||
use std::path::Path;
|
||||
use std::sync::Mutex;
|
||||
|
||||
use crossbeam::channel as crossbeam_channel;
|
||||
use rustc_hash::FxHashMap;
|
||||
use tracing::subscriber::Interest;
|
||||
use tracing::{Level, Metadata};
|
||||
@@ -12,11 +13,12 @@ use tracing_subscriber::layer::{Context, Filter, SubscriberExt};
|
||||
use tracing_subscriber::{Layer, Registry};
|
||||
use tracing_tree::time::Uptime;
|
||||
|
||||
use red_knot::cancellation::CancellationTokenSource;
|
||||
use red_knot::db::{HasJar, SourceDb, SourceJar};
|
||||
use red_knot::db::{
|
||||
Database, HasJar, ParallelDatabase, QueryError, SemanticDb, SourceDb, SourceJar,
|
||||
};
|
||||
use red_knot::files::FileId;
|
||||
use red_knot::module::{ModuleSearchPath, ModuleSearchPathKind};
|
||||
use red_knot::program::check::{CheckError, RayonCheckScheduler};
|
||||
use red_knot::program::check::ExecutionMode;
|
||||
use red_knot::program::{FileChange, FileChangeKind, Program};
|
||||
use red_knot::watch::FileWatcher;
|
||||
use red_knot::Workspace;
|
||||
@@ -51,7 +53,8 @@ fn main() -> anyhow::Result<()> {
|
||||
workspace.root().to_path_buf(),
|
||||
ModuleSearchPathKind::FirstParty,
|
||||
);
|
||||
let mut program = Program::new(workspace, vec![workspace_search_path]);
|
||||
let mut program = Program::new(workspace);
|
||||
program.set_module_search_paths(vec![workspace_search_path]);
|
||||
|
||||
let entry_id = program.file_id(entry_point);
|
||||
program.workspace_mut().open_file(entry_id);
|
||||
@@ -82,7 +85,7 @@ fn main() -> anyhow::Result<()> {
|
||||
|
||||
main_loop.run(&mut program);
|
||||
|
||||
let source_jar: &SourceJar = program.jar();
|
||||
let source_jar: &SourceJar = program.jar().unwrap();
|
||||
|
||||
dbg!(source_jar.parsed.statistics());
|
||||
dbg!(source_jar.sources.statistics());
|
||||
@@ -101,10 +104,9 @@ impl MainLoop {
|
||||
let (main_loop_sender, main_loop_receiver) = crossbeam_channel::bounded(1);
|
||||
|
||||
let mut orchestrator = Orchestrator {
|
||||
pending_analysis: None,
|
||||
receiver: orchestrator_receiver,
|
||||
sender: main_loop_sender.clone(),
|
||||
aggregated_changes: AggregatedChanges::default(),
|
||||
revision: 0,
|
||||
};
|
||||
|
||||
std::thread::spawn(move || {
|
||||
@@ -137,34 +139,32 @@ impl MainLoop {
|
||||
tracing::trace!("Main Loop: Tick");
|
||||
|
||||
match message {
|
||||
MainLoopMessage::CheckProgram => {
|
||||
// Remove mutability from program.
|
||||
let program = &*program;
|
||||
let run_cancellation_token_source = CancellationTokenSource::new();
|
||||
let run_cancellation_token = run_cancellation_token_source.token();
|
||||
let sender = &self.orchestrator_sender;
|
||||
MainLoopMessage::CheckProgram { revision } => {
|
||||
{
|
||||
let program = program.snapshot();
|
||||
let sender = self.orchestrator_sender.clone();
|
||||
|
||||
sender
|
||||
.send(OrchestratorMessage::CheckProgramStarted {
|
||||
cancellation_token: run_cancellation_token_source,
|
||||
})
|
||||
.unwrap();
|
||||
// Spawn a new task that checks the program. This needs to be done in a separate thread
|
||||
// to prevent blocking the main loop here.
|
||||
rayon::spawn(move || match program.check(ExecutionMode::ThreadPool) {
|
||||
Ok(result) => {
|
||||
sender
|
||||
.send(OrchestratorMessage::CheckProgramCompleted {
|
||||
diagnostics: result,
|
||||
revision,
|
||||
})
|
||||
.unwrap();
|
||||
}
|
||||
Err(QueryError::Cancelled) => {}
|
||||
});
|
||||
}
|
||||
|
||||
rayon::in_place_scope(|scope| {
|
||||
let scheduler = RayonCheckScheduler::new(program, scope);
|
||||
|
||||
let result = program.check(&scheduler, run_cancellation_token);
|
||||
match result {
|
||||
Ok(result) => sender
|
||||
.send(OrchestratorMessage::CheckProgramCompleted(result))
|
||||
.unwrap(),
|
||||
Err(CheckError::Cancelled) => sender
|
||||
.send(OrchestratorMessage::CheckProgramCancelled)
|
||||
.unwrap(),
|
||||
}
|
||||
});
|
||||
if !program.is_cancelled() {
|
||||
let _ = program.format();
|
||||
}
|
||||
}
|
||||
MainLoopMessage::ApplyChanges(changes) => {
|
||||
// Automatically cancels any pending queries and waits for them to complete.
|
||||
program.apply_changes(changes.iter());
|
||||
}
|
||||
MainLoopMessage::CheckCompleted(diagnostics) => {
|
||||
@@ -211,13 +211,11 @@ impl MainLoopCancellationToken {
|
||||
}
|
||||
|
||||
struct Orchestrator {
|
||||
aggregated_changes: AggregatedChanges,
|
||||
pending_analysis: Option<PendingAnalysisState>,
|
||||
|
||||
/// Sends messages to the main loop.
|
||||
sender: crossbeam_channel::Sender<MainLoopMessage>,
|
||||
/// Receives messages from the main loop.
|
||||
receiver: crossbeam_channel::Receiver<OrchestratorMessage>,
|
||||
revision: usize,
|
||||
}
|
||||
|
||||
impl Orchestrator {
|
||||
@@ -225,51 +223,33 @@ impl Orchestrator {
|
||||
while let Ok(message) = self.receiver.recv() {
|
||||
match message {
|
||||
OrchestratorMessage::Run => {
|
||||
self.pending_analysis = None;
|
||||
self.sender.send(MainLoopMessage::CheckProgram).unwrap();
|
||||
}
|
||||
|
||||
OrchestratorMessage::CheckProgramStarted { cancellation_token } => {
|
||||
debug_assert!(self.pending_analysis.is_none());
|
||||
|
||||
self.pending_analysis = Some(PendingAnalysisState { cancellation_token });
|
||||
}
|
||||
|
||||
OrchestratorMessage::CheckProgramCompleted(diagnostics) => {
|
||||
self.pending_analysis
|
||||
.take()
|
||||
.expect("Expected a pending analysis.");
|
||||
|
||||
self.sender
|
||||
.send(MainLoopMessage::CheckCompleted(diagnostics))
|
||||
.send(MainLoopMessage::CheckProgram {
|
||||
revision: self.revision,
|
||||
})
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
OrchestratorMessage::CheckProgramCancelled => {
|
||||
self.pending_analysis
|
||||
.take()
|
||||
.expect("Expected a pending analysis.");
|
||||
|
||||
self.debounce_changes();
|
||||
OrchestratorMessage::CheckProgramCompleted {
|
||||
diagnostics,
|
||||
revision,
|
||||
} => {
|
||||
// Only take the diagnostics if they are for the latest revision.
|
||||
if self.revision == revision {
|
||||
self.sender
|
||||
.send(MainLoopMessage::CheckCompleted(diagnostics))
|
||||
.unwrap();
|
||||
} else {
|
||||
tracing::debug!("Discarding diagnostics for outdated revision {revision} (current: {}).", self.revision);
|
||||
}
|
||||
}
|
||||
|
||||
OrchestratorMessage::FileChanges(changes) => {
|
||||
// Request cancellation, but wait until all analysis tasks have completed to
|
||||
// avoid stale messages in the next main loop.
|
||||
let pending = if let Some(pending_state) = self.pending_analysis.as_ref() {
|
||||
pending_state.cancellation_token.cancel();
|
||||
true
|
||||
} else {
|
||||
false
|
||||
};
|
||||
|
||||
self.aggregated_changes.extend(changes);
|
||||
|
||||
// If there are no pending analysis tasks, apply the file changes. Otherwise
|
||||
// keep running until all file checks have completed.
|
||||
if !pending {
|
||||
self.debounce_changes();
|
||||
}
|
||||
self.revision += 1;
|
||||
self.debounce_changes(changes);
|
||||
}
|
||||
OrchestratorMessage::Shutdown => {
|
||||
return self.shutdown();
|
||||
@@ -278,8 +258,9 @@ impl Orchestrator {
|
||||
}
|
||||
}
|
||||
|
||||
fn debounce_changes(&mut self) {
|
||||
debug_assert!(self.pending_analysis.is_none());
|
||||
fn debounce_changes(&self, changes: Vec<FileChange>) {
|
||||
let mut aggregated_changes = AggregatedChanges::default();
|
||||
aggregated_changes.extend(changes);
|
||||
|
||||
loop {
|
||||
// Consume possibly incoming file change messages before running a new analysis, but don't wait for more than 100ms.
|
||||
@@ -290,10 +271,12 @@ impl Orchestrator {
|
||||
return self.shutdown();
|
||||
}
|
||||
Ok(OrchestratorMessage::FileChanges(file_changes)) => {
|
||||
self.aggregated_changes.extend(file_changes);
|
||||
aggregated_changes.extend(file_changes);
|
||||
}
|
||||
|
||||
Ok(OrchestratorMessage::CheckProgramStarted {..}| OrchestratorMessage::CheckProgramCompleted(_) | OrchestratorMessage::CheckProgramCancelled) => unreachable!("No program check should be running while debouncing changes."),
|
||||
Ok(OrchestratorMessage::CheckProgramCompleted { .. })=> {
|
||||
// disregard any outdated completion message.
|
||||
}
|
||||
Ok(OrchestratorMessage::Run) => unreachable!("The orchestrator is already running."),
|
||||
|
||||
Err(_) => {
|
||||
@@ -302,10 +285,10 @@ impl Orchestrator {
|
||||
}
|
||||
}
|
||||
},
|
||||
default(std::time::Duration::from_millis(100)) => {
|
||||
// No more file changes after 100 ms, send the changes and schedule a new analysis
|
||||
self.sender.send(MainLoopMessage::ApplyChanges(std::mem::take(&mut self.aggregated_changes))).unwrap();
|
||||
self.sender.send(MainLoopMessage::CheckProgram).unwrap();
|
||||
default(std::time::Duration::from_millis(10)) => {
|
||||
// No more file changes after 10 ms, send the changes and schedule a new analysis
|
||||
self.sender.send(MainLoopMessage::ApplyChanges(aggregated_changes)).unwrap();
|
||||
self.sender.send(MainLoopMessage::CheckProgram { revision: self.revision}).unwrap();
|
||||
return;
|
||||
}
|
||||
}
|
||||
@@ -318,15 +301,10 @@ impl Orchestrator {
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct PendingAnalysisState {
|
||||
cancellation_token: CancellationTokenSource,
|
||||
}
|
||||
|
||||
/// Message sent from the orchestrator to the main loop.
|
||||
#[derive(Debug)]
|
||||
enum MainLoopMessage {
|
||||
CheckProgram,
|
||||
CheckProgram { revision: usize },
|
||||
CheckCompleted(Vec<String>),
|
||||
ApplyChanges(AggregatedChanges),
|
||||
Exit,
|
||||
@@ -337,11 +315,10 @@ enum OrchestratorMessage {
|
||||
Run,
|
||||
Shutdown,
|
||||
|
||||
CheckProgramStarted {
|
||||
cancellation_token: CancellationTokenSource,
|
||||
CheckProgramCompleted {
|
||||
diagnostics: Vec<String>,
|
||||
revision: usize,
|
||||
},
|
||||
CheckProgramCompleted(Vec<String>),
|
||||
CheckProgramCancelled,
|
||||
|
||||
FileChanges(Vec<FileChange>),
|
||||
}
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
use std::fmt::Formatter;
|
||||
use std::ops::Deref;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::atomic::AtomicU32;
|
||||
use std::sync::Arc;
|
||||
@@ -6,8 +7,9 @@ use std::sync::Arc;
|
||||
use dashmap::mapref::entry::Entry;
|
||||
use smol_str::SmolStr;
|
||||
|
||||
use crate::db::{HasJar, SemanticDb, SemanticJar};
|
||||
use crate::db::{HasJar, QueryResult, SemanticDb, SemanticJar};
|
||||
use crate::files::FileId;
|
||||
use crate::symbols::Dependency;
|
||||
use crate::FxDashMap;
|
||||
|
||||
/// ID uniquely identifying a module.
|
||||
@@ -15,53 +17,62 @@ use crate::FxDashMap;
|
||||
pub struct Module(u32);
|
||||
|
||||
impl Module {
|
||||
pub fn name<Db>(&self, db: &Db) -> ModuleName
|
||||
pub fn name<Db>(&self, db: &Db) -> QueryResult<ModuleName>
|
||||
where
|
||||
Db: HasJar<SemanticJar>,
|
||||
{
|
||||
let modules = &db.jar().module_resolver;
|
||||
let modules = &db.jar()?.module_resolver;
|
||||
|
||||
modules.modules.get(self).unwrap().name.clone()
|
||||
Ok(modules.modules.get(self).unwrap().name.clone())
|
||||
}
|
||||
|
||||
pub fn path<Db>(&self, db: &Db) -> ModulePath
|
||||
pub fn path<Db>(&self, db: &Db) -> QueryResult<ModulePath>
|
||||
where
|
||||
Db: HasJar<SemanticJar>,
|
||||
{
|
||||
let modules = &db.jar().module_resolver;
|
||||
let modules = &db.jar()?.module_resolver;
|
||||
|
||||
modules.modules.get(self).unwrap().path.clone()
|
||||
Ok(modules.modules.get(self).unwrap().path.clone())
|
||||
}
|
||||
|
||||
pub fn kind<Db>(&self, db: &Db) -> ModuleKind
|
||||
pub fn kind<Db>(&self, db: &Db) -> QueryResult<ModuleKind>
|
||||
where
|
||||
Db: HasJar<SemanticJar>,
|
||||
{
|
||||
let modules = &db.jar().module_resolver;
|
||||
let modules = &db.jar()?.module_resolver;
|
||||
|
||||
modules.modules.get(self).unwrap().kind
|
||||
Ok(modules.modules.get(self).unwrap().kind)
|
||||
}
|
||||
|
||||
pub fn relative_name<Db>(&self, db: &Db, level: u32, module: Option<&str>) -> Option<ModuleName>
|
||||
pub fn resolve_dependency<Db>(
|
||||
&self,
|
||||
db: &Db,
|
||||
dependency: &Dependency,
|
||||
) -> QueryResult<Option<ModuleName>>
|
||||
where
|
||||
Db: HasJar<SemanticJar>,
|
||||
{
|
||||
let name = self.name(db);
|
||||
let kind = self.kind(db);
|
||||
let (level, module) = match dependency {
|
||||
Dependency::Module(module) => return Ok(Some(module.clone())),
|
||||
Dependency::Relative { level, module } => (*level, module.as_deref()),
|
||||
};
|
||||
|
||||
let name = self.name(db)?;
|
||||
let kind = self.kind(db)?;
|
||||
|
||||
let mut components = name.components().peekable();
|
||||
|
||||
if level > 0 {
|
||||
let start = match kind {
|
||||
// `.` resolves to the enclosing package
|
||||
ModuleKind::Module => 0,
|
||||
// `.` resolves to the current package
|
||||
ModuleKind::Package => 1,
|
||||
};
|
||||
let start = match kind {
|
||||
// `.` resolves to the enclosing package
|
||||
ModuleKind::Module => 0,
|
||||
// `.` resolves to the current package
|
||||
ModuleKind::Package => 1,
|
||||
};
|
||||
|
||||
// Skip over the relative parts.
|
||||
for _ in start..level {
|
||||
components.next_back()?;
|
||||
// Skip over the relative parts.
|
||||
for _ in start..level.get() {
|
||||
if components.next_back().is_none() {
|
||||
return Ok(None);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -75,11 +86,11 @@ impl Module {
|
||||
name.push_str(part);
|
||||
}
|
||||
|
||||
if name.is_empty() {
|
||||
Ok(if name.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(ModuleName(SmolStr::new(name)))
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -141,6 +152,14 @@ impl ModuleName {
|
||||
}
|
||||
}
|
||||
|
||||
impl Deref for ModuleName {
|
||||
type Target = str;
|
||||
|
||||
fn deref(&self) -> &Self::Target {
|
||||
self.as_str()
|
||||
}
|
||||
}
|
||||
|
||||
impl std::fmt::Display for ModuleName {
|
||||
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
|
||||
f.write_str(&self.0)
|
||||
@@ -225,20 +244,25 @@ pub struct ModuleData {
|
||||
/// TODO: This would not work with Salsa because `ModuleName` isn't an ingredient and, therefore, cannot be used as part of a query.
|
||||
/// For this to work with salsa, it would be necessary to intern all `ModuleName`s.
|
||||
#[tracing::instrument(level = "debug", skip(db))]
|
||||
pub fn resolve_module<Db>(db: &Db, name: ModuleName) -> Option<Module>
|
||||
pub fn resolve_module<Db>(db: &Db, name: ModuleName) -> QueryResult<Option<Module>>
|
||||
where
|
||||
Db: SemanticDb + HasJar<SemanticJar>,
|
||||
{
|
||||
let jar = db.jar();
|
||||
let modules = &jar.module_resolver;
|
||||
let modules = &jar?.module_resolver;
|
||||
|
||||
let entry = modules.by_name.entry(name.clone());
|
||||
|
||||
match entry {
|
||||
Entry::Occupied(entry) => Some(*entry.get()),
|
||||
Entry::Occupied(entry) => Ok(Some(*entry.get())),
|
||||
Entry::Vacant(entry) => {
|
||||
let (root_path, absolute_path, kind) = resolve_name(&name, &modules.search_paths)?;
|
||||
let normalized = absolute_path.canonicalize().ok()?;
|
||||
let Some((root_path, absolute_path, kind)) = resolve_name(&name, &modules.search_paths)
|
||||
else {
|
||||
return Ok(None);
|
||||
};
|
||||
let Ok(normalized) = absolute_path.canonicalize() else {
|
||||
return Ok(None);
|
||||
};
|
||||
|
||||
let file_id = db.file_id(&normalized);
|
||||
let path = ModulePath::new(root_path.clone(), file_id);
|
||||
@@ -260,59 +284,76 @@ where
|
||||
// ```
|
||||
// Here, both `foo` and `bar` resolve to the same module but through different paths.
|
||||
// That's why we need to insert the absolute path and not the normalized path here.
|
||||
modules.by_path.insert(absolute_path, id);
|
||||
let absolute_id = if absolute_path == normalized {
|
||||
file_id
|
||||
} else {
|
||||
db.file_id(&absolute_path)
|
||||
};
|
||||
|
||||
modules.by_file.insert(absolute_id, id);
|
||||
|
||||
entry.insert_entry(id);
|
||||
|
||||
Some(id)
|
||||
Ok(Some(id))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Resolves the module id for the file with the given id.
|
||||
///
|
||||
/// Returns `None` if the file is not a module in `sys.path`.
|
||||
#[tracing::instrument(level = "debug", skip(db))]
|
||||
pub fn file_to_module<Db>(db: &Db, file: FileId) -> Option<Module>
|
||||
where
|
||||
Db: SemanticDb + HasJar<SemanticJar>,
|
||||
{
|
||||
let path = db.file_path(file);
|
||||
path_to_module(db, &path)
|
||||
}
|
||||
|
||||
/// Resolves the module id for the given path.
|
||||
///
|
||||
/// Returns `None` if the path is not a module in `sys.path`.
|
||||
#[tracing::instrument(level = "debug", skip(db))]
|
||||
pub fn path_to_module<Db>(db: &Db, path: &Path) -> Option<Module>
|
||||
pub fn path_to_module<Db>(db: &Db, path: &Path) -> QueryResult<Option<Module>>
|
||||
where
|
||||
Db: SemanticDb + HasJar<SemanticJar>,
|
||||
{
|
||||
let jar = db.jar();
|
||||
let modules = &jar.module_resolver;
|
||||
debug_assert!(path.is_absolute());
|
||||
let file = db.file_id(path);
|
||||
file_to_module(db, file)
|
||||
}
|
||||
|
||||
if let Some(existing) = modules.by_path.get(path) {
|
||||
return Some(*existing);
|
||||
/// Resolves the module id for the file with the given id.
|
||||
///
|
||||
/// Returns `None` if the file is not a module in `sys.path`.
|
||||
#[tracing::instrument(level = "debug", skip(db))]
|
||||
pub fn file_to_module<Db>(db: &Db, file: FileId) -> QueryResult<Option<Module>>
|
||||
where
|
||||
Db: SemanticDb + HasJar<SemanticJar>,
|
||||
{
|
||||
let jar = db.jar()?;
|
||||
let modules = &jar.module_resolver;
|
||||
|
||||
if let Some(existing) = modules.by_file.get(&file) {
|
||||
return Ok(Some(*existing));
|
||||
}
|
||||
|
||||
let (root_path, relative_path) = modules.search_paths.iter().find_map(|root| {
|
||||
let path = db.file_path(file);
|
||||
|
||||
debug_assert!(path.is_absolute());
|
||||
|
||||
let Some((root_path, relative_path)) = modules.search_paths.iter().find_map(|root| {
|
||||
let relative_path = path.strip_prefix(root.path()).ok()?;
|
||||
Some((root.clone(), relative_path))
|
||||
})?;
|
||||
}) else {
|
||||
return Ok(None);
|
||||
};
|
||||
|
||||
let module_name = ModuleName::from_relative_path(relative_path)?;
|
||||
let Some(module_name) = ModuleName::from_relative_path(relative_path) else {
|
||||
return Ok(None);
|
||||
};
|
||||
|
||||
// Resolve the module name to see if Python would resolve the name to the same path.
|
||||
// If it doesn't, then that means that multiple modules have the same in different
|
||||
// root paths, but that the module corresponding to the past path is in a lower priority path,
|
||||
// in which case we ignore it.
|
||||
let module_id = resolve_module(db, module_name)?;
|
||||
let module_path = module_id.path(db);
|
||||
let Some(module_id) = resolve_module(db, module_name)? else {
|
||||
return Ok(None);
|
||||
};
|
||||
let module_path = module_id.path(db)?;
|
||||
|
||||
if module_path.root() == &root_path {
|
||||
let normalized = path.canonicalize().ok()?;
|
||||
let Ok(normalized) = path.canonicalize() else {
|
||||
return Ok(None);
|
||||
};
|
||||
let interned_normalized = db.file_id(&normalized);
|
||||
|
||||
if interned_normalized != module_path.file() {
|
||||
@@ -323,15 +364,15 @@ where
|
||||
// ```
|
||||
// The module name of `src/foo.py` is `foo`, but the module loaded by Python is `src/foo/__init__.py`.
|
||||
// That means we need to ignore `src/foo.py` even though it resolves to the same module name.
|
||||
return None;
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
// Path has been inserted by `resolved`
|
||||
Some(module_id)
|
||||
Ok(Some(module_id))
|
||||
} else {
|
||||
// This path is for a module with the same name but in a module search path with a lower priority.
|
||||
// Ignore it.
|
||||
None
|
||||
Ok(None)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -365,7 +406,7 @@ where
|
||||
// TODO This needs tests
|
||||
|
||||
// Note: Intentionally by-pass caching here. Module should not be in the cache yet.
|
||||
let module = path_to_module(db, path)?;
|
||||
let module = path_to_module(db, path).ok()??;
|
||||
|
||||
// The code below is to handle the addition of `__init__.py` files.
|
||||
// When an `__init__.py` file is added, we need to remove all modules that are part of the same package.
|
||||
@@ -379,7 +420,7 @@ where
|
||||
return Some((module, Vec::new()));
|
||||
}
|
||||
|
||||
let Some(parent_name) = module.name(db).parent() else {
|
||||
let Some(parent_name) = module.name(db).ok()?.parent() else {
|
||||
return Some((module, Vec::new()));
|
||||
};
|
||||
|
||||
@@ -388,7 +429,7 @@ where
|
||||
let jar = db.jar_mut();
|
||||
let modules = &mut jar.module_resolver;
|
||||
|
||||
modules.by_path.retain(|_, id| {
|
||||
modules.by_file.retain(|_, id| {
|
||||
if modules
|
||||
.modules
|
||||
.get(id)
|
||||
@@ -427,7 +468,7 @@ pub struct ModuleResolver {
|
||||
|
||||
/// Lookup from absolute path to module.
|
||||
/// The same module might be reachable from different paths when symlinks are involved.
|
||||
by_path: FxDashMap<PathBuf, Module>,
|
||||
by_file: FxDashMap<FileId, Module>,
|
||||
next_module_id: AtomicU32,
|
||||
}
|
||||
|
||||
@@ -437,14 +478,14 @@ impl ModuleResolver {
|
||||
search_paths,
|
||||
modules: FxDashMap::default(),
|
||||
by_name: FxDashMap::default(),
|
||||
by_path: FxDashMap::default(),
|
||||
by_file: FxDashMap::default(),
|
||||
next_module_id: AtomicU32::new(0),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn remove_module(&mut self, path: &Path) {
|
||||
pub(crate) fn remove_module(&mut self, file_id: FileId) {
|
||||
// No locking is required because we're holding a mutable reference to `self`.
|
||||
let Some((_, id)) = self.by_path.remove(path) else {
|
||||
let Some((_, id)) = self.by_file.remove(&file_id) else {
|
||||
return;
|
||||
};
|
||||
|
||||
@@ -457,7 +498,7 @@ impl ModuleResolver {
|
||||
self.by_name.remove(&module.name).unwrap();
|
||||
|
||||
// It's possible that multiple paths map to the same id. Search all other paths referencing the same module id.
|
||||
self.by_path.retain(|_, current_id| *current_id != id);
|
||||
self.by_file.retain(|_, current_id| *current_id != id);
|
||||
|
||||
module
|
||||
}
|
||||
@@ -638,6 +679,8 @@ mod tests {
|
||||
use crate::db::tests::TestDb;
|
||||
use crate::db::{SemanticDb, SourceDb};
|
||||
use crate::module::{ModuleKind, ModuleName, ModuleSearchPath, ModuleSearchPathKind};
|
||||
use crate::symbols::Dependency;
|
||||
use std::num::NonZeroU32;
|
||||
|
||||
struct TestCase {
|
||||
temp_dir: tempfile::TempDir,
|
||||
@@ -676,7 +719,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn first_party_module() -> std::io::Result<()> {
|
||||
fn first_party_module() -> anyhow::Result<()> {
|
||||
let TestCase {
|
||||
db,
|
||||
src,
|
||||
@@ -687,22 +730,22 @@ mod tests {
|
||||
let foo_path = src.path().join("foo.py");
|
||||
std::fs::write(&foo_path, "print('Hello, world!')")?;
|
||||
|
||||
let foo_module = db.resolve_module(ModuleName::new("foo")).unwrap();
|
||||
let foo_module = db.resolve_module(ModuleName::new("foo"))?.unwrap();
|
||||
|
||||
assert_eq!(Some(foo_module), db.resolve_module(ModuleName::new("foo")));
|
||||
assert_eq!(Some(foo_module), db.resolve_module(ModuleName::new("foo"))?);
|
||||
|
||||
assert_eq!(ModuleName::new("foo"), foo_module.name(&db));
|
||||
assert_eq!(&src, foo_module.path(&db).root());
|
||||
assert_eq!(ModuleKind::Module, foo_module.kind(&db));
|
||||
assert_eq!(&foo_path, &*db.file_path(foo_module.path(&db).file()));
|
||||
assert_eq!(ModuleName::new("foo"), foo_module.name(&db)?);
|
||||
assert_eq!(&src, foo_module.path(&db)?.root());
|
||||
assert_eq!(ModuleKind::Module, foo_module.kind(&db)?);
|
||||
assert_eq!(&foo_path, &*db.file_path(foo_module.path(&db)?.file()));
|
||||
|
||||
assert_eq!(Some(foo_module), db.path_to_module(&foo_path));
|
||||
assert_eq!(Some(foo_module), db.path_to_module(&foo_path)?);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_package() -> std::io::Result<()> {
|
||||
fn resolve_package() -> anyhow::Result<()> {
|
||||
let TestCase {
|
||||
src,
|
||||
db,
|
||||
@@ -715,22 +758,22 @@ mod tests {
|
||||
std::fs::create_dir(&foo_dir)?;
|
||||
std::fs::write(&foo_path, "print('Hello, world!')")?;
|
||||
|
||||
let foo_module = db.resolve_module(ModuleName::new("foo")).unwrap();
|
||||
let foo_module = db.resolve_module(ModuleName::new("foo"))?.unwrap();
|
||||
|
||||
assert_eq!(ModuleName::new("foo"), foo_module.name(&db));
|
||||
assert_eq!(&src, foo_module.path(&db).root());
|
||||
assert_eq!(&foo_path, &*db.file_path(foo_module.path(&db).file()));
|
||||
assert_eq!(ModuleName::new("foo"), foo_module.name(&db)?);
|
||||
assert_eq!(&src, foo_module.path(&db)?.root());
|
||||
assert_eq!(&foo_path, &*db.file_path(foo_module.path(&db)?.file()));
|
||||
|
||||
assert_eq!(Some(foo_module), db.path_to_module(&foo_path));
|
||||
assert_eq!(Some(foo_module), db.path_to_module(&foo_path)?);
|
||||
|
||||
// Resolving by directory doesn't resolve to the init file.
|
||||
assert_eq!(None, db.path_to_module(&foo_dir));
|
||||
assert_eq!(None, db.path_to_module(&foo_dir)?);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn package_priority_over_module() -> std::io::Result<()> {
|
||||
fn package_priority_over_module() -> anyhow::Result<()> {
|
||||
let TestCase {
|
||||
db,
|
||||
temp_dir: _temp_dir,
|
||||
@@ -746,20 +789,20 @@ mod tests {
|
||||
let foo_py = src.path().join("foo.py");
|
||||
std::fs::write(&foo_py, "print('Hello, world!')")?;
|
||||
|
||||
let foo_module = db.resolve_module(ModuleName::new("foo")).unwrap();
|
||||
let foo_module = db.resolve_module(ModuleName::new("foo"))?.unwrap();
|
||||
|
||||
assert_eq!(&src, foo_module.path(&db).root());
|
||||
assert_eq!(&foo_init, &*db.file_path(foo_module.path(&db).file()));
|
||||
assert_eq!(ModuleKind::Package, foo_module.kind(&db));
|
||||
assert_eq!(&src, foo_module.path(&db)?.root());
|
||||
assert_eq!(&foo_init, &*db.file_path(foo_module.path(&db)?.file()));
|
||||
assert_eq!(ModuleKind::Package, foo_module.kind(&db)?);
|
||||
|
||||
assert_eq!(Some(foo_module), db.path_to_module(&foo_init));
|
||||
assert_eq!(None, db.path_to_module(&foo_py));
|
||||
assert_eq!(Some(foo_module), db.path_to_module(&foo_init)?);
|
||||
assert_eq!(None, db.path_to_module(&foo_py)?);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn typing_stub_over_module() -> std::io::Result<()> {
|
||||
fn typing_stub_over_module() -> anyhow::Result<()> {
|
||||
let TestCase {
|
||||
db,
|
||||
src,
|
||||
@@ -772,19 +815,19 @@ mod tests {
|
||||
std::fs::write(&foo_stub, "x: int")?;
|
||||
std::fs::write(&foo_py, "print('Hello, world!')")?;
|
||||
|
||||
let foo = db.resolve_module(ModuleName::new("foo")).unwrap();
|
||||
let foo = db.resolve_module(ModuleName::new("foo"))?.unwrap();
|
||||
|
||||
assert_eq!(&src, foo.path(&db).root());
|
||||
assert_eq!(&foo_stub, &*db.file_path(foo.path(&db).file()));
|
||||
assert_eq!(&src, foo.path(&db)?.root());
|
||||
assert_eq!(&foo_stub, &*db.file_path(foo.path(&db)?.file()));
|
||||
|
||||
assert_eq!(Some(foo), db.path_to_module(&foo_stub));
|
||||
assert_eq!(None, db.path_to_module(&foo_py));
|
||||
assert_eq!(Some(foo), db.path_to_module(&foo_stub)?);
|
||||
assert_eq!(None, db.path_to_module(&foo_py)?);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sub_packages() -> std::io::Result<()> {
|
||||
fn sub_packages() -> anyhow::Result<()> {
|
||||
let TestCase {
|
||||
db,
|
||||
src,
|
||||
@@ -801,18 +844,18 @@ mod tests {
|
||||
std::fs::write(bar.join("__init__.py"), "")?;
|
||||
std::fs::write(&baz, "print('Hello, world!')")?;
|
||||
|
||||
let baz_module = db.resolve_module(ModuleName::new("foo.bar.baz")).unwrap();
|
||||
let baz_module = db.resolve_module(ModuleName::new("foo.bar.baz"))?.unwrap();
|
||||
|
||||
assert_eq!(&src, baz_module.path(&db).root());
|
||||
assert_eq!(&baz, &*db.file_path(baz_module.path(&db).file()));
|
||||
assert_eq!(&src, baz_module.path(&db)?.root());
|
||||
assert_eq!(&baz, &*db.file_path(baz_module.path(&db)?.file()));
|
||||
|
||||
assert_eq!(Some(baz_module), db.path_to_module(&baz));
|
||||
assert_eq!(Some(baz_module), db.path_to_module(&baz)?);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn namespace_package() -> std::io::Result<()> {
|
||||
fn namespace_package() -> anyhow::Result<()> {
|
||||
let TestCase {
|
||||
db,
|
||||
temp_dir: _,
|
||||
@@ -848,21 +891,21 @@ mod tests {
|
||||
std::fs::write(&two, "print('Hello, world!')")?;
|
||||
|
||||
let one_module = db
|
||||
.resolve_module(ModuleName::new("parent.child.one"))
|
||||
.resolve_module(ModuleName::new("parent.child.one"))?
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(Some(one_module), db.path_to_module(&one));
|
||||
assert_eq!(Some(one_module), db.path_to_module(&one)?);
|
||||
|
||||
let two_module = db
|
||||
.resolve_module(ModuleName::new("parent.child.two"))
|
||||
.resolve_module(ModuleName::new("parent.child.two"))?
|
||||
.unwrap();
|
||||
assert_eq!(Some(two_module), db.path_to_module(&two));
|
||||
assert_eq!(Some(two_module), db.path_to_module(&two)?);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn regular_package_in_namespace_package() -> std::io::Result<()> {
|
||||
fn regular_package_in_namespace_package() -> anyhow::Result<()> {
|
||||
let TestCase {
|
||||
db,
|
||||
temp_dir: _,
|
||||
@@ -899,17 +942,20 @@ mod tests {
|
||||
std::fs::write(two, "print('Hello, world!')")?;
|
||||
|
||||
let one_module = db
|
||||
.resolve_module(ModuleName::new("parent.child.one"))
|
||||
.resolve_module(ModuleName::new("parent.child.one"))?
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(Some(one_module), db.path_to_module(&one));
|
||||
assert_eq!(Some(one_module), db.path_to_module(&one)?);
|
||||
|
||||
assert_eq!(None, db.resolve_module(ModuleName::new("parent.child.two")));
|
||||
assert_eq!(
|
||||
None,
|
||||
db.resolve_module(ModuleName::new("parent.child.two"))?
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn module_search_path_priority() -> std::io::Result<()> {
|
||||
fn module_search_path_priority() -> anyhow::Result<()> {
|
||||
let TestCase {
|
||||
db,
|
||||
src,
|
||||
@@ -923,20 +969,20 @@ mod tests {
|
||||
std::fs::write(&foo_src, "")?;
|
||||
std::fs::write(&foo_site_packages, "")?;
|
||||
|
||||
let foo_module = db.resolve_module(ModuleName::new("foo")).unwrap();
|
||||
let foo_module = db.resolve_module(ModuleName::new("foo"))?.unwrap();
|
||||
|
||||
assert_eq!(&src, foo_module.path(&db).root());
|
||||
assert_eq!(&foo_src, &*db.file_path(foo_module.path(&db).file()));
|
||||
assert_eq!(&src, foo_module.path(&db)?.root());
|
||||
assert_eq!(&foo_src, &*db.file_path(foo_module.path(&db)?.file()));
|
||||
|
||||
assert_eq!(Some(foo_module), db.path_to_module(&foo_src));
|
||||
assert_eq!(None, db.path_to_module(&foo_site_packages));
|
||||
assert_eq!(Some(foo_module), db.path_to_module(&foo_src)?);
|
||||
assert_eq!(None, db.path_to_module(&foo_site_packages)?);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[cfg(target_family = "unix")]
|
||||
fn symlink() -> std::io::Result<()> {
|
||||
fn symlink() -> anyhow::Result<()> {
|
||||
let TestCase {
|
||||
db,
|
||||
src,
|
||||
@@ -950,28 +996,28 @@ mod tests {
|
||||
std::fs::write(&foo, "")?;
|
||||
std::os::unix::fs::symlink(&foo, &bar)?;
|
||||
|
||||
let foo_module = db.resolve_module(ModuleName::new("foo")).unwrap();
|
||||
let bar_module = db.resolve_module(ModuleName::new("bar")).unwrap();
|
||||
let foo_module = db.resolve_module(ModuleName::new("foo"))?.unwrap();
|
||||
let bar_module = db.resolve_module(ModuleName::new("bar"))?.unwrap();
|
||||
|
||||
assert_ne!(foo_module, bar_module);
|
||||
|
||||
assert_eq!(&src, foo_module.path(&db).root());
|
||||
assert_eq!(&foo, &*db.file_path(foo_module.path(&db).file()));
|
||||
assert_eq!(&src, foo_module.path(&db)?.root());
|
||||
assert_eq!(&foo, &*db.file_path(foo_module.path(&db)?.file()));
|
||||
|
||||
// Bar has a different name but it should point to the same file.
|
||||
|
||||
assert_eq!(&src, bar_module.path(&db).root());
|
||||
assert_eq!(foo_module.path(&db).file(), bar_module.path(&db).file());
|
||||
assert_eq!(&foo, &*db.file_path(bar_module.path(&db).file()));
|
||||
assert_eq!(&src, bar_module.path(&db)?.root());
|
||||
assert_eq!(foo_module.path(&db)?.file(), bar_module.path(&db)?.file());
|
||||
assert_eq!(&foo, &*db.file_path(bar_module.path(&db)?.file()));
|
||||
|
||||
assert_eq!(Some(foo_module), db.path_to_module(&foo));
|
||||
assert_eq!(Some(bar_module), db.path_to_module(&bar));
|
||||
assert_eq!(Some(foo_module), db.path_to_module(&foo)?);
|
||||
assert_eq!(Some(bar_module), db.path_to_module(&bar)?);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn relative_name() -> std::io::Result<()> {
|
||||
fn resolve_dependency() -> anyhow::Result<()> {
|
||||
let TestCase {
|
||||
src,
|
||||
db,
|
||||
@@ -987,46 +1033,79 @@ mod tests {
|
||||
std::fs::write(foo_path, "from .bar import test")?;
|
||||
std::fs::write(bar_path, "test = 'Hello world'")?;
|
||||
|
||||
let foo_module = db.resolve_module(ModuleName::new("foo")).unwrap();
|
||||
let bar_module = db.resolve_module(ModuleName::new("foo.bar")).unwrap();
|
||||
let foo_module = db.resolve_module(ModuleName::new("foo"))?.unwrap();
|
||||
let bar_module = db.resolve_module(ModuleName::new("foo.bar"))?.unwrap();
|
||||
|
||||
// `from . import bar` in `foo/__init__.py` resolves to `foo`
|
||||
assert_eq!(
|
||||
Some(ModuleName::new("foo")),
|
||||
foo_module.relative_name(&db, 1, None)
|
||||
foo_module.resolve_dependency(
|
||||
&db,
|
||||
&Dependency::Relative {
|
||||
level: NonZeroU32::new(1).unwrap(),
|
||||
module: None,
|
||||
}
|
||||
)?
|
||||
);
|
||||
|
||||
// `from baz import bar` in `foo/__init__.py` should resolve to `foo/baz.py`
|
||||
// `from baz import bar` in `foo/__init__.py` should resolve to `baz.py`
|
||||
assert_eq!(
|
||||
Some(ModuleName::new("foo.baz")),
|
||||
foo_module.relative_name(&db, 0, Some("baz"))
|
||||
Some(ModuleName::new("baz")),
|
||||
foo_module.resolve_dependency(&db, &Dependency::Module(ModuleName::new("baz")))?
|
||||
);
|
||||
|
||||
// from .bar import test in `foo/__init__.py` should resolve to `foo/bar.py`
|
||||
assert_eq!(
|
||||
Some(ModuleName::new("foo.bar")),
|
||||
foo_module.relative_name(&db, 1, Some("bar"))
|
||||
foo_module.resolve_dependency(
|
||||
&db,
|
||||
&Dependency::Relative {
|
||||
level: NonZeroU32::new(1).unwrap(),
|
||||
module: Some(ModuleName::new("bar"))
|
||||
}
|
||||
)?
|
||||
);
|
||||
|
||||
// from .. import test in `foo/__init__.py` resolves to `` which is not a module
|
||||
assert_eq!(None, foo_module.relative_name(&db, 2, None));
|
||||
assert_eq!(
|
||||
None,
|
||||
foo_module.resolve_dependency(
|
||||
&db,
|
||||
&Dependency::Relative {
|
||||
level: NonZeroU32::new(2).unwrap(),
|
||||
module: None
|
||||
}
|
||||
)?
|
||||
);
|
||||
|
||||
// `from . import test` in `foo/bar.py` resolves to `foo`
|
||||
assert_eq!(
|
||||
Some(ModuleName::new("foo")),
|
||||
bar_module.relative_name(&db, 1, None)
|
||||
bar_module.resolve_dependency(
|
||||
&db,
|
||||
&Dependency::Relative {
|
||||
level: NonZeroU32::new(1).unwrap(),
|
||||
module: None
|
||||
}
|
||||
)?
|
||||
);
|
||||
|
||||
// `from baz import test` in `foo/bar.py` resolves to `foo.bar.baz`
|
||||
// `from baz import test` in `foo/bar.py` resolves to `baz`
|
||||
assert_eq!(
|
||||
Some(ModuleName::new("foo.bar.baz")),
|
||||
bar_module.relative_name(&db, 0, Some("baz"))
|
||||
Some(ModuleName::new("baz")),
|
||||
bar_module.resolve_dependency(&db, &Dependency::Module(ModuleName::new("baz")))?
|
||||
);
|
||||
|
||||
// `from .baz import test` in `foo/bar.py` resolves to `foo.baz`.
|
||||
assert_eq!(
|
||||
Some(ModuleName::new("foo.baz")),
|
||||
bar_module.relative_name(&db, 1, Some("baz"))
|
||||
bar_module.resolve_dependency(
|
||||
&db,
|
||||
&Dependency::Relative {
|
||||
level: NonZeroU32::new(1).unwrap(),
|
||||
module: Some(ModuleName::new("baz"))
|
||||
}
|
||||
)?
|
||||
);
|
||||
|
||||
Ok(())
|
||||
|
||||
@@ -6,7 +6,7 @@ use ruff_python_parser::{Mode, ParseError};
|
||||
use ruff_text_size::{Ranged, TextRange};
|
||||
|
||||
use crate::cache::KeyValueCache;
|
||||
use crate::db::{HasJar, SourceDb, SourceJar};
|
||||
use crate::db::{HasJar, QueryResult, SourceDb, SourceJar};
|
||||
use crate::files::FileId;
|
||||
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
@@ -64,16 +64,16 @@ impl Parsed {
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "debug", skip(db))]
|
||||
pub(crate) fn parse<Db>(db: &Db, file_id: FileId) -> Parsed
|
||||
pub(crate) fn parse<Db>(db: &Db, file_id: FileId) -> QueryResult<Parsed>
|
||||
where
|
||||
Db: SourceDb + HasJar<SourceJar>,
|
||||
{
|
||||
let parsed = db.jar();
|
||||
let parsed = db.jar()?;
|
||||
|
||||
parsed.parsed.get(&file_id, |file_id| {
|
||||
let source = db.source(*file_id);
|
||||
let source = db.source(*file_id)?;
|
||||
|
||||
Parsed::from_text(source.text())
|
||||
Ok(Parsed::from_text(source.text()))
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -1,66 +1,59 @@
|
||||
use crate::cancellation::CancellationToken;
|
||||
use crate::db::{SemanticDb, SourceDb};
|
||||
use rayon::{current_num_threads, yield_local};
|
||||
use rustc_hash::FxHashSet;
|
||||
|
||||
use crate::db::{Database, LintDb, QueryError, QueryResult, SemanticDb};
|
||||
use crate::files::FileId;
|
||||
use crate::format::{FormatDb, FormatError};
|
||||
use crate::lint::Diagnostics;
|
||||
use crate::program::Program;
|
||||
use rayon::max_num_threads;
|
||||
use rustc_hash::FxHashSet;
|
||||
use std::num::NonZeroUsize;
|
||||
use crate::symbols::Dependency;
|
||||
|
||||
impl Program {
|
||||
/// Checks all open files in the workspace and its dependencies.
|
||||
#[tracing::instrument(level = "debug", skip_all)]
|
||||
pub fn check(
|
||||
&self,
|
||||
scheduler: &dyn CheckScheduler,
|
||||
cancellation_token: CancellationToken,
|
||||
) -> Result<Vec<String>, CheckError> {
|
||||
let check_loop = CheckFilesLoop::new(scheduler, cancellation_token);
|
||||
pub fn check(&self, mode: ExecutionMode) -> QueryResult<Vec<String>> {
|
||||
self.cancelled()?;
|
||||
|
||||
check_loop.run(self.workspace().open_files.iter().copied())
|
||||
}
|
||||
let mut context = CheckContext::new(self);
|
||||
|
||||
/// Checks a single file and its dependencies.
|
||||
#[tracing::instrument(level = "debug", skip(self, scheduler, cancellation_token))]
|
||||
pub fn check_file(
|
||||
&self,
|
||||
file: FileId,
|
||||
scheduler: &dyn CheckScheduler,
|
||||
cancellation_token: CancellationToken,
|
||||
) -> Result<Vec<String>, CheckError> {
|
||||
let check_loop = CheckFilesLoop::new(scheduler, cancellation_token);
|
||||
match mode {
|
||||
ExecutionMode::SingleThreaded => SingleThreadedExecutor.run(&mut context)?,
|
||||
ExecutionMode::ThreadPool => ThreadPoolExecutor.run(&mut context)?,
|
||||
};
|
||||
|
||||
check_loop.run([file].into_iter())
|
||||
Ok(context.finish())
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "debug", skip(self, context))]
|
||||
fn do_check_file(
|
||||
&self,
|
||||
file: FileId,
|
||||
context: &CheckContext,
|
||||
) -> Result<Diagnostics, CheckError> {
|
||||
context.cancelled_ok()?;
|
||||
fn check_file(&self, file: FileId, context: &CheckFileContext) -> QueryResult<Diagnostics> {
|
||||
self.cancelled()?;
|
||||
|
||||
let symbol_table = self.symbol_table(file);
|
||||
let symbol_table = self.symbol_table(file)?;
|
||||
let dependencies = symbol_table.dependencies();
|
||||
|
||||
if !dependencies.is_empty() {
|
||||
let module = self.file_to_module(file);
|
||||
let module = self.file_to_module(file)?;
|
||||
|
||||
// TODO scheduling all dependencies here is wasteful if we don't infer any types on them
|
||||
// but I think that's unlikely, so it is okay?
|
||||
// Anyway, we need to figure out a way to retrieve the dependencies of a module
|
||||
// from the persistent cache. So maybe it should be a separate query after all.
|
||||
for dependency in dependencies {
|
||||
let dependency_name = dependency.module_name(self, module);
|
||||
let dependency_name = match dependency {
|
||||
Dependency::Module(name) => Some(name.clone()),
|
||||
Dependency::Relative { .. } => match &module {
|
||||
Some(module) => module.resolve_dependency(self, dependency)?,
|
||||
None => None,
|
||||
},
|
||||
};
|
||||
|
||||
if let Some(dependency_name) = dependency_name {
|
||||
// TODO We may want to have a different check functions for non-first-party
|
||||
// files because we only need to index them and not check them.
|
||||
// Supporting non-first-party code also requires supporting typing stubs.
|
||||
if let Some(dependency) = self.resolve_module(dependency_name) {
|
||||
if dependency.path(self).root().kind().is_first_party() {
|
||||
context.schedule_check_file(dependency.path(self).file());
|
||||
if let Some(dependency) = self.resolve_module(dependency_name)? {
|
||||
if dependency.path(self)?.root().kind().is_first_party() {
|
||||
context.schedule_dependency(dependency.path(self)?.file());
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -70,237 +63,363 @@ impl Program {
|
||||
let mut diagnostics = Vec::new();
|
||||
|
||||
if self.workspace().is_file_open(file) {
|
||||
diagnostics.extend_from_slice(&self.lint_syntax(file));
|
||||
diagnostics.extend_from_slice(&self.lint_syntax(file)?);
|
||||
diagnostics.extend_from_slice(&self.lint_semantic(file)?);
|
||||
|
||||
match self.check_file_formatted(file) {
|
||||
Ok(format_diagnostics) => {
|
||||
diagnostics.extend_from_slice(&format_diagnostics);
|
||||
}
|
||||
Err(FormatError::Query(err)) => {
|
||||
return Err(err);
|
||||
}
|
||||
Err(FormatError::Format(error)) => {
|
||||
diagnostics.push(format!("Error formatting file: {error}"));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(Diagnostics::from(diagnostics))
|
||||
}
|
||||
}
|
||||
|
||||
/// Schedules checks for files.
|
||||
pub trait CheckScheduler {
|
||||
/// Schedules a check for a file.
|
||||
///
|
||||
/// The check can either be run immediately on the current thread or the check can be queued
|
||||
/// in a thread pool and ran asynchronously.
|
||||
///
|
||||
/// The order in which scheduled checks are executed is not guaranteed.
|
||||
///
|
||||
/// The implementation should call [`CheckFileTask::run`] to execute the check.
|
||||
fn check_file(&self, file_task: CheckFileTask);
|
||||
|
||||
/// The maximum number of checks that can be run concurrently.
|
||||
///
|
||||
/// Returns `None` if the checks run on the current thread (no concurrency).
|
||||
fn max_concurrency(&self) -> Option<NonZeroUsize>;
|
||||
#[derive(Copy, Clone, Debug, PartialEq, Eq)]
|
||||
pub enum ExecutionMode {
|
||||
SingleThreaded,
|
||||
ThreadPool,
|
||||
}
|
||||
|
||||
/// Scheduler that runs checks on a rayon thread pool.
|
||||
pub struct RayonCheckScheduler<'program, 'scope_ref, 'scope> {
|
||||
program: &'program Program,
|
||||
scope: &'scope_ref rayon::Scope<'scope>,
|
||||
/// Context that stores state information about the entire check operation.
|
||||
struct CheckContext<'a> {
|
||||
/// IDs of the files that have been queued for checking.
|
||||
///
|
||||
/// Used to avoid queuing the same file twice.
|
||||
scheduled_files: FxHashSet<FileId>,
|
||||
|
||||
/// Reference to the program that is checked.
|
||||
program: &'a Program,
|
||||
|
||||
/// The aggregated diagnostics
|
||||
diagnostics: Vec<String>,
|
||||
}
|
||||
|
||||
impl<'program, 'scope_ref, 'scope> RayonCheckScheduler<'program, 'scope_ref, 'scope> {
|
||||
pub fn new(program: &'program Program, scope: &'scope_ref rayon::Scope<'scope>) -> Self {
|
||||
Self { program, scope }
|
||||
impl<'a> CheckContext<'a> {
|
||||
fn new(program: &'a Program) -> Self {
|
||||
Self {
|
||||
scheduled_files: FxHashSet::default(),
|
||||
program,
|
||||
diagnostics: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the tasks to check all open files in the workspace.
|
||||
fn check_open_files(&mut self) -> Vec<CheckOpenFileTask> {
|
||||
self.scheduled_files
|
||||
.extend(self.program.workspace().open_files());
|
||||
|
||||
self.program
|
||||
.workspace()
|
||||
.open_files()
|
||||
.map(|file_id| CheckOpenFileTask { file_id })
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Returns the task to check a dependency.
|
||||
fn check_dependency(&mut self, file_id: FileId) -> Option<CheckDependencyTask> {
|
||||
if self.scheduled_files.insert(file_id) {
|
||||
Some(CheckDependencyTask { file_id })
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
/// Pushes the result for a single file check operation
|
||||
fn push_diagnostics(&mut self, diagnostics: &Diagnostics) {
|
||||
self.diagnostics.extend_from_slice(diagnostics);
|
||||
}
|
||||
|
||||
/// Returns a reference to the program that is being checked.
|
||||
fn program(&self) -> &'a Program {
|
||||
self.program
|
||||
}
|
||||
|
||||
/// Creates a task context that is used to check a single file.
|
||||
fn task_context<'b, S>(&self, dependency_scheduler: &'b S) -> CheckTaskContext<'a, 'b, S>
|
||||
where
|
||||
S: ScheduleDependency,
|
||||
{
|
||||
CheckTaskContext {
|
||||
program: self.program,
|
||||
dependency_scheduler,
|
||||
}
|
||||
}
|
||||
|
||||
fn finish(self) -> Vec<String> {
|
||||
self.diagnostics
|
||||
}
|
||||
}
|
||||
|
||||
impl<'program, 'scope_ref, 'scope> CheckScheduler
|
||||
for RayonCheckScheduler<'program, 'scope_ref, 'scope>
|
||||
/// Trait that abstracts away how a dependency of a file gets scheduled for checking.
|
||||
trait ScheduleDependency {
|
||||
/// Schedules the file with the given ID for checking.
|
||||
fn schedule(&self, file_id: FileId);
|
||||
}
|
||||
|
||||
impl<T> ScheduleDependency for T
|
||||
where
|
||||
'program: 'scope,
|
||||
T: Fn(FileId),
|
||||
{
|
||||
fn check_file(&self, check_file_task: CheckFileTask) {
|
||||
let child_span =
|
||||
tracing::trace_span!("check_file", file_id = check_file_task.file_id.as_u32());
|
||||
let program = self.program;
|
||||
|
||||
self.scope
|
||||
.spawn(move |_| child_span.in_scope(|| check_file_task.run(program)));
|
||||
}
|
||||
|
||||
fn max_concurrency(&self) -> Option<NonZeroUsize> {
|
||||
Some(NonZeroUsize::new(max_num_threads()).unwrap_or(NonZeroUsize::MIN))
|
||||
fn schedule(&self, file_id: FileId) {
|
||||
let f = self;
|
||||
f(file_id);
|
||||
}
|
||||
}
|
||||
|
||||
/// Scheduler that runs all checks on the current thread.
|
||||
pub struct SameThreadCheckScheduler<'a> {
|
||||
/// Context that is used to run a single file check task.
|
||||
///
|
||||
/// The task is generic over `S` because it is passed across thread boundaries and
|
||||
/// we don't want to add the requirement that [`ScheduleDependency`] must be [`Send`].
|
||||
struct CheckTaskContext<'a, 'scheduler, S>
|
||||
where
|
||||
S: ScheduleDependency,
|
||||
{
|
||||
dependency_scheduler: &'scheduler S,
|
||||
program: &'a Program,
|
||||
}
|
||||
|
||||
impl<'a> SameThreadCheckScheduler<'a> {
|
||||
pub fn new(program: &'a Program) -> Self {
|
||||
Self { program }
|
||||
impl<'a, 'scheduler, S> CheckTaskContext<'a, 'scheduler, S>
|
||||
where
|
||||
S: ScheduleDependency,
|
||||
{
|
||||
fn as_file_context(&self) -> CheckFileContext<'scheduler> {
|
||||
CheckFileContext {
|
||||
dependency_scheduler: self.dependency_scheduler,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl CheckScheduler for SameThreadCheckScheduler<'_> {
|
||||
fn check_file(&self, task: CheckFileTask) {
|
||||
task.run(self.program);
|
||||
}
|
||||
|
||||
fn max_concurrency(&self) -> Option<NonZeroUsize> {
|
||||
None
|
||||
}
|
||||
/// Context passed when checking a single file.
|
||||
///
|
||||
/// This is a trimmed down version of [`CheckTaskContext`] with the type parameter `S` erased
|
||||
/// to avoid monomorphization of [`Program:check_file`].
|
||||
struct CheckFileContext<'a> {
|
||||
dependency_scheduler: &'a dyn ScheduleDependency,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub enum CheckError {
|
||||
Cancelled,
|
||||
impl<'a> CheckFileContext<'a> {
|
||||
fn schedule_dependency(&self, file_id: FileId) {
|
||||
self.dependency_scheduler.schedule(file_id);
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct CheckFileTask {
|
||||
file_id: FileId,
|
||||
context: CheckContext,
|
||||
enum CheckFileTask {
|
||||
OpenFile(CheckOpenFileTask),
|
||||
Dependency(CheckDependencyTask),
|
||||
}
|
||||
|
||||
impl CheckFileTask {
|
||||
/// Runs the check and communicates the result to the orchestrator.
|
||||
pub fn run(self, program: &Program) {
|
||||
match program.do_check_file(self.file_id, &self.context) {
|
||||
Ok(diagnostics) => self
|
||||
.context
|
||||
.sender
|
||||
.send(CheckFileMessage::Completed(diagnostics))
|
||||
.unwrap(),
|
||||
Err(CheckError::Cancelled) => self
|
||||
.context
|
||||
.sender
|
||||
.send(CheckFileMessage::Cancelled)
|
||||
.unwrap(),
|
||||
/// Runs the task and returns the results for checking this file.
|
||||
fn run<S>(&self, context: &CheckTaskContext<S>) -> QueryResult<Diagnostics>
|
||||
where
|
||||
S: ScheduleDependency,
|
||||
{
|
||||
match self {
|
||||
Self::OpenFile(task) => task.run(context),
|
||||
Self::Dependency(task) => task.run(context),
|
||||
}
|
||||
}
|
||||
|
||||
fn file_id(&self) -> FileId {
|
||||
match self {
|
||||
CheckFileTask::OpenFile(task) => task.file_id,
|
||||
CheckFileTask::Dependency(task) => task.file_id,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
struct CheckContext {
|
||||
cancellation_token: CancellationToken,
|
||||
sender: crossbeam_channel::Sender<CheckFileMessage>,
|
||||
/// Task to check an open file.
|
||||
|
||||
#[derive(Debug)]
|
||||
struct CheckOpenFileTask {
|
||||
file_id: FileId,
|
||||
}
|
||||
|
||||
impl CheckContext {
|
||||
fn new(
|
||||
cancellation_token: CancellationToken,
|
||||
sender: crossbeam_channel::Sender<CheckFileMessage>,
|
||||
) -> Self {
|
||||
Self {
|
||||
cancellation_token,
|
||||
sender,
|
||||
}
|
||||
}
|
||||
|
||||
/// Queues a new file for checking using the [`CheckScheduler`].
|
||||
#[allow(unused)]
|
||||
fn schedule_check_file(&self, file_id: FileId) {
|
||||
self.sender.send(CheckFileMessage::Queue(file_id)).unwrap();
|
||||
}
|
||||
|
||||
/// Returns `true` if the check has been cancelled.
|
||||
fn is_cancelled(&self) -> bool {
|
||||
self.cancellation_token.is_cancelled()
|
||||
}
|
||||
|
||||
fn cancelled_ok(&self) -> Result<(), CheckError> {
|
||||
if self.is_cancelled() {
|
||||
Err(CheckError::Cancelled)
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
impl CheckOpenFileTask {
|
||||
fn run<S>(&self, context: &CheckTaskContext<S>) -> QueryResult<Diagnostics>
|
||||
where
|
||||
S: ScheduleDependency,
|
||||
{
|
||||
context
|
||||
.program
|
||||
.check_file(self.file_id, &context.as_file_context())
|
||||
}
|
||||
}
|
||||
|
||||
struct CheckFilesLoop<'a> {
|
||||
scheduler: &'a dyn CheckScheduler,
|
||||
cancellation_token: CancellationToken,
|
||||
pending: usize,
|
||||
queued_files: FxHashSet<FileId>,
|
||||
/// Task to check a dependency file.
|
||||
#[derive(Debug)]
|
||||
struct CheckDependencyTask {
|
||||
file_id: FileId,
|
||||
}
|
||||
|
||||
impl<'a> CheckFilesLoop<'a> {
|
||||
fn new(scheduler: &'a dyn CheckScheduler, cancellation_token: CancellationToken) -> Self {
|
||||
Self {
|
||||
scheduler,
|
||||
cancellation_token,
|
||||
|
||||
queued_files: FxHashSet::default(),
|
||||
pending: 0,
|
||||
}
|
||||
impl CheckDependencyTask {
|
||||
fn run<S>(&self, context: &CheckTaskContext<S>) -> QueryResult<Diagnostics>
|
||||
where
|
||||
S: ScheduleDependency,
|
||||
{
|
||||
context
|
||||
.program
|
||||
.check_file(self.file_id, &context.as_file_context())
|
||||
}
|
||||
}
|
||||
|
||||
fn run(mut self, files: impl Iterator<Item = FileId>) -> Result<Vec<String>, CheckError> {
|
||||
let (sender, receiver) = if let Some(max_concurrency) = self.scheduler.max_concurrency() {
|
||||
crossbeam_channel::bounded(max_concurrency.get())
|
||||
} else {
|
||||
// The checks run on the current thread. That means it is necessary to store all messages
|
||||
// or we risk deadlocking when the main loop never gets a chance to read the messages.
|
||||
crossbeam_channel::unbounded()
|
||||
};
|
||||
/// Executor that schedules the checking of individual program files.
|
||||
trait CheckExecutor {
|
||||
fn run(self, context: &mut CheckContext) -> QueryResult<()>;
|
||||
}
|
||||
|
||||
let context = CheckContext::new(self.cancellation_token.clone(), sender.clone());
|
||||
/// Executor that runs all check operations on the current thread.
|
||||
///
|
||||
/// The executor does not schedule dependencies for checking.
|
||||
/// The main motivation for scheduling dependencies
|
||||
/// in a multithreaded environment is to parse and index the dependencies concurrently.
|
||||
/// However, that doesn't make sense in a single threaded environment, because the dependencies then compute
|
||||
/// with checking the open files. Checking dependencies in a single threaded environment is more likely
|
||||
/// to hurt performance because we end up analyzing files in their entirety, even if we only need to type check parts of them.
|
||||
#[derive(Debug, Default)]
|
||||
struct SingleThreadedExecutor;
|
||||
|
||||
for file in files {
|
||||
self.queue_file(file, context.clone())?;
|
||||
}
|
||||
impl CheckExecutor for SingleThreadedExecutor {
|
||||
fn run(self, context: &mut CheckContext) -> QueryResult<()> {
|
||||
let mut queue = context.check_open_files();
|
||||
|
||||
self.run_impl(receiver, &context)
|
||||
}
|
||||
let noop_schedule_dependency = |_| {};
|
||||
|
||||
fn run_impl(
|
||||
mut self,
|
||||
receiver: crossbeam_channel::Receiver<CheckFileMessage>,
|
||||
context: &CheckContext,
|
||||
) -> Result<Vec<String>, CheckError> {
|
||||
if self.cancellation_token.is_cancelled() {
|
||||
return Err(CheckError::Cancelled);
|
||||
}
|
||||
while let Some(file) = queue.pop() {
|
||||
context.program().cancelled()?;
|
||||
|
||||
let mut result = Vec::default();
|
||||
|
||||
for message in receiver {
|
||||
match message {
|
||||
CheckFileMessage::Completed(diagnostics) => {
|
||||
result.extend_from_slice(&diagnostics);
|
||||
|
||||
self.pending -= 1;
|
||||
|
||||
if self.pending == 0 {
|
||||
break;
|
||||
}
|
||||
}
|
||||
CheckFileMessage::Queue(id) => {
|
||||
self.queue_file(id, context.clone())?;
|
||||
}
|
||||
CheckFileMessage::Cancelled => {
|
||||
return Err(CheckError::Cancelled);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
fn queue_file(&mut self, file_id: FileId, context: CheckContext) -> Result<(), CheckError> {
|
||||
if context.is_cancelled() {
|
||||
return Err(CheckError::Cancelled);
|
||||
}
|
||||
|
||||
if self.queued_files.insert(file_id) {
|
||||
self.pending += 1;
|
||||
|
||||
self.scheduler
|
||||
.check_file(CheckFileTask { file_id, context });
|
||||
let task_context = context.task_context(&noop_schedule_dependency);
|
||||
context.push_diagnostics(&file.run(&task_context)?);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
enum CheckFileMessage {
|
||||
Completed(Diagnostics),
|
||||
Queue(FileId),
|
||||
Cancelled,
|
||||
/// Executor that runs the check operations on a thread pool.
|
||||
///
|
||||
/// The executor runs each check operation as its own task using a thread pool.
|
||||
///
|
||||
/// Other than [`SingleThreadedExecutor`], this executor schedules dependencies for checking. It
|
||||
/// even schedules dependencies for checking when the thread pool size is 1 for a better debugging experience.
|
||||
#[derive(Debug, Default)]
|
||||
struct ThreadPoolExecutor;
|
||||
|
||||
impl CheckExecutor for ThreadPoolExecutor {
|
||||
fn run(self, context: &mut CheckContext) -> QueryResult<()> {
|
||||
let num_threads = current_num_threads();
|
||||
let single_threaded = num_threads == 1;
|
||||
let span = tracing::trace_span!("ThreadPoolExecutor::run", num_threads);
|
||||
let _ = span.enter();
|
||||
|
||||
let mut queue: Vec<_> = context
|
||||
.check_open_files()
|
||||
.into_iter()
|
||||
.map(CheckFileTask::OpenFile)
|
||||
.collect();
|
||||
|
||||
let (sender, receiver) = if single_threaded {
|
||||
// Use an unbounded queue for single threaded execution to prevent deadlocks
|
||||
// when a single file schedules multiple dependencies.
|
||||
crossbeam::channel::unbounded()
|
||||
} else {
|
||||
// Use a bounded queue to apply backpressure when the orchestration thread isn't able to keep
|
||||
// up processing messages from the worker threads.
|
||||
crossbeam::channel::bounded(num_threads)
|
||||
};
|
||||
|
||||
let schedule_sender = sender.clone();
|
||||
let schedule_dependency = move |file_id| {
|
||||
schedule_sender
|
||||
.send(ThreadPoolMessage::ScheduleDependency(file_id))
|
||||
.unwrap();
|
||||
};
|
||||
|
||||
let result = rayon::in_place_scope(|scope| {
|
||||
let mut pending = 0usize;
|
||||
|
||||
loop {
|
||||
context.program().cancelled()?;
|
||||
|
||||
// 1. Try to get a queued message to ensure that we have always remaining space in the channel to prevent blocking the worker threads.
|
||||
// 2. Try to process a queued file
|
||||
// 3. If there's no queued file wait for the next incoming message.
|
||||
// 4. Exit if there are no more messages and no senders.
|
||||
let message = if let Ok(message) = receiver.try_recv() {
|
||||
message
|
||||
} else if let Some(task) = queue.pop() {
|
||||
pending += 1;
|
||||
|
||||
let task_context = context.task_context(&schedule_dependency);
|
||||
let sender = sender.clone();
|
||||
let task_span = tracing::trace_span!(
|
||||
parent: &span,
|
||||
"CheckFileTask::run",
|
||||
file_id = task.file_id().as_u32(),
|
||||
);
|
||||
|
||||
scope.spawn(move |_| {
|
||||
task_span.in_scope(|| match task.run(&task_context) {
|
||||
Ok(result) => {
|
||||
sender.send(ThreadPoolMessage::Completed(result)).unwrap();
|
||||
}
|
||||
Err(err) => sender.send(ThreadPoolMessage::Errored(err)).unwrap(),
|
||||
});
|
||||
});
|
||||
|
||||
// If this is a single threaded rayon thread pool, yield the current thread
|
||||
// or we never start processing the work items.
|
||||
if single_threaded {
|
||||
yield_local();
|
||||
}
|
||||
|
||||
continue;
|
||||
} else if let Ok(message) = receiver.recv() {
|
||||
message
|
||||
} else {
|
||||
break;
|
||||
};
|
||||
|
||||
match message {
|
||||
ThreadPoolMessage::ScheduleDependency(dependency) => {
|
||||
if let Some(task) = context.check_dependency(dependency) {
|
||||
queue.push(CheckFileTask::Dependency(task));
|
||||
}
|
||||
}
|
||||
ThreadPoolMessage::Completed(diagnostics) => {
|
||||
context.push_diagnostics(&diagnostics);
|
||||
pending -= 1;
|
||||
|
||||
if pending == 0 && queue.is_empty() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
ThreadPoolMessage::Errored(err) => {
|
||||
return Err(err);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
});
|
||||
|
||||
result
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
enum ThreadPoolMessage {
|
||||
ScheduleDependency(FileId),
|
||||
Completed(Diagnostics),
|
||||
Errored(QueryError),
|
||||
}
|
||||
|
||||
44
crates/red_knot/src/program/format.rs
Normal file
44
crates/red_knot/src/program/format.rs
Normal file
@@ -0,0 +1,44 @@
|
||||
use crate::db::{QueryResult, SourceDb};
|
||||
use crate::format::{FormatDb, FormatError, FormattedFile};
|
||||
use crate::program::Program;
|
||||
|
||||
impl Program {
|
||||
#[tracing::instrument(level = "trace", skip(self))]
|
||||
pub fn format(&mut self) -> QueryResult<()> {
|
||||
// Formats all open files
|
||||
|
||||
// TODO make `Executor` from `check` reusable.
|
||||
for file in self.workspace.open_files() {
|
||||
match self.format_file(file) {
|
||||
Ok(FormattedFile::Formatted(content)) => {
|
||||
let path = self.file_path(file);
|
||||
|
||||
// TODO: This is problematic because it immediately re-triggers the file watcher.
|
||||
// A possible solution is to track the self "inflicted" changes inside of programs
|
||||
// by tracking the file revision right after the write. It could then use the revision
|
||||
// to determine which changes are safe to ignore (and in which context).
|
||||
// An other alternative is to not write as part of the `format` command and instead
|
||||
// return a Vec with the format results and leave the writing to the caller.
|
||||
// I think that's undesired because a) we still need a way to tell the formatter
|
||||
// that it won't be necessary to format the content again and
|
||||
// b) it would reduce concurrency because the writing would need to wait for the file
|
||||
// formatting to be complete, unless we use some form of communication channel.
|
||||
std::fs::write(path, content).expect("Unable to write file");
|
||||
}
|
||||
Ok(FormattedFile::Unchanged) => {
|
||||
// No op
|
||||
}
|
||||
Err(FormatError::Query(error)) => {
|
||||
return Err(error);
|
||||
}
|
||||
Err(FormatError::Format(error)) => {
|
||||
// TODO proper error handling. We should either propagate this error or
|
||||
// emit a diagnostic (probably this).
|
||||
tracing::warn!("Failed to format file: {}", error);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -1,42 +1,42 @@
|
||||
pub mod check;
|
||||
|
||||
use ruff_formatter::PrintedRange;
|
||||
use ruff_text_size::TextRange;
|
||||
use std::path::Path;
|
||||
use std::sync::Arc;
|
||||
|
||||
use crate::db::{Db, HasJar, SemanticDb, SemanticJar, SourceDb, SourceJar};
|
||||
use crate::db::{
|
||||
Database, Db, DbRuntime, HasJar, HasJars, JarsStorage, LintDb, LintJar, ParallelDatabase,
|
||||
QueryResult, SemanticDb, SemanticJar, Snapshot, SourceDb, SourceJar,
|
||||
};
|
||||
use crate::files::{FileId, Files};
|
||||
use crate::lint::{lint_syntax, Diagnostics, LintSyntaxStorage};
|
||||
use crate::format::{
|
||||
check_formatted, format_file, format_file_range, FormatDb, FormatError, FormatJar,
|
||||
FormattedFile,
|
||||
};
|
||||
use crate::lint::{lint_semantic, lint_syntax, Diagnostics};
|
||||
use crate::module::{
|
||||
add_module, file_to_module, path_to_module, resolve_module, set_module_search_paths, Module,
|
||||
ModuleData, ModuleName, ModuleResolver, ModuleSearchPath,
|
||||
ModuleData, ModuleName, ModuleSearchPath,
|
||||
};
|
||||
use crate::parse::{parse, Parsed, ParsedStorage};
|
||||
use crate::source::{source_text, Source, SourceStorage};
|
||||
use crate::symbols::{symbol_table, SymbolId, SymbolTable, SymbolTablesStorage};
|
||||
use crate::types::{infer_symbol_type, Type, TypeStore};
|
||||
use crate::parse::{parse, Parsed};
|
||||
use crate::source::{source_text, Source};
|
||||
use crate::symbols::{symbol_table, SymbolId, SymbolTable};
|
||||
use crate::types::{infer_symbol_type, Type};
|
||||
use crate::Workspace;
|
||||
|
||||
pub mod check;
|
||||
mod format;
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct Program {
|
||||
jars: JarsStorage<Program>,
|
||||
files: Files,
|
||||
source: SourceJar,
|
||||
semantic: SemanticJar,
|
||||
workspace: Workspace,
|
||||
}
|
||||
|
||||
impl Program {
|
||||
pub fn new(workspace: Workspace, module_search_paths: Vec<ModuleSearchPath>) -> Self {
|
||||
pub fn new(workspace: Workspace) -> Self {
|
||||
Self {
|
||||
source: SourceJar {
|
||||
sources: SourceStorage::default(),
|
||||
parsed: ParsedStorage::default(),
|
||||
lint_syntax: LintSyntaxStorage::default(),
|
||||
},
|
||||
semantic: SemanticJar {
|
||||
module_resolver: ModuleResolver::new(module_search_paths),
|
||||
symbol_tables: SymbolTablesStorage::default(),
|
||||
type_store: TypeStore::default(),
|
||||
},
|
||||
jars: JarsStorage::default(),
|
||||
files: Files::default(),
|
||||
workspace,
|
||||
}
|
||||
@@ -46,16 +46,17 @@ impl Program {
|
||||
where
|
||||
I: IntoIterator<Item = FileChange>,
|
||||
{
|
||||
let (source, semantic, lint, format) = self.jars_mut();
|
||||
for change in changes {
|
||||
self.semantic
|
||||
.module_resolver
|
||||
.remove_module(&self.file_path(change.id));
|
||||
self.semantic.symbol_tables.remove(&change.id);
|
||||
self.source.sources.remove(&change.id);
|
||||
self.source.parsed.remove(&change.id);
|
||||
self.source.lint_syntax.remove(&change.id);
|
||||
semantic.module_resolver.remove_module(change.id);
|
||||
semantic.symbol_tables.remove(&change.id);
|
||||
source.sources.remove(&change.id);
|
||||
source.parsed.remove(&change.id);
|
||||
// TODO: remove all dependent modules as well
|
||||
self.semantic.type_store.remove_module(change.id);
|
||||
semantic.type_store.remove_module(change.id);
|
||||
lint.lint_syntax.remove(&change.id);
|
||||
lint.lint_semantic.remove(&change.id);
|
||||
format.formatted.remove(&change.id);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -81,38 +82,37 @@ impl SourceDb for Program {
|
||||
self.files.path(file_id)
|
||||
}
|
||||
|
||||
fn source(&self, file_id: FileId) -> Source {
|
||||
fn source(&self, file_id: FileId) -> QueryResult<Source> {
|
||||
source_text(self, file_id)
|
||||
}
|
||||
|
||||
fn parse(&self, file_id: FileId) -> Parsed {
|
||||
fn parse(&self, file_id: FileId) -> QueryResult<Parsed> {
|
||||
parse(self, file_id)
|
||||
}
|
||||
|
||||
fn lint_syntax(&self, file_id: FileId) -> Diagnostics {
|
||||
lint_syntax(self, file_id)
|
||||
}
|
||||
}
|
||||
|
||||
impl SemanticDb for Program {
|
||||
fn resolve_module(&self, name: ModuleName) -> Option<Module> {
|
||||
fn resolve_module(&self, name: ModuleName) -> QueryResult<Option<Module>> {
|
||||
resolve_module(self, name)
|
||||
}
|
||||
|
||||
fn file_to_module(&self, file_id: FileId) -> Option<Module> {
|
||||
fn file_to_module(&self, file_id: FileId) -> QueryResult<Option<Module>> {
|
||||
file_to_module(self, file_id)
|
||||
}
|
||||
|
||||
fn path_to_module(&self, path: &Path) -> Option<Module> {
|
||||
fn path_to_module(&self, path: &Path) -> QueryResult<Option<Module>> {
|
||||
path_to_module(self, path)
|
||||
}
|
||||
|
||||
fn symbol_table(&self, file_id: FileId) -> Arc<SymbolTable> {
|
||||
fn symbol_table(&self, file_id: FileId) -> QueryResult<Arc<SymbolTable>> {
|
||||
symbol_table(self, file_id)
|
||||
}
|
||||
|
||||
// Mutations
|
||||
fn infer_symbol_type(&self, file_id: FileId, symbol_id: SymbolId) -> QueryResult<Type> {
|
||||
infer_symbol_type(self, file_id, symbol_id)
|
||||
}
|
||||
|
||||
// Mutations
|
||||
fn add_module(&mut self, path: &Path) -> Option<(Module, Vec<Arc<ModuleData>>)> {
|
||||
add_module(self, path)
|
||||
}
|
||||
@@ -120,31 +120,107 @@ impl SemanticDb for Program {
|
||||
fn set_module_search_paths(&mut self, paths: Vec<ModuleSearchPath>) {
|
||||
set_module_search_paths(self, paths);
|
||||
}
|
||||
}
|
||||
|
||||
fn infer_symbol_type(&mut self, file_id: FileId, symbol_id: SymbolId) -> Type {
|
||||
infer_symbol_type(self, file_id, symbol_id)
|
||||
impl LintDb for Program {
|
||||
fn lint_syntax(&self, file_id: FileId) -> QueryResult<Diagnostics> {
|
||||
lint_syntax(self, file_id)
|
||||
}
|
||||
|
||||
fn lint_semantic(&self, file_id: FileId) -> QueryResult<Diagnostics> {
|
||||
lint_semantic(self, file_id)
|
||||
}
|
||||
}
|
||||
|
||||
impl FormatDb for Program {
|
||||
fn format_file(&self, file_id: FileId) -> Result<FormattedFile, FormatError> {
|
||||
format_file(self, file_id)
|
||||
}
|
||||
|
||||
fn format_file_range(
|
||||
&self,
|
||||
file_id: FileId,
|
||||
range: TextRange,
|
||||
) -> Result<PrintedRange, FormatError> {
|
||||
format_file_range(self, file_id, range)
|
||||
}
|
||||
|
||||
fn check_file_formatted(&self, file_id: FileId) -> Result<Diagnostics, FormatError> {
|
||||
check_formatted(self, file_id)
|
||||
}
|
||||
}
|
||||
|
||||
impl Db for Program {}
|
||||
|
||||
impl Database for Program {
|
||||
fn runtime(&self) -> &DbRuntime {
|
||||
self.jars.runtime()
|
||||
}
|
||||
|
||||
fn runtime_mut(&mut self) -> &mut DbRuntime {
|
||||
self.jars.runtime_mut()
|
||||
}
|
||||
}
|
||||
|
||||
impl ParallelDatabase for Program {
|
||||
fn snapshot(&self) -> Snapshot<Self> {
|
||||
Snapshot::new(Self {
|
||||
jars: self.jars.snapshot(),
|
||||
files: self.files.clone(),
|
||||
workspace: self.workspace.clone(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl HasJars for Program {
|
||||
type Jars = (SourceJar, SemanticJar, LintJar, FormatJar);
|
||||
|
||||
fn jars(&self) -> QueryResult<&Self::Jars> {
|
||||
self.jars.jars()
|
||||
}
|
||||
|
||||
fn jars_mut(&mut self) -> &mut Self::Jars {
|
||||
self.jars.jars_mut()
|
||||
}
|
||||
}
|
||||
|
||||
impl HasJar<SourceJar> for Program {
|
||||
fn jar(&self) -> &SourceJar {
|
||||
&self.source
|
||||
fn jar(&self) -> QueryResult<&SourceJar> {
|
||||
Ok(&self.jars()?.0)
|
||||
}
|
||||
|
||||
fn jar_mut(&mut self) -> &mut SourceJar {
|
||||
&mut self.source
|
||||
&mut self.jars_mut().0
|
||||
}
|
||||
}
|
||||
|
||||
impl HasJar<SemanticJar> for Program {
|
||||
fn jar(&self) -> &SemanticJar {
|
||||
&self.semantic
|
||||
fn jar(&self) -> QueryResult<&SemanticJar> {
|
||||
Ok(&self.jars()?.1)
|
||||
}
|
||||
|
||||
fn jar_mut(&mut self) -> &mut SemanticJar {
|
||||
&mut self.semantic
|
||||
&mut self.jars_mut().1
|
||||
}
|
||||
}
|
||||
|
||||
impl HasJar<LintJar> for Program {
|
||||
fn jar(&self) -> QueryResult<&LintJar> {
|
||||
Ok(&self.jars()?.2)
|
||||
}
|
||||
|
||||
fn jar_mut(&mut self) -> &mut LintJar {
|
||||
&mut self.jars_mut().2
|
||||
}
|
||||
}
|
||||
|
||||
impl HasJar<FormatJar> for Program {
|
||||
fn jar(&self) -> QueryResult<&FormatJar> {
|
||||
Ok(&self.jars()?.3)
|
||||
}
|
||||
|
||||
fn jar_mut(&mut self) -> &mut FormatJar {
|
||||
&mut self.jars_mut().3
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
use crate::cache::KeyValueCache;
|
||||
use crate::db::{HasJar, SourceDb, SourceJar};
|
||||
use crate::db::{HasJar, QueryResult, SourceDb, SourceJar};
|
||||
use ruff_notebook::Notebook;
|
||||
use ruff_python_ast::PySourceType;
|
||||
use std::ops::{Deref, DerefMut};
|
||||
@@ -8,11 +8,11 @@ use std::sync::Arc;
|
||||
use crate::files::FileId;
|
||||
|
||||
#[tracing::instrument(level = "debug", skip(db))]
|
||||
pub(crate) fn source_text<Db>(db: &Db, file_id: FileId) -> Source
|
||||
pub(crate) fn source_text<Db>(db: &Db, file_id: FileId) -> QueryResult<Source>
|
||||
where
|
||||
Db: SourceDb + HasJar<SourceJar>,
|
||||
{
|
||||
let sources = &db.jar().sources;
|
||||
let sources = &db.jar()?.sources;
|
||||
|
||||
sources.get(&file_id, |file_id| {
|
||||
let path = db.file_path(*file_id);
|
||||
@@ -43,7 +43,7 @@ where
|
||||
}
|
||||
};
|
||||
|
||||
Source { kind }
|
||||
Ok(Source { kind })
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -2,9 +2,11 @@
|
||||
|
||||
use std::hash::{Hash, Hasher};
|
||||
use std::iter::{Copied, DoubleEndedIterator, FusedIterator};
|
||||
use std::num::NonZeroU32;
|
||||
use std::ops::{Deref, DerefMut};
|
||||
use std::sync::Arc;
|
||||
|
||||
use bitflags::bitflags;
|
||||
use hashbrown::hash_map::{Keys, RawEntryMut};
|
||||
use rustc_hash::{FxHashMap, FxHasher};
|
||||
|
||||
@@ -14,22 +16,22 @@ use ruff_python_ast::visitor::preorder::PreorderVisitor;
|
||||
|
||||
use crate::ast_ids::TypedNodeKey;
|
||||
use crate::cache::KeyValueCache;
|
||||
use crate::db::{HasJar, SemanticDb, SemanticJar};
|
||||
use crate::db::{HasJar, QueryResult, SemanticDb, SemanticJar};
|
||||
use crate::files::FileId;
|
||||
use crate::module::{Module, ModuleName};
|
||||
use crate::module::ModuleName;
|
||||
use crate::Name;
|
||||
|
||||
#[allow(unreachable_pub)]
|
||||
#[tracing::instrument(level = "debug", skip(db))]
|
||||
pub fn symbol_table<Db>(db: &Db, file_id: FileId) -> Arc<SymbolTable>
|
||||
pub fn symbol_table<Db>(db: &Db, file_id: FileId) -> QueryResult<Arc<SymbolTable>>
|
||||
where
|
||||
Db: SemanticDb + HasJar<SemanticJar>,
|
||||
{
|
||||
let jar = db.jar();
|
||||
let jar = db.jar()?;
|
||||
|
||||
jar.symbol_tables.get(&file_id, |_| {
|
||||
let parsed = db.parse(file_id);
|
||||
Arc::from(SymbolTable::from_ast(parsed.ast()))
|
||||
let parsed = db.parse(file_id)?;
|
||||
Ok(Arc::from(SymbolTable::from_ast(parsed.ast())))
|
||||
})
|
||||
}
|
||||
|
||||
@@ -80,21 +82,58 @@ impl Scope {
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub(crate) enum Kind {
|
||||
FreeVar,
|
||||
CellVar,
|
||||
CellVarAssigned,
|
||||
ExplicitGlobal,
|
||||
ImplicitGlobal,
|
||||
}
|
||||
|
||||
bitflags! {
|
||||
#[derive(Copy,Clone,Debug)]
|
||||
pub(crate) struct SymbolFlags: u8 {
|
||||
const IS_USED = 1 << 0;
|
||||
const IS_DEFINED = 1 << 1;
|
||||
/// TODO: This flag is not yet set by anything
|
||||
const MARKED_GLOBAL = 1 << 2;
|
||||
/// TODO: This flag is not yet set by anything
|
||||
const MARKED_NONLOCAL = 1 << 3;
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub(crate) struct Symbol {
|
||||
name: Name,
|
||||
flags: SymbolFlags,
|
||||
// kind: Kind,
|
||||
}
|
||||
|
||||
impl Symbol {
|
||||
pub(crate) fn name(&self) -> &str {
|
||||
self.name.as_str()
|
||||
}
|
||||
|
||||
/// Is the symbol used in its containing scope?
|
||||
pub(crate) fn is_used(&self) -> bool {
|
||||
self.flags.contains(SymbolFlags::IS_USED)
|
||||
}
|
||||
|
||||
/// Is the symbol defined in its containing scope?
|
||||
pub(crate) fn is_defined(&self) -> bool {
|
||||
self.flags.contains(SymbolFlags::IS_DEFINED)
|
||||
}
|
||||
|
||||
// TODO: implement Symbol.kind 2-pass analysis to categorize as: free-var, cell-var,
|
||||
// explicit-global, implicit-global and implement Symbol.kind by modifying the preorder
|
||||
// traversal code
|
||||
}
|
||||
|
||||
// TODO storing TypedNodeKey for definitions means we have to search to find them again in the AST;
|
||||
// this is at best O(log n). If looking up definitions is a bottleneck we should look for
|
||||
// alternatives here.
|
||||
#[derive(Debug)]
|
||||
#[derive(Clone, Debug)]
|
||||
pub(crate) enum Definition {
|
||||
// For the import cases, we don't need reference to any arbitrary AST subtrees (annotations,
|
||||
// RHS), and referencing just the import statement node is imprecise (a single import statement
|
||||
@@ -109,36 +148,39 @@ pub(crate) enum Definition {
|
||||
// TODO with statements, except handlers, function args...
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
#[derive(Clone, Debug)]
|
||||
pub(crate) struct ImportDefinition {
|
||||
pub(crate) module: Name,
|
||||
pub(crate) module: ModuleName,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
#[derive(Clone, Debug)]
|
||||
pub(crate) struct ImportFromDefinition {
|
||||
pub(crate) module: Option<Name>,
|
||||
pub(crate) module: Option<ModuleName>,
|
||||
pub(crate) name: Name,
|
||||
pub(crate) level: u32,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub(crate) enum Dependency {
|
||||
Module(Name),
|
||||
Relative { level: u32, module: Option<Name> },
|
||||
impl ImportFromDefinition {
|
||||
pub(crate) fn module(&self) -> Option<&ModuleName> {
|
||||
self.module.as_ref()
|
||||
}
|
||||
|
||||
pub(crate) fn name(&self) -> &Name {
|
||||
&self.name
|
||||
}
|
||||
|
||||
pub(crate) fn level(&self) -> u32 {
|
||||
self.level
|
||||
}
|
||||
}
|
||||
|
||||
impl Dependency {
|
||||
pub(crate) fn module_name<Db>(&self, db: &Db, relative_to: Option<Module>) -> Option<ModuleName>
|
||||
where
|
||||
Db: SemanticDb + HasJar<SemanticJar>,
|
||||
{
|
||||
match self {
|
||||
Dependency::Module(name) => Some(ModuleName::new(name.as_str())),
|
||||
Dependency::Relative { level, module } => {
|
||||
relative_to?.relative_name(db, *level, module.as_deref())
|
||||
}
|
||||
}
|
||||
}
|
||||
#[derive(Debug, Clone)]
|
||||
pub enum Dependency {
|
||||
Module(ModuleName),
|
||||
Relative {
|
||||
level: NonZeroU32,
|
||||
module: Option<ModuleName>,
|
||||
},
|
||||
}
|
||||
|
||||
/// Table of all symbols in all scopes for a module.
|
||||
@@ -156,6 +198,7 @@ impl SymbolTable {
|
||||
let mut builder = SymbolTableBuilder {
|
||||
table: SymbolTable::new(),
|
||||
scopes: vec![root_scope_id],
|
||||
current_definition: None,
|
||||
};
|
||||
builder.visit_body(&module.body);
|
||||
builder.table
|
||||
@@ -253,14 +296,25 @@ impl SymbolTable {
|
||||
self.symbol_by_name(SymbolTable::root_scope_id(), name)
|
||||
}
|
||||
|
||||
pub(crate) fn defs(&self, symbol_id: SymbolId) -> &[Definition] {
|
||||
pub(crate) fn definitions(&self, symbol_id: SymbolId) -> &[Definition] {
|
||||
self.defs
|
||||
.get(&symbol_id)
|
||||
.map(std::vec::Vec::as_slice)
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
fn add_symbol_to_scope(&mut self, scope_id: ScopeId, name: &str) -> SymbolId {
|
||||
pub(crate) fn all_definitions(&self) -> impl Iterator<Item = (SymbolId, &Definition)> + '_ {
|
||||
self.defs
|
||||
.iter()
|
||||
.flat_map(|(sym_id, defs)| defs.iter().map(move |def| (*sym_id, def)))
|
||||
}
|
||||
|
||||
fn add_or_update_symbol(
|
||||
&mut self,
|
||||
scope_id: ScopeId,
|
||||
name: &str,
|
||||
flags: SymbolFlags,
|
||||
) -> SymbolId {
|
||||
let hash = SymbolTable::hash_name(name);
|
||||
let scope = &mut self.scopes_by_id[scope_id];
|
||||
let name = Name::new(name);
|
||||
@@ -271,9 +325,14 @@ impl SymbolTable {
|
||||
.from_hash(hash, |existing| self.symbols_by_id[*existing].name == name);
|
||||
|
||||
match entry {
|
||||
RawEntryMut::Occupied(entry) => *entry.key(),
|
||||
RawEntryMut::Occupied(entry) => {
|
||||
if let Some(symbol) = self.symbols_by_id.get_mut(*entry.key()) {
|
||||
symbol.flags.insert(flags);
|
||||
};
|
||||
*entry.key()
|
||||
}
|
||||
RawEntryMut::Vacant(entry) => {
|
||||
let id = self.symbols_by_id.push(Symbol { name });
|
||||
let id = self.symbols_by_id.push(Symbol { name, flags });
|
||||
entry.insert_with_hasher(hash, id, (), |_| hash);
|
||||
id
|
||||
}
|
||||
@@ -376,15 +435,22 @@ where
|
||||
struct SymbolTableBuilder {
|
||||
table: SymbolTable,
|
||||
scopes: Vec<ScopeId>,
|
||||
/// the definition whose target(s) we are currently walking
|
||||
current_definition: Option<Definition>,
|
||||
}
|
||||
|
||||
impl SymbolTableBuilder {
|
||||
fn add_symbol(&mut self, identifier: &str) -> SymbolId {
|
||||
self.table.add_symbol_to_scope(self.cur_scope(), identifier)
|
||||
fn add_or_update_symbol(&mut self, identifier: &str, flags: SymbolFlags) -> SymbolId {
|
||||
self.table
|
||||
.add_or_update_symbol(self.cur_scope(), identifier, flags)
|
||||
}
|
||||
|
||||
fn add_symbol_with_def(&mut self, identifier: &str, definition: Definition) -> SymbolId {
|
||||
let symbol_id = self.add_symbol(identifier);
|
||||
fn add_or_update_symbol_with_def(
|
||||
&mut self,
|
||||
identifier: &str,
|
||||
definition: Definition,
|
||||
) -> SymbolId {
|
||||
let symbol_id = self.add_or_update_symbol(identifier, SymbolFlags::IS_DEFINED);
|
||||
self.table
|
||||
.defs
|
||||
.entry(symbol_id)
|
||||
@@ -426,7 +492,7 @@ impl SymbolTableBuilder {
|
||||
ast::TypeParam::ParamSpec(ast::TypeParamParamSpec { name, .. }) => name,
|
||||
ast::TypeParam::TypeVarTuple(ast::TypeParamTypeVarTuple { name, .. }) => name,
|
||||
};
|
||||
self.add_symbol(name);
|
||||
self.add_or_update_symbol(name, SymbolFlags::IS_DEFINED);
|
||||
}
|
||||
}
|
||||
nested(self);
|
||||
@@ -438,8 +504,19 @@ impl SymbolTableBuilder {
|
||||
|
||||
impl PreorderVisitor<'_> for SymbolTableBuilder {
|
||||
fn visit_expr(&mut self, expr: &ast::Expr) {
|
||||
if let ast::Expr::Name(ast::ExprName { id, .. }) = expr {
|
||||
self.add_symbol(id);
|
||||
if let ast::Expr::Name(ast::ExprName { id, ctx, .. }) = expr {
|
||||
let flags = match ctx {
|
||||
ast::ExprContext::Load => SymbolFlags::IS_USED,
|
||||
ast::ExprContext::Store => SymbolFlags::IS_DEFINED,
|
||||
ast::ExprContext::Del => SymbolFlags::IS_DEFINED,
|
||||
ast::ExprContext::Invalid => SymbolFlags::empty(),
|
||||
};
|
||||
self.add_or_update_symbol(id, flags);
|
||||
if flags.contains(SymbolFlags::IS_DEFINED) {
|
||||
if let Some(curdef) = self.current_definition.clone() {
|
||||
self.add_or_update_symbol_with_def(id, curdef);
|
||||
}
|
||||
}
|
||||
}
|
||||
ast::visitor::preorder::walk_expr(self, expr);
|
||||
}
|
||||
@@ -449,7 +526,7 @@ impl PreorderVisitor<'_> for SymbolTableBuilder {
|
||||
match stmt {
|
||||
ast::Stmt::ClassDef(node) => {
|
||||
let def = Definition::ClassDef(TypedNodeKey::from_node(node));
|
||||
self.add_symbol_with_def(&node.name, def);
|
||||
self.add_or_update_symbol_with_def(&node.name, def);
|
||||
self.with_type_params(&node.name, &node.type_params, |builder| {
|
||||
builder.push_scope(builder.cur_scope(), &node.name, ScopeKind::Class);
|
||||
ast::visitor::preorder::walk_stmt(builder, stmt);
|
||||
@@ -458,7 +535,7 @@ impl PreorderVisitor<'_> for SymbolTableBuilder {
|
||||
}
|
||||
ast::Stmt::FunctionDef(node) => {
|
||||
let def = Definition::FunctionDef(TypedNodeKey::from_node(node));
|
||||
self.add_symbol_with_def(&node.name, def);
|
||||
self.add_or_update_symbol_with_def(&node.name, def);
|
||||
self.with_type_params(&node.name, &node.type_params, |builder| {
|
||||
builder.push_scope(builder.cur_scope(), &node.name, ScopeKind::Function);
|
||||
ast::visitor::preorder::walk_stmt(builder, stmt);
|
||||
@@ -473,12 +550,12 @@ impl PreorderVisitor<'_> for SymbolTableBuilder {
|
||||
alias.name.id.split('.').next().unwrap()
|
||||
};
|
||||
|
||||
let module = Name::new(&alias.name.id);
|
||||
let module = ModuleName::new(&alias.name.id);
|
||||
|
||||
let def = Definition::Import(ImportDefinition {
|
||||
module: module.clone(),
|
||||
});
|
||||
self.add_symbol_with_def(symbol_name, def);
|
||||
self.add_or_update_symbol_with_def(symbol_name, def);
|
||||
self.table.dependencies.push(Dependency::Module(module));
|
||||
}
|
||||
}
|
||||
@@ -488,7 +565,7 @@ impl PreorderVisitor<'_> for SymbolTableBuilder {
|
||||
level,
|
||||
..
|
||||
}) => {
|
||||
let module = module.as_ref().map(|m| Name::new(&m.id));
|
||||
let module = module.as_ref().map(|m| ModuleName::new(&m.id));
|
||||
|
||||
for alias in names {
|
||||
let symbol_name = if let Some(asname) = &alias.asname {
|
||||
@@ -501,27 +578,34 @@ impl PreorderVisitor<'_> for SymbolTableBuilder {
|
||||
name: Name::new(&alias.name.id),
|
||||
level: *level,
|
||||
});
|
||||
self.add_symbol_with_def(symbol_name, def);
|
||||
self.add_or_update_symbol_with_def(symbol_name, def);
|
||||
}
|
||||
|
||||
let dependency = if let Some(module) = module {
|
||||
if *level == 0 {
|
||||
Dependency::Module(module)
|
||||
} else {
|
||||
Dependency::Relative {
|
||||
level: *level,
|
||||
match NonZeroU32::new(*level) {
|
||||
Some(level) => Dependency::Relative {
|
||||
level,
|
||||
module: Some(module),
|
||||
}
|
||||
},
|
||||
None => Dependency::Module(module),
|
||||
}
|
||||
} else {
|
||||
Dependency::Relative {
|
||||
level: *level,
|
||||
level: NonZeroU32::new(*level)
|
||||
.expect("Import without a module to have a level > 0"),
|
||||
module,
|
||||
}
|
||||
};
|
||||
|
||||
self.table.dependencies.push(dependency);
|
||||
}
|
||||
ast::Stmt::Assign(node) => {
|
||||
debug_assert!(self.current_definition.is_none());
|
||||
self.current_definition =
|
||||
Some(Definition::Assignment(TypedNodeKey::from_node(node)));
|
||||
ast::visitor::preorder::walk_stmt(self, stmt);
|
||||
self.current_definition = None;
|
||||
}
|
||||
_ => {
|
||||
ast::visitor::preorder::walk_stmt(self, stmt);
|
||||
}
|
||||
@@ -553,7 +637,7 @@ mod tests {
|
||||
use crate::parse::Parsed;
|
||||
use crate::symbols::ScopeKind;
|
||||
|
||||
use super::{SymbolId, SymbolIterator, SymbolTable};
|
||||
use super::{SymbolFlags, SymbolId, SymbolIterator, SymbolTable};
|
||||
|
||||
mod from_ast {
|
||||
use super::*;
|
||||
@@ -584,7 +668,9 @@ mod tests {
|
||||
let table = SymbolTable::from_ast(parsed.ast());
|
||||
assert_eq!(names(table.root_symbols()), vec!["x"]);
|
||||
assert_eq!(
|
||||
table.defs(table.root_symbol_id_by_name("x").unwrap()).len(),
|
||||
table
|
||||
.definitions(table.root_symbol_id_by_name("x").unwrap())
|
||||
.len(),
|
||||
0
|
||||
);
|
||||
}
|
||||
@@ -604,7 +690,7 @@ mod tests {
|
||||
assert_eq!(names(table.root_symbols()), vec!["foo"]);
|
||||
assert_eq!(
|
||||
table
|
||||
.defs(table.root_symbol_id_by_name("foo").unwrap())
|
||||
.definitions(table.root_symbol_id_by_name("foo").unwrap())
|
||||
.len(),
|
||||
1
|
||||
);
|
||||
@@ -631,10 +717,37 @@ mod tests {
|
||||
assert_eq!(names(table.root_symbols()), vec!["foo"]);
|
||||
assert_eq!(
|
||||
table
|
||||
.defs(table.root_symbol_id_by_name("foo").unwrap())
|
||||
.definitions(table.root_symbol_id_by_name("foo").unwrap())
|
||||
.len(),
|
||||
1
|
||||
);
|
||||
assert!(
|
||||
table.root_symbol_id_by_name("foo").is_some_and(|sid| {
|
||||
let s = sid.symbol(&table);
|
||||
s.is_defined() || !s.is_used()
|
||||
}),
|
||||
"symbols that are defined get the defined flag"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn assign() {
|
||||
let parsed = parse("x = foo");
|
||||
let table = SymbolTable::from_ast(parsed.ast());
|
||||
assert_eq!(names(table.root_symbols()), vec!["foo", "x"]);
|
||||
assert_eq!(
|
||||
table
|
||||
.definitions(table.root_symbol_id_by_name("x").unwrap())
|
||||
.len(),
|
||||
1
|
||||
);
|
||||
assert!(
|
||||
table.root_symbol_id_by_name("foo").is_some_and(|sid| {
|
||||
let s = sid.symbol(&table);
|
||||
!s.is_defined() && s.is_used()
|
||||
}),
|
||||
"a symbol used but not defined in a scope should have only the used flag"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -655,7 +768,9 @@ mod tests {
|
||||
assert_eq!(c_scope.name(), "C");
|
||||
assert_eq!(names(table.symbols_for_scope(scopes[0])), vec!["x"]);
|
||||
assert_eq!(
|
||||
table.defs(table.root_symbol_id_by_name("C").unwrap()).len(),
|
||||
table
|
||||
.definitions(table.root_symbol_id_by_name("C").unwrap())
|
||||
.len(),
|
||||
1
|
||||
);
|
||||
}
|
||||
@@ -679,7 +794,7 @@ mod tests {
|
||||
assert_eq!(names(table.symbols_for_scope(scopes[0])), vec!["x"]);
|
||||
assert_eq!(
|
||||
table
|
||||
.defs(table.root_symbol_id_by_name("func").unwrap())
|
||||
.definitions(table.root_symbol_id_by_name("func").unwrap())
|
||||
.len(),
|
||||
1
|
||||
);
|
||||
@@ -709,7 +824,7 @@ mod tests {
|
||||
assert_eq!(names(table.symbols_for_scope(scopes[1])), vec!["y"]);
|
||||
assert_eq!(
|
||||
table
|
||||
.defs(table.root_symbol_id_by_name("func").unwrap())
|
||||
.definitions(table.root_symbol_id_by_name("func").unwrap())
|
||||
.len(),
|
||||
2
|
||||
);
|
||||
@@ -758,6 +873,12 @@ mod tests {
|
||||
assert_eq!(ann_scope.kind(), ScopeKind::Annotation);
|
||||
assert_eq!(ann_scope.name(), "C");
|
||||
assert_eq!(names(table.symbols_for_scope(ann_scope_id)), vec!["T"]);
|
||||
assert!(
|
||||
table
|
||||
.symbol_by_name(ann_scope_id, "T")
|
||||
.is_some_and(|s| s.is_defined() && !s.is_used()),
|
||||
"type parameters are defined by the scope that introduces them"
|
||||
);
|
||||
let scopes = table.child_scope_ids_of(ann_scope_id);
|
||||
assert_eq!(scopes.len(), 1);
|
||||
let func_scope_id = scopes[0];
|
||||
@@ -772,17 +893,19 @@ mod tests {
|
||||
fn insert_same_name_symbol_twice() {
|
||||
let mut table = SymbolTable::new();
|
||||
let root_scope_id = SymbolTable::root_scope_id();
|
||||
let symbol_id_1 = table.add_symbol_to_scope(root_scope_id, "foo");
|
||||
let symbol_id_2 = table.add_symbol_to_scope(root_scope_id, "foo");
|
||||
let symbol_id_1 = table.add_or_update_symbol(root_scope_id, "foo", SymbolFlags::IS_DEFINED);
|
||||
let symbol_id_2 = table.add_or_update_symbol(root_scope_id, "foo", SymbolFlags::IS_USED);
|
||||
assert_eq!(symbol_id_1, symbol_id_2);
|
||||
assert!(symbol_id_1.symbol(&table).is_used(), "flags must merge");
|
||||
assert!(symbol_id_1.symbol(&table).is_defined(), "flags must merge");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn insert_different_named_symbols() {
|
||||
let mut table = SymbolTable::new();
|
||||
let root_scope_id = SymbolTable::root_scope_id();
|
||||
let symbol_id_1 = table.add_symbol_to_scope(root_scope_id, "foo");
|
||||
let symbol_id_2 = table.add_symbol_to_scope(root_scope_id, "bar");
|
||||
let symbol_id_1 = table.add_or_update_symbol(root_scope_id, "foo", SymbolFlags::empty());
|
||||
let symbol_id_2 = table.add_or_update_symbol(root_scope_id, "bar", SymbolFlags::empty());
|
||||
assert_ne!(symbol_id_1, symbol_id_2);
|
||||
}
|
||||
|
||||
@@ -790,9 +913,9 @@ mod tests {
|
||||
fn add_child_scope_with_symbol() {
|
||||
let mut table = SymbolTable::new();
|
||||
let root_scope_id = SymbolTable::root_scope_id();
|
||||
let foo_symbol_top = table.add_symbol_to_scope(root_scope_id, "foo");
|
||||
let foo_symbol_top = table.add_or_update_symbol(root_scope_id, "foo", SymbolFlags::empty());
|
||||
let c_scope = table.add_child_scope(root_scope_id, "C", ScopeKind::Class);
|
||||
let foo_symbol_inner = table.add_symbol_to_scope(c_scope, "foo");
|
||||
let foo_symbol_inner = table.add_or_update_symbol(c_scope, "foo", SymbolFlags::empty());
|
||||
assert_ne!(foo_symbol_top, foo_symbol_inner);
|
||||
}
|
||||
|
||||
@@ -809,7 +932,7 @@ mod tests {
|
||||
fn symbol_from_id() {
|
||||
let mut table = SymbolTable::new();
|
||||
let root_scope_id = SymbolTable::root_scope_id();
|
||||
let foo_symbol_id = table.add_symbol_to_scope(root_scope_id, "foo");
|
||||
let foo_symbol_id = table.add_or_update_symbol(root_scope_id, "foo", SymbolFlags::empty());
|
||||
let symbol = foo_symbol_id.symbol(&table);
|
||||
assert_eq!(symbol.name.as_str(), "foo");
|
||||
}
|
||||
|
||||
@@ -37,6 +37,32 @@ impl Type {
|
||||
fn display<'a>(&'a self, store: &'a TypeStore) -> DisplayType<'a> {
|
||||
DisplayType { ty: self, store }
|
||||
}
|
||||
|
||||
pub const fn is_unbound(&self) -> bool {
|
||||
matches!(self, Type::Unbound)
|
||||
}
|
||||
|
||||
pub const fn is_unknown(&self) -> bool {
|
||||
matches!(self, Type::Unknown)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<FunctionTypeId> for Type {
|
||||
fn from(id: FunctionTypeId) -> Self {
|
||||
Type::Function(id)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<UnionTypeId> for Type {
|
||||
fn from(id: UnionTypeId) -> Self {
|
||||
Type::Union(id)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<IntersectionTypeId> for Type {
|
||||
fn from(id: IntersectionTypeId) -> Self {
|
||||
Type::Intersection(id)
|
||||
}
|
||||
}
|
||||
|
||||
// TODO: currently calling `get_function` et al and holding on to the `FunctionTypeRef` will lock a
|
||||
@@ -53,13 +79,13 @@ impl TypeStore {
|
||||
self.modules.remove(&file_id);
|
||||
}
|
||||
|
||||
pub fn cache_symbol_type(&mut self, file_id: FileId, symbol_id: SymbolId, ty: Type) {
|
||||
pub fn cache_symbol_type(&self, file_id: FileId, symbol_id: SymbolId, ty: Type) {
|
||||
self.add_or_get_module(file_id)
|
||||
.symbol_types
|
||||
.insert(symbol_id, ty);
|
||||
}
|
||||
|
||||
pub fn cache_node_type(&mut self, file_id: FileId, node_key: NodeKey, ty: Type) {
|
||||
pub fn cache_node_type(&self, file_id: FileId, node_key: NodeKey, ty: Type) {
|
||||
self.add_or_get_module(file_id)
|
||||
.node_types
|
||||
.insert(node_key, ty);
|
||||
@@ -79,7 +105,7 @@ impl TypeStore {
|
||||
.copied()
|
||||
}
|
||||
|
||||
fn add_or_get_module(&mut self, file_id: FileId) -> ModuleStoreRefMut {
|
||||
fn add_or_get_module(&self, file_id: FileId) -> ModuleStoreRefMut {
|
||||
self.modules
|
||||
.entry(file_id)
|
||||
.or_insert_with(|| ModuleTypeStore::new(file_id))
|
||||
@@ -93,12 +119,12 @@ impl TypeStore {
|
||||
self.modules.get(&file_id)
|
||||
}
|
||||
|
||||
fn add_function(&mut self, file_id: FileId, name: &str) -> FunctionTypeId {
|
||||
fn add_function(&self, file_id: FileId, name: &str) -> FunctionTypeId {
|
||||
self.add_or_get_module(file_id).add_function(name)
|
||||
}
|
||||
|
||||
fn add_class(&mut self, file_id: FileId, name: &str) -> ClassTypeId {
|
||||
self.add_or_get_module(file_id).add_class(name)
|
||||
fn add_class(&self, file_id: FileId, name: &str, bases: Vec<Type>) -> ClassTypeId {
|
||||
self.add_or_get_module(file_id).add_class(name, bases)
|
||||
}
|
||||
|
||||
fn add_union(&mut self, file_id: FileId, elems: &[Type]) -> UnionTypeId {
|
||||
@@ -290,9 +316,11 @@ impl ModuleTypeStore {
|
||||
}
|
||||
}
|
||||
|
||||
fn add_class(&mut self, name: &str) -> ClassTypeId {
|
||||
fn add_class(&mut self, name: &str, bases: Vec<Type>) -> ClassTypeId {
|
||||
let class_id = self.classes.push(ClassType {
|
||||
name: Name::new(name),
|
||||
// TODO: if no bases are given, that should imply [object]
|
||||
bases,
|
||||
});
|
||||
ClassTypeId {
|
||||
file_id: self.file_id,
|
||||
@@ -376,12 +404,17 @@ impl std::fmt::Display for DisplayType<'_> {
|
||||
#[derive(Debug)]
|
||||
pub(crate) struct ClassType {
|
||||
name: Name,
|
||||
bases: Vec<Type>,
|
||||
}
|
||||
|
||||
impl ClassType {
|
||||
fn name(&self) -> &str {
|
||||
self.name.as_str()
|
||||
}
|
||||
|
||||
fn bases(&self) -> &[Type] {
|
||||
self.bases.as_slice()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
@@ -462,10 +495,10 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn add_class() {
|
||||
let mut store = TypeStore::default();
|
||||
let store = TypeStore::default();
|
||||
let files = Files::default();
|
||||
let file_id = files.intern(Path::new("/foo"));
|
||||
let id = store.add_class(file_id, "C");
|
||||
let id = store.add_class(file_id, "C", Vec::new());
|
||||
assert_eq!(store.get_class(id).name(), "C");
|
||||
let inst = Type::Instance(id);
|
||||
assert_eq!(format!("{}", inst.display(&store)), "C");
|
||||
@@ -473,7 +506,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn add_function() {
|
||||
let mut store = TypeStore::default();
|
||||
let store = TypeStore::default();
|
||||
let files = Files::default();
|
||||
let file_id = files.intern(Path::new("/foo"));
|
||||
let id = store.add_function(file_id, "func");
|
||||
@@ -487,8 +520,8 @@ mod tests {
|
||||
let mut store = TypeStore::default();
|
||||
let files = Files::default();
|
||||
let file_id = files.intern(Path::new("/foo"));
|
||||
let c1 = store.add_class(file_id, "C1");
|
||||
let c2 = store.add_class(file_id, "C2");
|
||||
let c1 = store.add_class(file_id, "C1", Vec::new());
|
||||
let c2 = store.add_class(file_id, "C2", Vec::new());
|
||||
let elems = vec![Type::Instance(c1), Type::Instance(c2)];
|
||||
let id = store.add_union(file_id, &elems);
|
||||
assert_eq!(
|
||||
@@ -504,9 +537,9 @@ mod tests {
|
||||
let mut store = TypeStore::default();
|
||||
let files = Files::default();
|
||||
let file_id = files.intern(Path::new("/foo"));
|
||||
let c1 = store.add_class(file_id, "C1");
|
||||
let c2 = store.add_class(file_id, "C2");
|
||||
let c3 = store.add_class(file_id, "C3");
|
||||
let c1 = store.add_class(file_id, "C1", Vec::new());
|
||||
let c2 = store.add_class(file_id, "C2", Vec::new());
|
||||
let c3 = store.add_class(file_id, "C3", Vec::new());
|
||||
let pos = vec![Type::Instance(c1), Type::Instance(c2)];
|
||||
let neg = vec![Type::Instance(c3)];
|
||||
let id = store.add_intersection(file_id, &pos, &neg);
|
||||
|
||||
@@ -1,31 +1,34 @@
|
||||
#![allow(dead_code)]
|
||||
use crate::db::{HasJar, SemanticDb, SemanticJar};
|
||||
|
||||
use ruff_python_ast::AstNode;
|
||||
|
||||
use crate::db::{HasJar, QueryResult, SemanticDb, SemanticJar};
|
||||
use crate::module::ModuleName;
|
||||
use crate::symbols::{Definition, ImportFromDefinition, SymbolId};
|
||||
use crate::types::Type;
|
||||
use crate::FileId;
|
||||
use ruff_python_ast::AstNode;
|
||||
use ruff_python_ast as ast;
|
||||
|
||||
// TODO this should not take a &mut db, it should be a query, not a mutation. This means we'll need
|
||||
// to use interior mutability in TypeStore instead, and avoid races in populating the cache.
|
||||
// FIXME: Figure out proper dead-lock free synchronisation now that this takes `&db` instead of `&mut db`.
|
||||
#[tracing::instrument(level = "trace", skip(db))]
|
||||
pub fn infer_symbol_type<Db>(db: &mut Db, file_id: FileId, symbol_id: SymbolId) -> Type
|
||||
pub fn infer_symbol_type<Db>(db: &Db, file_id: FileId, symbol_id: SymbolId) -> QueryResult<Type>
|
||||
where
|
||||
Db: SemanticDb + HasJar<SemanticJar>,
|
||||
{
|
||||
let symbols = db.symbol_table(file_id);
|
||||
let defs = symbols.defs(symbol_id);
|
||||
let symbols = db.symbol_table(file_id)?;
|
||||
let defs = symbols.definitions(symbol_id);
|
||||
|
||||
if let Some(ty) = db
|
||||
.jar()
|
||||
.jar()?
|
||||
.type_store
|
||||
.get_cached_symbol_type(file_id, symbol_id)
|
||||
{
|
||||
return ty;
|
||||
return Ok(ty);
|
||||
}
|
||||
|
||||
// TODO handle multiple defs, conditional defs...
|
||||
assert_eq!(defs.len(), 1);
|
||||
let type_store = &db.jar()?.type_store;
|
||||
|
||||
let ty = match &defs[0] {
|
||||
Definition::ImportFrom(ImportFromDefinition {
|
||||
@@ -36,11 +39,11 @@ where
|
||||
// TODO relative imports
|
||||
assert!(matches!(level, 0));
|
||||
let module_name = ModuleName::new(module.as_ref().expect("TODO relative imports"));
|
||||
if let Some(module) = db.resolve_module(module_name) {
|
||||
let remote_file_id = module.path(db).file();
|
||||
let remote_symbols = db.symbol_table(remote_file_id);
|
||||
if let Some(module) = db.resolve_module(module_name)? {
|
||||
let remote_file_id = module.path(db)?.file();
|
||||
let remote_symbols = db.symbol_table(remote_file_id)?;
|
||||
if let Some(remote_symbol_id) = remote_symbols.root_symbol_id_by_name(name) {
|
||||
db.infer_symbol_type(remote_file_id, remote_symbol_id)
|
||||
db.infer_symbol_type(remote_file_id, remote_symbol_id)?
|
||||
} else {
|
||||
Type::Unknown
|
||||
}
|
||||
@@ -49,31 +52,71 @@ where
|
||||
}
|
||||
}
|
||||
Definition::ClassDef(node_key) => {
|
||||
if let Some(ty) = db
|
||||
.jar()
|
||||
.type_store
|
||||
.get_cached_node_type(file_id, node_key.erased())
|
||||
{
|
||||
if let Some(ty) = type_store.get_cached_node_type(file_id, node_key.erased()) {
|
||||
ty
|
||||
} else {
|
||||
let parsed = db.parse(file_id);
|
||||
let parsed = db.parse(file_id)?;
|
||||
let ast = parsed.ast();
|
||||
let node = node_key.resolve_unwrap(ast.as_any_node_ref());
|
||||
|
||||
let store = &mut db.jar_mut().type_store;
|
||||
let ty = Type::Class(store.add_class(file_id, &node.name.id));
|
||||
store.cache_node_type(file_id, *node_key.erased(), ty);
|
||||
let mut bases = Vec::with_capacity(node.bases().len());
|
||||
|
||||
for base in node.bases() {
|
||||
bases.push(infer_expr_type(db, file_id, base)?);
|
||||
}
|
||||
|
||||
let ty = Type::Class(type_store.add_class(file_id, &node.name.id, bases));
|
||||
type_store.cache_node_type(file_id, *node_key.erased(), ty);
|
||||
ty
|
||||
}
|
||||
}
|
||||
Definition::FunctionDef(node_key) => {
|
||||
if let Some(ty) = type_store.get_cached_node_type(file_id, node_key.erased()) {
|
||||
ty
|
||||
} else {
|
||||
let parsed = db.parse(file_id)?;
|
||||
let ast = parsed.ast();
|
||||
let node = node_key
|
||||
.resolve(ast.as_any_node_ref())
|
||||
.expect("node key should resolve");
|
||||
|
||||
let ty = type_store.add_function(file_id, &node.name.id).into();
|
||||
type_store.cache_node_type(file_id, *node_key.erased(), ty);
|
||||
ty
|
||||
}
|
||||
}
|
||||
Definition::Assignment(node_key) => {
|
||||
let parsed = db.parse(file_id)?;
|
||||
let ast = parsed.ast();
|
||||
let node = node_key.resolve_unwrap(ast.as_any_node_ref());
|
||||
// TODO handle unpacking assignment correctly
|
||||
infer_expr_type(db, file_id, &node.value)?
|
||||
}
|
||||
_ => todo!("other kinds of definitions"),
|
||||
};
|
||||
|
||||
db.jar_mut()
|
||||
.type_store
|
||||
.cache_symbol_type(file_id, symbol_id, ty);
|
||||
type_store.cache_symbol_type(file_id, symbol_id, ty);
|
||||
|
||||
// TODO record dependencies
|
||||
ty
|
||||
Ok(ty)
|
||||
}
|
||||
|
||||
fn infer_expr_type<Db>(db: &Db, file_id: FileId, expr: &ast::Expr) -> QueryResult<Type>
|
||||
where
|
||||
Db: SemanticDb + HasJar<SemanticJar>,
|
||||
{
|
||||
// TODO cache the resolution of the type on the node
|
||||
let symbols = db.symbol_table(file_id)?;
|
||||
match expr {
|
||||
ast::Expr::Name(name) => {
|
||||
if let Some(symbol_id) = symbols.root_symbol_id_by_name(&name.id) {
|
||||
db.infer_symbol_type(file_id, symbol_id)
|
||||
} else {
|
||||
Ok(Type::Unknown)
|
||||
}
|
||||
}
|
||||
_ => todo!("full expression type resolution"),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -109,33 +152,66 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn follow_import_to_class() -> std::io::Result<()> {
|
||||
let TestCase {
|
||||
src,
|
||||
mut db,
|
||||
temp_dir: _temp_dir,
|
||||
} = create_test()?;
|
||||
fn follow_import_to_class() -> anyhow::Result<()> {
|
||||
let case = create_test()?;
|
||||
let db = &case.db;
|
||||
|
||||
let a_path = src.path().join("a.py");
|
||||
let b_path = src.path().join("b.py");
|
||||
std::fs::write(a_path, "from b import C as D")?;
|
||||
let a_path = case.src.path().join("a.py");
|
||||
let b_path = case.src.path().join("b.py");
|
||||
std::fs::write(a_path, "from b import C as D; E = D")?;
|
||||
std::fs::write(b_path, "class C: pass")?;
|
||||
let a_file = db
|
||||
.resolve_module(ModuleName::new("a"))
|
||||
.resolve_module(ModuleName::new("a"))?
|
||||
.expect("module should be found")
|
||||
.path(&db)
|
||||
.path(db)?
|
||||
.file();
|
||||
let a_syms = db.symbol_table(a_file);
|
||||
let d_sym = a_syms
|
||||
.root_symbol_id_by_name("D")
|
||||
.expect("D symbol should be found");
|
||||
let a_syms = db.symbol_table(a_file)?;
|
||||
let e_sym = a_syms
|
||||
.root_symbol_id_by_name("E")
|
||||
.expect("E symbol should be found");
|
||||
|
||||
let ty = db.infer_symbol_type(a_file, d_sym);
|
||||
|
||||
let jar = HasJar::<SemanticJar>::jar(&db);
|
||||
let ty = db.infer_symbol_type(a_file, e_sym)?;
|
||||
|
||||
let jar = HasJar::<SemanticJar>::jar(db)?;
|
||||
assert!(matches!(ty, Type::Class(_)));
|
||||
assert_eq!(format!("{}", ty.display(&jar.type_store)), "Literal[C]");
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_base_class_by_name() -> anyhow::Result<()> {
|
||||
let case = create_test()?;
|
||||
let db = &case.db;
|
||||
|
||||
let path = case.src.path().join("mod.py");
|
||||
std::fs::write(path, "class Base: pass\nclass Sub(Base): pass")?;
|
||||
let file = db
|
||||
.resolve_module(ModuleName::new("mod"))?
|
||||
.expect("module should be found")
|
||||
.path(db)?
|
||||
.file();
|
||||
let syms = db.symbol_table(file)?;
|
||||
let sym = syms
|
||||
.root_symbol_id_by_name("Sub")
|
||||
.expect("Sub symbol should be found");
|
||||
|
||||
let ty = db.infer_symbol_type(file, sym)?;
|
||||
|
||||
let Type::Class(class_id) = ty else {
|
||||
panic!("Sub is not a Class")
|
||||
};
|
||||
let jar = HasJar::<SemanticJar>::jar(db)?;
|
||||
let base_names: Vec<_> = jar
|
||||
.type_store
|
||||
.get_class(class_id)
|
||||
.bases()
|
||||
.iter()
|
||||
.map(|base_ty| format!("{}", base_ty.display(&jar.type_store)))
|
||||
.collect();
|
||||
|
||||
assert_eq!(base_names, vec!["Literal[Base]"]);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -115,25 +115,25 @@ class non_keyword_abcmeta_2(abc.ABCMeta): # safe
|
||||
|
||||
|
||||
# very invalid code, but that's up to mypy et al to check
|
||||
class keyword_abc_1(metaclass=ABC): # safe
|
||||
class keyword_abc_1(metaclass=ABC): # incorrect but outside scope of this check
|
||||
def method(self):
|
||||
foo()
|
||||
|
||||
|
||||
class keyword_abc_2(metaclass=abc.ABC): # safe
|
||||
class keyword_abc_2(metaclass=abc.ABC): # incorrect but outside scope of this check
|
||||
def method(self):
|
||||
foo()
|
||||
|
||||
|
||||
class abc_set_class_variable_1(ABC): # safe
|
||||
class abc_set_class_variable_1(ABC): # safe (abstract attribute)
|
||||
foo: int
|
||||
|
||||
|
||||
class abc_set_class_variable_2(ABC): # safe
|
||||
class abc_set_class_variable_2(ABC): # error (not an abstract attribute)
|
||||
foo = 2
|
||||
|
||||
|
||||
class abc_set_class_variable_3(ABC): # safe
|
||||
class abc_set_class_variable_3(ABC): # error (not an abstract attribute)
|
||||
foo: int = 2
|
||||
|
||||
|
||||
|
||||
@@ -28,3 +28,13 @@ class MyClassBase(metaclass=ABCMeta):
|
||||
@abstractmethod
|
||||
def example(self, value):
|
||||
"""Setter."""
|
||||
|
||||
|
||||
class VariadicParameters:
|
||||
@property
|
||||
def attribute_var_args(self, *args): # [property-with-parameters]
|
||||
return sum(args)
|
||||
|
||||
@property
|
||||
def attribute_var_kwargs(self, **kwargs): #[property-with-parameters]
|
||||
return {key: value * 2 for key, value in kwargs.items()}
|
||||
|
||||
@@ -73,3 +73,8 @@ def foo():
|
||||
|
||||
async def test():
|
||||
return [check async for check in async_func()]
|
||||
|
||||
|
||||
async def test() -> str:
|
||||
vals = [str(val) for val in await async_func(1)]
|
||||
return ",".join(vals)
|
||||
|
||||
@@ -31,8 +31,8 @@ use std::path::Path;
|
||||
use itertools::Itertools;
|
||||
use log::debug;
|
||||
use ruff_python_ast::{
|
||||
self as ast, Comprehension, ElifElseClause, ExceptHandler, Expr, ExprContext, FStringElement,
|
||||
Keyword, MatchCase, Parameter, ParameterWithDefault, Parameters, Pattern, Stmt, Suite, UnaryOp,
|
||||
self as ast, AnyParameterRef, Comprehension, ElifElseClause, ExceptHandler, Expr, ExprContext,
|
||||
FStringElement, Keyword, MatchCase, Parameter, Parameters, Pattern, Stmt, Suite, UnaryOp,
|
||||
};
|
||||
use ruff_text_size::{Ranged, TextRange, TextSize};
|
||||
|
||||
@@ -604,15 +604,11 @@ impl<'a> Visitor<'a> for Checker<'a> {
|
||||
self.visit_type_params(type_params);
|
||||
}
|
||||
|
||||
for parameter_with_default in parameters
|
||||
.posonlyargs
|
||||
.iter()
|
||||
.chain(¶meters.args)
|
||||
.chain(¶meters.kwonlyargs)
|
||||
{
|
||||
if let Some(expr) = ¶meter_with_default.parameter.annotation {
|
||||
if singledispatch {
|
||||
for parameter in &**parameters {
|
||||
if let Some(expr) = parameter.annotation() {
|
||||
if singledispatch && !parameter.is_variadic() {
|
||||
self.visit_runtime_required_annotation(expr);
|
||||
singledispatch = false;
|
||||
} else {
|
||||
match annotation {
|
||||
AnnotationContext::RuntimeRequired => {
|
||||
@@ -625,42 +621,11 @@ impl<'a> Visitor<'a> for Checker<'a> {
|
||||
self.visit_annotation(expr);
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
}
|
||||
if let Some(expr) = ¶meter_with_default.default {
|
||||
if let Some(expr) = parameter.default() {
|
||||
self.visit_expr(expr);
|
||||
}
|
||||
singledispatch = false;
|
||||
}
|
||||
if let Some(arg) = ¶meters.vararg {
|
||||
if let Some(expr) = &arg.annotation {
|
||||
match annotation {
|
||||
AnnotationContext::RuntimeRequired => {
|
||||
self.visit_runtime_required_annotation(expr);
|
||||
}
|
||||
AnnotationContext::RuntimeEvaluated => {
|
||||
self.visit_runtime_evaluated_annotation(expr);
|
||||
}
|
||||
AnnotationContext::TypingOnly => {
|
||||
self.visit_annotation(expr);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if let Some(arg) = ¶meters.kwarg {
|
||||
if let Some(expr) = &arg.annotation {
|
||||
match annotation {
|
||||
AnnotationContext::RuntimeRequired => {
|
||||
self.visit_runtime_required_annotation(expr);
|
||||
}
|
||||
AnnotationContext::RuntimeEvaluated => {
|
||||
self.visit_runtime_evaluated_annotation(expr);
|
||||
}
|
||||
AnnotationContext::TypingOnly => {
|
||||
self.visit_annotation(expr);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
for expr in returns {
|
||||
match annotation {
|
||||
@@ -1043,19 +1008,11 @@ impl<'a> Visitor<'a> for Checker<'a> {
|
||||
) => {
|
||||
// Visit the default arguments, but avoid the body, which will be deferred.
|
||||
if let Some(parameters) = parameters {
|
||||
for ParameterWithDefault {
|
||||
default,
|
||||
parameter: _,
|
||||
range: _,
|
||||
} in parameters
|
||||
.posonlyargs
|
||||
.iter()
|
||||
.chain(¶meters.args)
|
||||
.chain(¶meters.kwonlyargs)
|
||||
for default in parameters
|
||||
.iter_non_variadic_params()
|
||||
.filter_map(|param| param.default.as_deref())
|
||||
{
|
||||
if let Some(expr) = &default {
|
||||
self.visit_expr(expr);
|
||||
}
|
||||
self.visit_expr(default);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1483,20 +1440,8 @@ impl<'a> Visitor<'a> for Checker<'a> {
|
||||
// Step 1: Binding.
|
||||
// Bind, but intentionally avoid walking default expressions, as we handle them
|
||||
// upstream.
|
||||
for parameter_with_default in ¶meters.posonlyargs {
|
||||
self.visit_parameter(¶meter_with_default.parameter);
|
||||
}
|
||||
for parameter_with_default in ¶meters.args {
|
||||
self.visit_parameter(¶meter_with_default.parameter);
|
||||
}
|
||||
if let Some(arg) = ¶meters.vararg {
|
||||
self.visit_parameter(arg);
|
||||
}
|
||||
for parameter_with_default in ¶meters.kwonlyargs {
|
||||
self.visit_parameter(¶meter_with_default.parameter);
|
||||
}
|
||||
if let Some(arg) = ¶meters.kwarg {
|
||||
self.visit_parameter(arg);
|
||||
for parameter in parameters.iter().map(AnyParameterRef::as_parameter) {
|
||||
self.visit_parameter(parameter);
|
||||
}
|
||||
|
||||
// Step 4: Analysis
|
||||
|
||||
@@ -971,32 +971,32 @@ pub fn code_to_rule(linter: Linter, code: &str) -> Option<(RuleGroup, Rule)> {
|
||||
(Ruff, "100") => (RuleGroup::Stable, rules::ruff::rules::UnusedNOQA),
|
||||
(Ruff, "101") => (RuleGroup::Preview, rules::ruff::rules::RedirectedNOQA),
|
||||
(Ruff, "200") => (RuleGroup::Stable, rules::ruff::rules::InvalidPyprojectToml),
|
||||
#[cfg(feature = "test-rules")]
|
||||
#[cfg(any(feature = "test-rules", test))]
|
||||
(Ruff, "900") => (RuleGroup::Stable, rules::ruff::rules::StableTestRule),
|
||||
#[cfg(feature = "test-rules")]
|
||||
#[cfg(any(feature = "test-rules", test))]
|
||||
(Ruff, "901") => (RuleGroup::Stable, rules::ruff::rules::StableTestRuleSafeFix),
|
||||
#[cfg(feature = "test-rules")]
|
||||
#[cfg(any(feature = "test-rules", test))]
|
||||
(Ruff, "902") => (RuleGroup::Stable, rules::ruff::rules::StableTestRuleUnsafeFix),
|
||||
#[cfg(feature = "test-rules")]
|
||||
#[cfg(any(feature = "test-rules", test))]
|
||||
(Ruff, "903") => (RuleGroup::Stable, rules::ruff::rules::StableTestRuleDisplayOnlyFix),
|
||||
#[cfg(feature = "test-rules")]
|
||||
#[cfg(any(feature = "test-rules", test))]
|
||||
(Ruff, "911") => (RuleGroup::Preview, rules::ruff::rules::PreviewTestRule),
|
||||
#[cfg(feature = "test-rules")]
|
||||
#[cfg(any(feature = "test-rules", test))]
|
||||
#[allow(deprecated)]
|
||||
(Ruff, "912") => (RuleGroup::Nursery, rules::ruff::rules::NurseryTestRule),
|
||||
#[cfg(feature = "test-rules")]
|
||||
#[cfg(any(feature = "test-rules", test))]
|
||||
(Ruff, "920") => (RuleGroup::Deprecated, rules::ruff::rules::DeprecatedTestRule),
|
||||
#[cfg(feature = "test-rules")]
|
||||
#[cfg(any(feature = "test-rules", test))]
|
||||
(Ruff, "921") => (RuleGroup::Deprecated, rules::ruff::rules::AnotherDeprecatedTestRule),
|
||||
#[cfg(feature = "test-rules")]
|
||||
#[cfg(any(feature = "test-rules", test))]
|
||||
(Ruff, "930") => (RuleGroup::Removed, rules::ruff::rules::RemovedTestRule),
|
||||
#[cfg(feature = "test-rules")]
|
||||
#[cfg(any(feature = "test-rules", test))]
|
||||
(Ruff, "931") => (RuleGroup::Removed, rules::ruff::rules::AnotherRemovedTestRule),
|
||||
#[cfg(feature = "test-rules")]
|
||||
#[cfg(any(feature = "test-rules", test))]
|
||||
(Ruff, "940") => (RuleGroup::Removed, rules::ruff::rules::RedirectedFromTestRule),
|
||||
#[cfg(feature = "test-rules")]
|
||||
#[cfg(any(feature = "test-rules", test))]
|
||||
(Ruff, "950") => (RuleGroup::Stable, rules::ruff::rules::RedirectedToTestRule),
|
||||
#[cfg(feature = "test-rules")]
|
||||
#[cfg(any(feature = "test-rules", test))]
|
||||
(Ruff, "960") => (RuleGroup::Removed, rules::ruff::rules::RedirectedFromPrefixTestRule),
|
||||
|
||||
|
||||
|
||||
@@ -33,7 +33,7 @@ use crate::message::Message;
|
||||
use crate::noqa::add_noqa;
|
||||
use crate::registry::{AsRule, Rule, RuleSet};
|
||||
use crate::rules::pycodestyle;
|
||||
#[cfg(feature = "test-rules")]
|
||||
#[cfg(any(feature = "test-rules", test))]
|
||||
use crate::rules::ruff::rules::test_rules::{self, TestRule, TEST_RULES};
|
||||
use crate::settings::types::UnsafeFixes;
|
||||
use crate::settings::{flags, LinterSettings};
|
||||
@@ -218,7 +218,7 @@ pub fn check_path(
|
||||
}
|
||||
|
||||
// Raise violations for internal test rules
|
||||
#[cfg(feature = "test-rules")]
|
||||
#[cfg(any(feature = "test-rules", test))]
|
||||
{
|
||||
for test_rule in TEST_RULES {
|
||||
if !settings.rules.enabled(*test_rule) {
|
||||
|
||||
@@ -104,10 +104,10 @@ static REDIRECTS: Lazy<HashMap<&'static str, &'static str>> = Lazy::new(|| {
|
||||
("PGH001", "S307"),
|
||||
("PGH002", "G010"),
|
||||
// Test redirect by exact code
|
||||
#[cfg(feature = "test-rules")]
|
||||
#[cfg(any(feature = "test-rules", test))]
|
||||
("RUF940", "RUF950"),
|
||||
// Test redirect by prefix
|
||||
#[cfg(feature = "test-rules")]
|
||||
#[cfg(any(feature = "test-rules", test))]
|
||||
("RUF96", "RUF95"),
|
||||
// See: https://github.com/astral-sh/ruff/issues/10791
|
||||
("PLW0117", "PLW0177"),
|
||||
|
||||
@@ -323,7 +323,7 @@ mod schema {
|
||||
})
|
||||
.filter(|_rule| {
|
||||
// Filter out all test-only rules
|
||||
#[cfg(feature = "test-rules")]
|
||||
#[cfg(any(feature = "test-rules", test))]
|
||||
#[allow(clippy::used_underscore_binding)]
|
||||
if _rule.starts_with("RUF9") {
|
||||
return false;
|
||||
|
||||
@@ -582,6 +582,7 @@ fn is_stub_function(function_def: &ast::StmtFunctionDef, checker: &Checker) -> b
|
||||
}
|
||||
|
||||
/// Generate flake8-annotation checks for a given `Definition`.
|
||||
/// ANN001, ANN401
|
||||
pub(crate) fn definition(
|
||||
checker: &Checker,
|
||||
definition: &Definition,
|
||||
@@ -615,23 +616,14 @@ pub(crate) fn definition(
|
||||
|
||||
let is_overridden = visibility::is_override(decorator_list, checker.semantic());
|
||||
|
||||
// ANN001, ANN401
|
||||
// If this is a non-static method, skip `cls` or `self`.
|
||||
for ParameterWithDefault {
|
||||
parameter,
|
||||
default: _,
|
||||
range: _,
|
||||
} in parameters
|
||||
.posonlyargs
|
||||
.iter()
|
||||
.chain(¶meters.args)
|
||||
.chain(¶meters.kwonlyargs)
|
||||
.skip(
|
||||
// If this is a non-static method, skip `cls` or `self`.
|
||||
usize::from(
|
||||
is_method && !visibility::is_staticmethod(decorator_list, checker.semantic()),
|
||||
),
|
||||
)
|
||||
{
|
||||
} in parameters.iter_non_variadic_params().skip(usize::from(
|
||||
is_method && !visibility::is_staticmethod(decorator_list, checker.semantic()),
|
||||
)) {
|
||||
// ANN401 for dynamically typed parameters
|
||||
if let Some(annotation) = ¶meter.annotation {
|
||||
has_any_typed_arg = true;
|
||||
|
||||
@@ -74,11 +74,7 @@ pub(crate) fn hardcoded_password_default(checker: &mut Checker, parameters: &Par
|
||||
parameter,
|
||||
default,
|
||||
range: _,
|
||||
} in parameters
|
||||
.posonlyargs
|
||||
.iter()
|
||||
.chain(¶meters.args)
|
||||
.chain(¶meters.kwonlyargs)
|
||||
} in parameters.iter_non_variadic_params()
|
||||
{
|
||||
let Some(default) = default else {
|
||||
continue;
|
||||
|
||||
@@ -49,44 +49,35 @@ impl Violation for SslWithBadDefaults {
|
||||
|
||||
/// S503
|
||||
pub(crate) fn ssl_with_bad_defaults(checker: &mut Checker, function_def: &StmtFunctionDef) {
|
||||
function_def
|
||||
for default in function_def
|
||||
.parameters
|
||||
.posonlyargs
|
||||
.iter()
|
||||
.chain(
|
||||
function_def
|
||||
.parameters
|
||||
.args
|
||||
.iter()
|
||||
.chain(function_def.parameters.kwonlyargs.iter()),
|
||||
)
|
||||
.for_each(|param| {
|
||||
if let Some(default) = ¶m.default {
|
||||
match default.as_ref() {
|
||||
Expr::Name(ast::ExprName { id, range, .. }) => {
|
||||
if is_insecure_protocol(id.as_str()) {
|
||||
checker.diagnostics.push(Diagnostic::new(
|
||||
SslWithBadDefaults {
|
||||
protocol: id.to_string(),
|
||||
},
|
||||
*range,
|
||||
));
|
||||
}
|
||||
}
|
||||
Expr::Attribute(ast::ExprAttribute { attr, range, .. }) => {
|
||||
if is_insecure_protocol(attr.as_str()) {
|
||||
checker.diagnostics.push(Diagnostic::new(
|
||||
SslWithBadDefaults {
|
||||
protocol: attr.to_string(),
|
||||
},
|
||||
*range,
|
||||
));
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
.iter_non_variadic_params()
|
||||
.filter_map(|param| param.default.as_deref())
|
||||
{
|
||||
match default {
|
||||
Expr::Name(ast::ExprName { id, range, .. }) => {
|
||||
if is_insecure_protocol(id.as_str()) {
|
||||
checker.diagnostics.push(Diagnostic::new(
|
||||
SslWithBadDefaults {
|
||||
protocol: id.to_string(),
|
||||
},
|
||||
*range,
|
||||
));
|
||||
}
|
||||
}
|
||||
});
|
||||
Expr::Attribute(ast::ExprAttribute { attr, range, .. }) => {
|
||||
if is_insecure_protocol(attr.as_str()) {
|
||||
checker.diagnostics.push(Diagnostic::new(
|
||||
SslWithBadDefaults {
|
||||
protocol: attr.to_string(),
|
||||
},
|
||||
*range,
|
||||
));
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns `true` if the given protocol name is insecure.
|
||||
|
||||
@@ -56,6 +56,7 @@ impl Violation for AbstractBaseClassWithoutAbstractMethod {
|
||||
format!("`{name}` is an abstract base class, but it has no abstract methods")
|
||||
}
|
||||
}
|
||||
|
||||
/// ## What it does
|
||||
/// Checks for empty methods in abstract base classes without an abstract
|
||||
/// decorator.
|
||||
@@ -156,8 +157,13 @@ pub(crate) fn abstract_base_class(
|
||||
let mut has_abstract_method = false;
|
||||
for stmt in body {
|
||||
// https://github.com/PyCQA/flake8-bugbear/issues/293
|
||||
// Ignore abc's that declares a class attribute that must be set
|
||||
if let Stmt::AnnAssign(_) | Stmt::Assign(_) = stmt {
|
||||
// If an ABC declares an attribute by providing a type annotation
|
||||
// but does not actually assign a value for that attribute,
|
||||
// assume it is intended to be an "abstract attribute"
|
||||
if matches!(
|
||||
stmt,
|
||||
Stmt::AnnAssign(ast::StmtAnnAssign { value: None, .. })
|
||||
) {
|
||||
has_abstract_method = true;
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -139,11 +139,7 @@ pub(crate) fn function_call_in_argument_default(checker: &mut Checker, parameter
|
||||
default,
|
||||
parameter,
|
||||
range: _,
|
||||
} in parameters
|
||||
.posonlyargs
|
||||
.iter()
|
||||
.chain(¶meters.args)
|
||||
.chain(¶meters.kwonlyargs)
|
||||
} in parameters.iter_non_variadic_params()
|
||||
{
|
||||
if let Some(expr) = &default {
|
||||
if !parameter.annotation.as_ref().is_some_and(|expr| {
|
||||
|
||||
@@ -105,11 +105,7 @@ impl<'a> Visitor<'a> for NameFinder<'a> {
|
||||
parameter,
|
||||
default: _,
|
||||
range: _,
|
||||
} in parameters
|
||||
.posonlyargs
|
||||
.iter()
|
||||
.chain(¶meters.args)
|
||||
.chain(¶meters.kwonlyargs)
|
||||
} in parameters.iter_non_variadic_params()
|
||||
{
|
||||
self.names.remove(parameter.name.as_str());
|
||||
}
|
||||
|
||||
@@ -89,12 +89,7 @@ pub(crate) fn mutable_argument_default(checker: &mut Checker, function_def: &ast
|
||||
parameter,
|
||||
default,
|
||||
range: _,
|
||||
} in function_def
|
||||
.parameters
|
||||
.posonlyargs
|
||||
.iter()
|
||||
.chain(&function_def.parameters.args)
|
||||
.chain(&function_def.parameters.kwonlyargs)
|
||||
} in function_def.parameters.iter_non_variadic_params()
|
||||
{
|
||||
let Some(default) = default else {
|
||||
continue;
|
||||
|
||||
@@ -41,6 +41,20 @@ B024.py:92:7: B024 `notabc_Base_1` is an abstract base class, but it has no abst
|
||||
94 | foo()
|
||||
|
|
||||
|
||||
B024.py:132:7: B024 `abc_set_class_variable_2` is an abstract base class, but it has no abstract methods
|
||||
|
|
||||
132 | class abc_set_class_variable_2(ABC): # error (not an abstract attribute)
|
||||
| ^^^^^^^^^^^^^^^^^^^^^^^^ B024
|
||||
133 | foo = 2
|
||||
|
|
||||
|
||||
B024.py:136:7: B024 `abc_set_class_variable_3` is an abstract base class, but it has no abstract methods
|
||||
|
|
||||
136 | class abc_set_class_variable_3(ABC): # error (not an abstract attribute)
|
||||
| ^^^^^^^^^^^^^^^^^^^^^^^^ B024
|
||||
137 | foo: int = 2
|
||||
|
|
||||
|
||||
B024.py:141:7: B024 `abc_set_class_variable_4` is an abstract base class, but it has no abstract methods
|
||||
|
|
||||
140 | # this doesn't actually declare a class variable, it's just an expression
|
||||
@@ -48,5 +62,3 @@ B024.py:141:7: B024 `abc_set_class_variable_4` is an abstract base class, but it
|
||||
| ^^^^^^^^^^^^^^^^^^^^^^^^ B024
|
||||
142 | foo
|
||||
|
|
||||
|
||||
|
||||
|
||||
@@ -107,10 +107,7 @@ pub(crate) fn unnecessary_map(
|
||||
if parameters.as_ref().is_some_and(|parameters| {
|
||||
late_binding(parameters, body)
|
||||
|| parameters
|
||||
.posonlyargs
|
||||
.iter()
|
||||
.chain(¶meters.args)
|
||||
.chain(¶meters.kwonlyargs)
|
||||
.iter_non_variadic_params()
|
||||
.any(|param| param.default.is_some())
|
||||
|| parameters.vararg.is_some()
|
||||
|| parameters.kwarg.is_some()
|
||||
@@ -152,10 +149,7 @@ pub(crate) fn unnecessary_map(
|
||||
if parameters.as_ref().is_some_and(|parameters| {
|
||||
late_binding(parameters, body)
|
||||
|| parameters
|
||||
.posonlyargs
|
||||
.iter()
|
||||
.chain(¶meters.args)
|
||||
.chain(¶meters.kwonlyargs)
|
||||
.iter_non_variadic_params()
|
||||
.any(|param| param.default.is_some())
|
||||
|| parameters.vararg.is_some()
|
||||
|| parameters.kwarg.is_some()
|
||||
@@ -207,10 +201,7 @@ pub(crate) fn unnecessary_map(
|
||||
if parameters.as_ref().is_some_and(|parameters| {
|
||||
late_binding(parameters, body)
|
||||
|| parameters
|
||||
.posonlyargs
|
||||
.iter()
|
||||
.chain(¶meters.args)
|
||||
.chain(¶meters.kwonlyargs)
|
||||
.iter_non_variadic_params()
|
||||
.any(|param| param.default.is_some())
|
||||
|| parameters.vararg.is_some()
|
||||
|| parameters.kwarg.is_some()
|
||||
|
||||
@@ -2,7 +2,7 @@ use std::fmt;
|
||||
|
||||
use ruff_diagnostics::{Diagnostic, Violation};
|
||||
use ruff_macros::{derive_message_formats, violation};
|
||||
use ruff_python_ast::{Expr, Parameters};
|
||||
use ruff_python_ast::{AnyParameterRef, Parameters};
|
||||
use ruff_text_size::Ranged;
|
||||
|
||||
use crate::checkers::ast::Checker;
|
||||
@@ -58,43 +58,21 @@ pub(crate) fn no_return_argument_annotation(checker: &mut Checker, parameters: &
|
||||
// Ex) def func(arg: NoReturn): ...
|
||||
// Ex) def func(arg: NoReturn, /): ...
|
||||
// Ex) def func(*, arg: NoReturn): ...
|
||||
for annotation in parameters
|
||||
.posonlyargs
|
||||
.iter()
|
||||
.chain(¶meters.args)
|
||||
.chain(¶meters.kwonlyargs)
|
||||
.filter_map(|arg| arg.parameter.annotation.as_ref())
|
||||
{
|
||||
check_no_return_argument_annotation(checker, annotation);
|
||||
}
|
||||
|
||||
// Ex) def func(*args: NoReturn): ...
|
||||
if let Some(arg) = ¶meters.vararg {
|
||||
if let Some(annotation) = &arg.annotation {
|
||||
check_no_return_argument_annotation(checker, annotation);
|
||||
}
|
||||
}
|
||||
|
||||
// Ex) def func(**kwargs: NoReturn): ...
|
||||
if let Some(arg) = ¶meters.kwarg {
|
||||
if let Some(annotation) = &arg.annotation {
|
||||
check_no_return_argument_annotation(checker, annotation);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn check_no_return_argument_annotation(checker: &mut Checker, annotation: &Expr) {
|
||||
if checker.semantic().match_typing_expr(annotation, "NoReturn") {
|
||||
checker.diagnostics.push(Diagnostic::new(
|
||||
NoReturnArgumentAnnotationInStub {
|
||||
module: if checker.settings.target_version >= Py311 {
|
||||
TypingModule::Typing
|
||||
} else {
|
||||
TypingModule::TypingExtensions
|
||||
for annotation in parameters.iter().filter_map(AnyParameterRef::annotation) {
|
||||
if checker.semantic().match_typing_expr(annotation, "NoReturn") {
|
||||
checker.diagnostics.push(Diagnostic::new(
|
||||
NoReturnArgumentAnnotationInStub {
|
||||
module: if checker.settings.target_version >= Py311 {
|
||||
TypingModule::Typing
|
||||
} else {
|
||||
TypingModule::TypingExtensions
|
||||
},
|
||||
},
|
||||
},
|
||||
annotation.range(),
|
||||
));
|
||||
annotation.range(),
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
use ruff_diagnostics::{Diagnostic, Violation};
|
||||
use ruff_macros::{derive_message_formats, violation};
|
||||
use ruff_python_ast::{Expr, Parameters};
|
||||
use ruff_python_ast::{AnyParameterRef, Expr, Parameters};
|
||||
use ruff_python_semantic::analyze::typing::traverse_union;
|
||||
use ruff_text_size::Ranged;
|
||||
|
||||
@@ -61,25 +61,7 @@ impl Violation for RedundantNumericUnion {
|
||||
|
||||
/// PYI041
|
||||
pub(crate) fn redundant_numeric_union(checker: &mut Checker, parameters: &Parameters) {
|
||||
for annotation in parameters
|
||||
.args
|
||||
.iter()
|
||||
.chain(parameters.posonlyargs.iter())
|
||||
.chain(parameters.kwonlyargs.iter())
|
||||
.filter_map(|arg| arg.parameter.annotation.as_ref())
|
||||
{
|
||||
check_annotation(checker, annotation);
|
||||
}
|
||||
|
||||
// If annotations on `args` or `kwargs` are flagged by this rule, the annotations themselves
|
||||
// are not accurate, but check them anyway. It's possible that flagging them will help the user
|
||||
// realize they're incorrect.
|
||||
for annotation in parameters
|
||||
.vararg
|
||||
.iter()
|
||||
.chain(parameters.kwarg.iter())
|
||||
.filter_map(|arg| arg.annotation.as_ref())
|
||||
{
|
||||
for annotation in parameters.iter().filter_map(AnyParameterRef::annotation) {
|
||||
check_annotation(checker, annotation);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -495,11 +495,7 @@ pub(crate) fn typed_argument_simple_defaults(checker: &mut Checker, parameters:
|
||||
parameter,
|
||||
default,
|
||||
range: _,
|
||||
} in parameters
|
||||
.posonlyargs
|
||||
.iter()
|
||||
.chain(¶meters.args)
|
||||
.chain(¶meters.kwonlyargs)
|
||||
} in parameters.iter_non_variadic_params()
|
||||
{
|
||||
let Some(default) = default else {
|
||||
continue;
|
||||
@@ -530,11 +526,7 @@ pub(crate) fn argument_simple_defaults(checker: &mut Checker, parameters: &Param
|
||||
parameter,
|
||||
default,
|
||||
range: _,
|
||||
} in parameters
|
||||
.posonlyargs
|
||||
.iter()
|
||||
.chain(¶meters.args)
|
||||
.chain(¶meters.kwonlyargs)
|
||||
} in parameters.iter_non_variadic_params()
|
||||
{
|
||||
let Some(default) = default else {
|
||||
continue;
|
||||
|
||||
@@ -8,7 +8,7 @@ use ruff_python_ast::name::UnqualifiedName;
|
||||
use ruff_python_ast::visitor;
|
||||
use ruff_python_ast::visitor::Visitor;
|
||||
use ruff_python_ast::Decorator;
|
||||
use ruff_python_ast::{self as ast, Expr, ParameterWithDefault, Parameters, Stmt};
|
||||
use ruff_python_ast::{self as ast, Expr, Parameters, Stmt};
|
||||
use ruff_python_semantic::analyze::visibility::is_abstract;
|
||||
use ruff_python_semantic::SemanticModel;
|
||||
use ruff_text_size::Ranged;
|
||||
@@ -841,28 +841,17 @@ fn check_fixture_returns(
|
||||
|
||||
/// PT019
|
||||
fn check_test_function_args(checker: &mut Checker, parameters: &Parameters) {
|
||||
parameters
|
||||
.posonlyargs
|
||||
.iter()
|
||||
.chain(¶meters.args)
|
||||
.chain(¶meters.kwonlyargs)
|
||||
.for_each(
|
||||
|ParameterWithDefault {
|
||||
parameter,
|
||||
default: _,
|
||||
range: _,
|
||||
}| {
|
||||
let name = ¶meter.name;
|
||||
if name.starts_with('_') {
|
||||
checker.diagnostics.push(Diagnostic::new(
|
||||
PytestFixtureParamWithoutValue {
|
||||
name: name.to_string(),
|
||||
},
|
||||
parameter.range(),
|
||||
));
|
||||
}
|
||||
},
|
||||
);
|
||||
for parameter in parameters.iter_non_variadic_params() {
|
||||
let name = ¶meter.parameter.name;
|
||||
if name.starts_with('_') {
|
||||
checker.diagnostics.push(Diagnostic::new(
|
||||
PytestFixtureParamWithoutValue {
|
||||
name: name.to_string(),
|
||||
},
|
||||
parameter.range(),
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// PT020
|
||||
|
||||
@@ -1,5 +1,3 @@
|
||||
use std::iter;
|
||||
|
||||
use regex::Regex;
|
||||
use ruff_python_ast as ast;
|
||||
use ruff_python_ast::{Parameter, Parameters};
|
||||
@@ -224,19 +222,20 @@ fn function(
|
||||
diagnostics: &mut Vec<Diagnostic>,
|
||||
) {
|
||||
let args = parameters
|
||||
.posonlyargs
|
||||
.iter()
|
||||
.chain(¶meters.args)
|
||||
.chain(¶meters.kwonlyargs)
|
||||
.iter_non_variadic_params()
|
||||
.map(|parameter_with_default| ¶meter_with_default.parameter)
|
||||
.chain(
|
||||
iter::once::<Option<&Parameter>>(parameters.vararg.as_deref())
|
||||
.flatten()
|
||||
parameters
|
||||
.vararg
|
||||
.as_deref()
|
||||
.into_iter()
|
||||
.skip(usize::from(ignore_variadic_names)),
|
||||
)
|
||||
.chain(
|
||||
iter::once::<Option<&Parameter>>(parameters.kwarg.as_deref())
|
||||
.flatten()
|
||||
parameters
|
||||
.kwarg
|
||||
.as_deref()
|
||||
.into_iter()
|
||||
.skip(usize::from(ignore_variadic_names)),
|
||||
);
|
||||
call(
|
||||
@@ -260,20 +259,21 @@ fn method(
|
||||
diagnostics: &mut Vec<Diagnostic>,
|
||||
) {
|
||||
let args = parameters
|
||||
.posonlyargs
|
||||
.iter()
|
||||
.chain(¶meters.args)
|
||||
.chain(¶meters.kwonlyargs)
|
||||
.iter_non_variadic_params()
|
||||
.skip(1)
|
||||
.map(|parameter_with_default| ¶meter_with_default.parameter)
|
||||
.chain(
|
||||
iter::once::<Option<&Parameter>>(parameters.vararg.as_deref())
|
||||
.flatten()
|
||||
parameters
|
||||
.vararg
|
||||
.as_deref()
|
||||
.into_iter()
|
||||
.skip(usize::from(ignore_variadic_names)),
|
||||
)
|
||||
.chain(
|
||||
iter::once::<Option<&Parameter>>(parameters.kwarg.as_deref())
|
||||
.flatten()
|
||||
parameters
|
||||
.kwarg
|
||||
.as_deref()
|
||||
.into_iter()
|
||||
.skip(usize::from(ignore_variadic_names)),
|
||||
);
|
||||
call(
|
||||
|
||||
@@ -257,15 +257,9 @@ fn rename_parameter(
|
||||
) -> Result<Option<Fix>> {
|
||||
// Don't fix if another parameter has the valid name.
|
||||
if parameters
|
||||
.posonlyargs
|
||||
.iter()
|
||||
.chain(¶meters.args)
|
||||
.chain(¶meters.kwonlyargs)
|
||||
.skip(1)
|
||||
.map(|parameter_with_default| ¶meter_with_default.parameter)
|
||||
.chain(parameters.vararg.as_deref())
|
||||
.chain(parameters.kwarg.as_deref())
|
||||
.any(|parameter| ¶meter.name == function_type.valid_first_argument_name())
|
||||
.any(|parameter| parameter.name() == function_type.valid_first_argument_name())
|
||||
{
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
@@ -199,8 +199,8 @@ fn function(
|
||||
let parameters = parameters.cloned().unwrap_or_default();
|
||||
if let Some(annotation) = annotation {
|
||||
if let Some((arg_types, return_type)) = extract_types(annotation, semantic) {
|
||||
// A `lambda` expression can only have positional and positional-only
|
||||
// arguments. The order is always positional-only first, then positional.
|
||||
// A `lambda` expression can only have positional-only and positional-or-keyword
|
||||
// arguments. The order is always positional-only first, then positional-or-keyword.
|
||||
let new_posonlyargs = parameters
|
||||
.posonlyargs
|
||||
.iter()
|
||||
|
||||
@@ -1755,23 +1755,19 @@ fn missing_args(checker: &mut Checker, docstring: &Docstring, docstrings_args: &
|
||||
|
||||
// Look for arguments that weren't included in the docstring.
|
||||
let mut missing_arg_names: FxHashSet<String> = FxHashSet::default();
|
||||
|
||||
// If this is a non-static method, skip `cls` or `self`.
|
||||
for ParameterWithDefault {
|
||||
parameter,
|
||||
default: _,
|
||||
range: _,
|
||||
} in function
|
||||
.parameters
|
||||
.posonlyargs
|
||||
.iter()
|
||||
.chain(&function.parameters.args)
|
||||
.chain(&function.parameters.kwonlyargs)
|
||||
.skip(
|
||||
// If this is a non-static method, skip `cls` or `self`.
|
||||
usize::from(
|
||||
docstring.definition.is_method()
|
||||
&& !is_staticmethod(&function.decorator_list, checker.semantic()),
|
||||
),
|
||||
)
|
||||
.iter_non_variadic_params()
|
||||
.skip(usize::from(
|
||||
docstring.definition.is_method()
|
||||
&& !is_staticmethod(&function.decorator_list, checker.semantic()),
|
||||
))
|
||||
{
|
||||
let arg_name = parameter.name.as_str();
|
||||
if !arg_name.starts_with('_') && !docstrings_args.contains(arg_name) {
|
||||
|
||||
@@ -51,20 +51,13 @@ pub(crate) fn property_with_parameters(
|
||||
decorator_list: &[Decorator],
|
||||
parameters: &Parameters,
|
||||
) {
|
||||
let semantic = checker.semantic();
|
||||
if !decorator_list
|
||||
.iter()
|
||||
.any(|decorator| semantic.match_builtin_expr(&decorator.expression, "property"))
|
||||
{
|
||||
if parameters.len() <= 1 {
|
||||
return;
|
||||
}
|
||||
if parameters
|
||||
.posonlyargs
|
||||
let semantic = checker.semantic();
|
||||
if decorator_list
|
||||
.iter()
|
||||
.chain(¶meters.args)
|
||||
.chain(¶meters.kwonlyargs)
|
||||
.count()
|
||||
> 1
|
||||
.any(|decorator| semantic.match_builtin_expr(&decorator.expression, "property"))
|
||||
{
|
||||
checker
|
||||
.diagnostics
|
||||
|
||||
@@ -61,10 +61,7 @@ impl Violation for TooManyArguments {
|
||||
pub(crate) fn too_many_arguments(checker: &mut Checker, function_def: &ast::StmtFunctionDef) {
|
||||
let num_arguments = function_def
|
||||
.parameters
|
||||
.args
|
||||
.iter()
|
||||
.chain(&function_def.parameters.kwonlyargs)
|
||||
.chain(&function_def.parameters.posonlyargs)
|
||||
.iter_non_variadic_params()
|
||||
.filter(|arg| {
|
||||
!checker
|
||||
.settings
|
||||
|
||||
@@ -62,14 +62,14 @@ pub(crate) fn too_many_positional(checker: &mut Checker, function_def: &ast::Stm
|
||||
// Count the number of positional arguments.
|
||||
let num_positional_args = function_def
|
||||
.parameters
|
||||
.args
|
||||
.posonlyargs
|
||||
.iter()
|
||||
.chain(&function_def.parameters.posonlyargs)
|
||||
.filter(|arg| {
|
||||
.chain(&function_def.parameters.args)
|
||||
.filter(|param| {
|
||||
!checker
|
||||
.settings
|
||||
.dummy_variable_rgx
|
||||
.is_match(&arg.parameter.name)
|
||||
.is_match(¶m.parameter.name)
|
||||
})
|
||||
.count();
|
||||
|
||||
|
||||
@@ -26,4 +26,19 @@ property_with_parameters.py:15:9: PLR0206 Cannot have defined parameters for pro
|
||||
16 | return param + param1
|
||||
|
|
||||
|
||||
property_with_parameters.py:35:9: PLR0206 Cannot have defined parameters for properties
|
||||
|
|
||||
33 | class VariadicParameters:
|
||||
34 | @property
|
||||
35 | def attribute_var_args(self, *args): # [property-with-parameters]
|
||||
| ^^^^^^^^^^^^^^^^^^ PLR0206
|
||||
36 | return sum(args)
|
||||
|
|
||||
|
||||
property_with_parameters.py:39:9: PLR0206 Cannot have defined parameters for properties
|
||||
|
|
||||
38 | @property
|
||||
39 | def attribute_var_kwargs(self, **kwargs): #[property-with-parameters]
|
||||
| ^^^^^^^^^^^^^^^^^^^^ PLR0206
|
||||
40 | return {key: value * 2 for key, value in kwargs.items()}
|
||||
|
|
||||
|
||||
@@ -167,11 +167,7 @@ pub(crate) fn implicit_optional(checker: &mut Checker, parameters: &Parameters)
|
||||
parameter,
|
||||
default,
|
||||
range: _,
|
||||
} in parameters
|
||||
.posonlyargs
|
||||
.iter()
|
||||
.chain(¶meters.args)
|
||||
.chain(¶meters.kwonlyargs)
|
||||
} in parameters.iter_non_variadic_params()
|
||||
{
|
||||
let Some(default) = default else { continue };
|
||||
if !default.is_none_literal_expr() {
|
||||
|
||||
@@ -21,7 +21,7 @@ pub(crate) use redirected_noqa::*;
|
||||
pub(crate) use sort_dunder_all::*;
|
||||
pub(crate) use sort_dunder_slots::*;
|
||||
pub(crate) use static_key_dict_comprehension::*;
|
||||
#[cfg(feature = "test-rules")]
|
||||
#[cfg(any(feature = "test-rules", test))]
|
||||
pub(crate) use test_rules::*;
|
||||
pub(crate) use unnecessary_dict_comprehension_for_iterable::*;
|
||||
pub(crate) use unnecessary_iterable_allocation_for_first_element::*;
|
||||
@@ -56,7 +56,7 @@ mod sort_dunder_all;
|
||||
mod sort_dunder_slots;
|
||||
mod static_key_dict_comprehension;
|
||||
mod suppression_comment_visitor;
|
||||
#[cfg(feature = "test-rules")]
|
||||
#[cfg(any(feature = "test-rules", test))]
|
||||
pub(crate) mod test_rules;
|
||||
mod unnecessary_dict_comprehension_for_iterable;
|
||||
mod unnecessary_iterable_allocation_for_first_element;
|
||||
|
||||
@@ -58,14 +58,6 @@ impl<'a> preorder::PreorderVisitor<'a> for AsyncExprVisitor {
|
||||
preorder::TraversalSignal::Traverse
|
||||
}
|
||||
}
|
||||
fn visit_expr(&mut self, expr: &'a Expr) {
|
||||
match expr {
|
||||
Expr::Await(_) => {
|
||||
self.found_await_or_async = true;
|
||||
}
|
||||
_ => preorder::walk_expr(self, expr),
|
||||
}
|
||||
}
|
||||
fn visit_stmt(&mut self, stmt: &'a Stmt) {
|
||||
match stmt {
|
||||
Stmt::With(ast::StmtWith { is_async: true, .. }) => {
|
||||
@@ -84,9 +76,19 @@ impl<'a> preorder::PreorderVisitor<'a> for AsyncExprVisitor {
|
||||
_ => preorder::walk_stmt(self, stmt),
|
||||
}
|
||||
}
|
||||
fn visit_expr(&mut self, expr: &'a Expr) {
|
||||
match expr {
|
||||
Expr::Await(_) => {
|
||||
self.found_await_or_async = true;
|
||||
}
|
||||
_ => preorder::walk_expr(self, expr),
|
||||
}
|
||||
}
|
||||
fn visit_comprehension(&mut self, comprehension: &'a ast::Comprehension) {
|
||||
if comprehension.is_async {
|
||||
self.found_await_or_async = true;
|
||||
} else {
|
||||
preorder::walk_comprehension(self, comprehension);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -348,39 +348,18 @@ pub fn any_over_stmt(stmt: &Stmt, func: &dyn Fn(&Expr) -> bool) -> bool {
|
||||
returns,
|
||||
..
|
||||
}) => {
|
||||
parameters
|
||||
.posonlyargs
|
||||
.iter()
|
||||
.chain(parameters.args.iter().chain(parameters.kwonlyargs.iter()))
|
||||
.any(|parameter| {
|
||||
parameter
|
||||
.default
|
||||
.as_ref()
|
||||
.is_some_and(|expr| any_over_expr(expr, func))
|
||||
|| parameter
|
||||
.parameter
|
||||
.annotation
|
||||
.as_ref()
|
||||
.is_some_and(|expr| any_over_expr(expr, func))
|
||||
})
|
||||
|| parameters.vararg.as_ref().is_some_and(|parameter| {
|
||||
parameter
|
||||
.annotation
|
||||
.as_ref()
|
||||
.is_some_and(|expr| any_over_expr(expr, func))
|
||||
})
|
||||
|| parameters.kwarg.as_ref().is_some_and(|parameter| {
|
||||
parameter
|
||||
.annotation
|
||||
.as_ref()
|
||||
.is_some_and(|expr| any_over_expr(expr, func))
|
||||
})
|
||||
|| type_params.as_ref().is_some_and(|type_params| {
|
||||
type_params
|
||||
.iter()
|
||||
.any(|type_param| any_over_type_param(type_param, func))
|
||||
})
|
||||
|| body.iter().any(|stmt| any_over_stmt(stmt, func))
|
||||
parameters.iter().any(|param| {
|
||||
param
|
||||
.default()
|
||||
.is_some_and(|default| any_over_expr(default, func))
|
||||
|| param
|
||||
.annotation()
|
||||
.is_some_and(|annotation| any_over_expr(annotation, func))
|
||||
}) || type_params.as_ref().is_some_and(|type_params| {
|
||||
type_params
|
||||
.iter()
|
||||
.any(|type_param| any_over_type_param(type_param, func))
|
||||
}) || body.iter().any(|stmt| any_over_stmt(stmt, func))
|
||||
|| decorator_list
|
||||
.iter()
|
||||
.any(|decorator| any_over_expr(&decorator.expression, func))
|
||||
|
||||
@@ -1,12 +1,13 @@
|
||||
use crate::visitor::preorder::PreorderVisitor;
|
||||
use crate::{
|
||||
self as ast, Alias, ArgOrKeyword, Arguments, Comprehension, Decorator, ExceptHandler, Expr,
|
||||
FStringElement, Keyword, MatchCase, Mod, Parameter, ParameterWithDefault, Parameters, Pattern,
|
||||
PatternArguments, PatternKeyword, Stmt, StmtAnnAssign, StmtAssert, StmtAssign, StmtAugAssign,
|
||||
StmtBreak, StmtClassDef, StmtContinue, StmtDelete, StmtExpr, StmtFor, StmtFunctionDef,
|
||||
StmtGlobal, StmtIf, StmtImport, StmtImportFrom, StmtIpyEscapeCommand, StmtMatch, StmtNonlocal,
|
||||
StmtPass, StmtRaise, StmtReturn, StmtTry, StmtTypeAlias, StmtWhile, StmtWith, TypeParam,
|
||||
TypeParamParamSpec, TypeParamTypeVar, TypeParamTypeVarTuple, TypeParams, WithItem,
|
||||
self as ast, Alias, AnyParameterRef, ArgOrKeyword, Arguments, Comprehension, Decorator,
|
||||
ExceptHandler, Expr, FStringElement, Keyword, MatchCase, Mod, Parameter, ParameterWithDefault,
|
||||
Parameters, Pattern, PatternArguments, PatternKeyword, Stmt, StmtAnnAssign, StmtAssert,
|
||||
StmtAssign, StmtAugAssign, StmtBreak, StmtClassDef, StmtContinue, StmtDelete, StmtExpr,
|
||||
StmtFor, StmtFunctionDef, StmtGlobal, StmtIf, StmtImport, StmtImportFrom, StmtIpyEscapeCommand,
|
||||
StmtMatch, StmtNonlocal, StmtPass, StmtRaise, StmtReturn, StmtTry, StmtTypeAlias, StmtWhile,
|
||||
StmtWith, TypeParam, TypeParamParamSpec, TypeParamTypeVar, TypeParamTypeVarTuple, TypeParams,
|
||||
WithItem,
|
||||
};
|
||||
use ruff_text_size::{Ranged, TextRange};
|
||||
use std::ptr::NonNull;
|
||||
@@ -4221,28 +4222,13 @@ impl AstNode for Parameters {
|
||||
where
|
||||
V: PreorderVisitor<'a> + ?Sized,
|
||||
{
|
||||
let ast::Parameters {
|
||||
range: _,
|
||||
posonlyargs,
|
||||
args,
|
||||
vararg,
|
||||
kwonlyargs,
|
||||
kwarg,
|
||||
} = self;
|
||||
for arg in posonlyargs.iter().chain(args) {
|
||||
visitor.visit_parameter_with_default(arg);
|
||||
}
|
||||
|
||||
if let Some(arg) = vararg {
|
||||
visitor.visit_parameter(arg);
|
||||
}
|
||||
|
||||
for arg in kwonlyargs {
|
||||
visitor.visit_parameter_with_default(arg);
|
||||
}
|
||||
|
||||
if let Some(arg) = kwarg {
|
||||
visitor.visit_parameter(arg);
|
||||
for parameter in self {
|
||||
match parameter {
|
||||
AnyParameterRef::NonVariadic(parameter_with_default) => {
|
||||
visitor.visit_parameter_with_default(parameter_with_default);
|
||||
}
|
||||
AnyParameterRef::Variadic(parameter) => visitor.visit_parameter(parameter),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
use std::fmt;
|
||||
use std::fmt::Debug;
|
||||
use std::iter::FusedIterator;
|
||||
use std::ops::Deref;
|
||||
use std::slice::{Iter, IterMut};
|
||||
use std::sync::OnceLock;
|
||||
@@ -3175,6 +3176,63 @@ pub struct Decorator {
|
||||
pub expression: Expr,
|
||||
}
|
||||
|
||||
/// Enumeration of the two kinds of parameter
|
||||
#[derive(Debug, PartialEq, Clone, Copy)]
|
||||
pub enum AnyParameterRef<'a> {
|
||||
/// Variadic parameters cannot have default values,
|
||||
/// e.g. both `*args` and `**kwargs` in the following function:
|
||||
///
|
||||
/// ```python
|
||||
/// def foo(*args, **kwargs): pass
|
||||
/// ```
|
||||
Variadic(&'a Parameter),
|
||||
|
||||
/// Non-variadic parameters can have default values,
|
||||
/// though they won't necessarily always have them:
|
||||
///
|
||||
/// ```python
|
||||
/// def bar(a=1, /, b=2, *, c=3): pass
|
||||
/// ```
|
||||
NonVariadic(&'a ParameterWithDefault),
|
||||
}
|
||||
|
||||
impl<'a> AnyParameterRef<'a> {
|
||||
pub const fn as_parameter(self) -> &'a Parameter {
|
||||
match self {
|
||||
Self::NonVariadic(param) => ¶m.parameter,
|
||||
Self::Variadic(param) => param,
|
||||
}
|
||||
}
|
||||
|
||||
pub const fn name(self) -> &'a Identifier {
|
||||
&self.as_parameter().name
|
||||
}
|
||||
|
||||
pub const fn is_variadic(self) -> bool {
|
||||
matches!(self, Self::Variadic(_))
|
||||
}
|
||||
|
||||
pub fn annotation(self) -> Option<&'a Expr> {
|
||||
self.as_parameter().annotation.as_deref()
|
||||
}
|
||||
|
||||
pub fn default(self) -> Option<&'a Expr> {
|
||||
match self {
|
||||
Self::NonVariadic(param) => param.default.as_deref(),
|
||||
Self::Variadic(_) => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Ranged for AnyParameterRef<'_> {
|
||||
fn range(&self) -> TextRange {
|
||||
match self {
|
||||
Self::NonVariadic(param) => param.range,
|
||||
Self::Variadic(param) => param.range,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// An alternative type of AST `arguments`. This is ruff_python_parser-friendly and human-friendly definition of function arguments.
|
||||
/// This form also has advantage to implement pre-order traverse.
|
||||
///
|
||||
@@ -3196,37 +3254,56 @@ pub struct Parameters {
|
||||
}
|
||||
|
||||
impl Parameters {
|
||||
/// Returns the [`ParameterWithDefault`] with the given name, or `None` if no such [`ParameterWithDefault`] exists.
|
||||
pub fn find(&self, name: &str) -> Option<&ParameterWithDefault> {
|
||||
/// Returns an iterator over all non-variadic parameters included in this [`Parameters`] node.
|
||||
///
|
||||
/// The variadic parameters (`.vararg` and `.kwarg`) can never have default values;
|
||||
/// non-variadic parameters sometimes will.
|
||||
pub fn iter_non_variadic_params(&self) -> impl Iterator<Item = &ParameterWithDefault> {
|
||||
self.posonlyargs
|
||||
.iter()
|
||||
.chain(&self.args)
|
||||
.chain(&self.kwonlyargs)
|
||||
}
|
||||
|
||||
/// Returns the [`ParameterWithDefault`] with the given name, or `None` if no such [`ParameterWithDefault`] exists.
|
||||
pub fn find(&self, name: &str) -> Option<&ParameterWithDefault> {
|
||||
self.iter_non_variadic_params()
|
||||
.find(|arg| arg.parameter.name.as_str() == name)
|
||||
}
|
||||
|
||||
/// Returns `true` if a parameter with the given name included in this [`Parameters`].
|
||||
/// Returns an iterator over all parameters included in this [`Parameters`] node.
|
||||
pub fn iter(&self) -> ParametersIterator {
|
||||
ParametersIterator::new(self)
|
||||
}
|
||||
|
||||
/// Returns the total number of parameters included in this [`Parameters`] node.
|
||||
pub fn len(&self) -> usize {
|
||||
let Parameters {
|
||||
range: _,
|
||||
posonlyargs,
|
||||
args,
|
||||
vararg,
|
||||
kwonlyargs,
|
||||
kwarg,
|
||||
} = self;
|
||||
// Safety: a Python function can have an arbitrary number of parameters,
|
||||
// so theoretically this could be a number that wouldn't fit into a usize,
|
||||
// which would lead to a panic. A Python function with that many parameters
|
||||
// is extremely unlikely outside of generated code, however, and it's even
|
||||
// more unlikely that we'd find a function with that many parameters in a
|
||||
// source-code file <=4GB large (Ruff's maximum).
|
||||
posonlyargs
|
||||
.len()
|
||||
.checked_add(args.len())
|
||||
.and_then(|length| length.checked_add(usize::from(vararg.is_some())))
|
||||
.and_then(|length| length.checked_add(kwonlyargs.len()))
|
||||
.and_then(|length| length.checked_add(usize::from(kwarg.is_some())))
|
||||
.expect("Failed to fit the number of parameters into a usize")
|
||||
}
|
||||
|
||||
/// Returns `true` if a parameter with the given name is included in this [`Parameters`].
|
||||
pub fn includes(&self, name: &str) -> bool {
|
||||
if self
|
||||
.posonlyargs
|
||||
.iter()
|
||||
.chain(&self.args)
|
||||
.chain(&self.kwonlyargs)
|
||||
.any(|arg| arg.parameter.name.as_str() == name)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
if let Some(arg) = &self.vararg {
|
||||
if arg.name.as_str() == name {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
if let Some(arg) = &self.kwarg {
|
||||
if arg.name.as_str() == name {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
false
|
||||
self.iter().any(|param| param.name() == name)
|
||||
}
|
||||
|
||||
/// Returns `true` if the [`Parameters`] is empty.
|
||||
@@ -3239,6 +3316,136 @@ impl Parameters {
|
||||
}
|
||||
}
|
||||
|
||||
pub struct ParametersIterator<'a> {
|
||||
posonlyargs: Iter<'a, ParameterWithDefault>,
|
||||
args: Iter<'a, ParameterWithDefault>,
|
||||
vararg: Option<&'a Parameter>,
|
||||
kwonlyargs: Iter<'a, ParameterWithDefault>,
|
||||
kwarg: Option<&'a Parameter>,
|
||||
}
|
||||
|
||||
impl<'a> ParametersIterator<'a> {
|
||||
fn new(parameters: &'a Parameters) -> Self {
|
||||
let Parameters {
|
||||
range: _,
|
||||
posonlyargs,
|
||||
args,
|
||||
vararg,
|
||||
kwonlyargs,
|
||||
kwarg,
|
||||
} = parameters;
|
||||
Self {
|
||||
posonlyargs: posonlyargs.iter(),
|
||||
args: args.iter(),
|
||||
vararg: vararg.as_deref(),
|
||||
kwonlyargs: kwonlyargs.iter(),
|
||||
kwarg: kwarg.as_deref(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a> Iterator for ParametersIterator<'a> {
|
||||
type Item = AnyParameterRef<'a>;
|
||||
|
||||
fn next(&mut self) -> Option<Self::Item> {
|
||||
let ParametersIterator {
|
||||
posonlyargs,
|
||||
args,
|
||||
vararg,
|
||||
kwonlyargs,
|
||||
kwarg,
|
||||
} = self;
|
||||
|
||||
if let Some(param) = posonlyargs.next() {
|
||||
return Some(AnyParameterRef::NonVariadic(param));
|
||||
}
|
||||
if let Some(param) = args.next() {
|
||||
return Some(AnyParameterRef::NonVariadic(param));
|
||||
}
|
||||
if let Some(param) = vararg.take() {
|
||||
return Some(AnyParameterRef::Variadic(param));
|
||||
}
|
||||
if let Some(param) = kwonlyargs.next() {
|
||||
return Some(AnyParameterRef::NonVariadic(param));
|
||||
}
|
||||
kwarg.take().map(AnyParameterRef::Variadic)
|
||||
}
|
||||
|
||||
fn size_hint(&self) -> (usize, Option<usize>) {
|
||||
let ParametersIterator {
|
||||
posonlyargs,
|
||||
args,
|
||||
vararg,
|
||||
kwonlyargs,
|
||||
kwarg,
|
||||
} = self;
|
||||
|
||||
let posonlyargs_len = posonlyargs.len();
|
||||
let args_len = args.len();
|
||||
let vararg_len = usize::from(vararg.is_some());
|
||||
let kwonlyargs_len = kwonlyargs.len();
|
||||
let kwarg_len = usize::from(kwarg.is_some());
|
||||
|
||||
let lower = posonlyargs_len
|
||||
.saturating_add(args_len)
|
||||
.saturating_add(vararg_len)
|
||||
.saturating_add(kwonlyargs_len)
|
||||
.saturating_add(kwarg_len);
|
||||
|
||||
let upper = posonlyargs_len
|
||||
.checked_add(args_len)
|
||||
.and_then(|length| length.checked_add(vararg_len))
|
||||
.and_then(|length| length.checked_add(kwonlyargs_len))
|
||||
.and_then(|length| length.checked_add(kwarg_len));
|
||||
|
||||
(lower, upper)
|
||||
}
|
||||
|
||||
fn last(mut self) -> Option<Self::Item> {
|
||||
self.next_back()
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a> DoubleEndedIterator for ParametersIterator<'a> {
|
||||
fn next_back(&mut self) -> Option<Self::Item> {
|
||||
let ParametersIterator {
|
||||
posonlyargs,
|
||||
args,
|
||||
vararg,
|
||||
kwonlyargs,
|
||||
kwarg,
|
||||
} = self;
|
||||
|
||||
if let Some(param) = kwarg.take() {
|
||||
return Some(AnyParameterRef::Variadic(param));
|
||||
}
|
||||
if let Some(param) = kwonlyargs.next_back() {
|
||||
return Some(AnyParameterRef::NonVariadic(param));
|
||||
}
|
||||
if let Some(param) = vararg.take() {
|
||||
return Some(AnyParameterRef::Variadic(param));
|
||||
}
|
||||
if let Some(param) = args.next_back() {
|
||||
return Some(AnyParameterRef::NonVariadic(param));
|
||||
}
|
||||
posonlyargs.next_back().map(AnyParameterRef::NonVariadic)
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a> FusedIterator for ParametersIterator<'a> {}
|
||||
|
||||
/// We rely on the same invariants outlined in the comment above `Parameters::len()`
|
||||
/// in order to implement `ExactSizeIterator` here
|
||||
impl<'a> ExactSizeIterator for ParametersIterator<'a> {}
|
||||
|
||||
impl<'a> IntoIterator for &'a Parameters {
|
||||
type IntoIter = ParametersIterator<'a>;
|
||||
type Item = AnyParameterRef<'a>;
|
||||
fn into_iter(self) -> Self::IntoIter {
|
||||
self.iter()
|
||||
}
|
||||
}
|
||||
|
||||
/// An alternative type of AST `arg`. This is used for each function argument that might have a default value.
|
||||
/// Used by `Arguments` original type.
|
||||
///
|
||||
@@ -3408,6 +3615,9 @@ impl Deref for TypeParams {
|
||||
}
|
||||
}
|
||||
|
||||
/// A suite represents a [Vec] of [Stmt].
|
||||
///
|
||||
/// See: <https://docs.python.org/3/reference/compound_stmts.html#grammar-token-python-grammar-suite>
|
||||
pub type Suite = Vec<Stmt>;
|
||||
|
||||
/// The kind of escape command as defined in [IPython Syntax] in the IPython codebase.
|
||||
|
||||
@@ -4,11 +4,11 @@ pub mod preorder;
|
||||
pub mod transformer;
|
||||
|
||||
use crate::{
|
||||
self as ast, Alias, Arguments, BoolOp, BytesLiteral, CmpOp, Comprehension, Decorator,
|
||||
ElifElseClause, ExceptHandler, Expr, ExprContext, FString, FStringElement, FStringPart,
|
||||
Keyword, MatchCase, Operator, Parameter, Parameters, Pattern, PatternArguments, PatternKeyword,
|
||||
Stmt, StringLiteral, TypeParam, TypeParamParamSpec, TypeParamTypeVar, TypeParamTypeVarTuple,
|
||||
TypeParams, UnaryOp, WithItem,
|
||||
self as ast, Alias, AnyParameterRef, Arguments, BoolOp, BytesLiteral, CmpOp, Comprehension,
|
||||
Decorator, ElifElseClause, ExceptHandler, Expr, ExprContext, FString, FStringElement,
|
||||
FStringPart, Keyword, MatchCase, Operator, Parameter, Parameters, Pattern, PatternArguments,
|
||||
PatternKeyword, Stmt, StringLiteral, TypeParam, TypeParamParamSpec, TypeParamTypeVar,
|
||||
TypeParamTypeVarTuple, TypeParams, UnaryOp, WithItem,
|
||||
};
|
||||
|
||||
/// A trait for AST visitors. Visits all nodes in the AST recursively in evaluation-order.
|
||||
@@ -607,36 +607,15 @@ pub fn walk_arguments<'a, V: Visitor<'a> + ?Sized>(visitor: &mut V, arguments: &
|
||||
|
||||
pub fn walk_parameters<'a, V: Visitor<'a> + ?Sized>(visitor: &mut V, parameters: &'a Parameters) {
|
||||
// Defaults are evaluated before annotations.
|
||||
for arg in ¶meters.posonlyargs {
|
||||
if let Some(default) = &arg.default {
|
||||
visitor.visit_expr(default);
|
||||
}
|
||||
}
|
||||
for arg in ¶meters.args {
|
||||
if let Some(default) = &arg.default {
|
||||
visitor.visit_expr(default);
|
||||
}
|
||||
}
|
||||
for arg in ¶meters.kwonlyargs {
|
||||
if let Some(default) = &arg.default {
|
||||
visitor.visit_expr(default);
|
||||
}
|
||||
for default in parameters
|
||||
.iter_non_variadic_params()
|
||||
.filter_map(|param| param.default.as_deref())
|
||||
{
|
||||
visitor.visit_expr(default);
|
||||
}
|
||||
|
||||
for arg in ¶meters.posonlyargs {
|
||||
visitor.visit_parameter(&arg.parameter);
|
||||
}
|
||||
for arg in ¶meters.args {
|
||||
visitor.visit_parameter(&arg.parameter);
|
||||
}
|
||||
if let Some(arg) = ¶meters.vararg {
|
||||
visitor.visit_parameter(arg);
|
||||
}
|
||||
for arg in ¶meters.kwonlyargs {
|
||||
visitor.visit_parameter(&arg.parameter);
|
||||
}
|
||||
if let Some(arg) = ¶meters.kwarg {
|
||||
visitor.visit_parameter(arg);
|
||||
for parameter in parameters.iter().map(AnyParameterRef::as_parameter) {
|
||||
visitor.visit_parameter(parameter);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -240,11 +240,7 @@ impl FormatNodeRule<Parameters> for FormatParameters {
|
||||
Ok(())
|
||||
});
|
||||
|
||||
let num_parameters = posonlyargs.len()
|
||||
+ args.len()
|
||||
+ usize::from(vararg.is_some())
|
||||
+ kwonlyargs.len()
|
||||
+ usize::from(kwarg.is_some());
|
||||
let num_parameters = item.len();
|
||||
|
||||
if self.parentheses == ParametersParentheses::Never {
|
||||
write!(f, [group(&format_inner), dangling_comments(dangling)])
|
||||
|
||||
@@ -52,3 +52,49 @@ Then, run the Parser test suite with the following command:
|
||||
```sh
|
||||
cargo test --package ruff_python_parser
|
||||
```
|
||||
|
||||
### Python-based fuzzer
|
||||
|
||||
The Ruff project includes a Python-based fuzzer that can be used to run the parser on
|
||||
randomly generated (but syntactically valid) Python source code files.
|
||||
|
||||
To run the fuzzer, first install the required dependencies:
|
||||
|
||||
```sh
|
||||
uv pip install -r scripts/fuzz-parser/requirements.txt
|
||||
```
|
||||
|
||||
Then, run the fuzzer with the following command:
|
||||
|
||||
```sh
|
||||
python scripts/fuzz-parser/fuzz.py
|
||||
```
|
||||
|
||||
Refer to the [fuzz.py](https://github.com/astral-sh/ruff/blob/main/scripts/fuzz-parser/fuzz.py)
|
||||
script for more information or use the `--help` flag to see the available options.
|
||||
|
||||
#### CI
|
||||
|
||||
The fuzzer is run as part of the CI pipeline. The purpose of running the fuzzer in the CI is to
|
||||
catch any regresssions introduced by any new changes to the parser. This is why the fuzzer is run on
|
||||
the same set of seeds on every run.
|
||||
|
||||
## Benchmarks
|
||||
|
||||
The `ruff_benchmark` crate can benchmark both the lexer and the parser.
|
||||
|
||||
To run the lexer benchmarks, use the following command:
|
||||
|
||||
```sh
|
||||
cargo bench --package ruff_benchmark --bench lexer
|
||||
```
|
||||
|
||||
And to run the parser benchmarks, use the following command:
|
||||
|
||||
```sh
|
||||
cargo bench --package ruff_benchmark --bench parser
|
||||
```
|
||||
|
||||
Refer to the [Benchmarking and
|
||||
Profiling](https://docs.astral.sh/ruff/contributing/#benchmark-driven-development) section in the
|
||||
contributing guide for more information.
|
||||
|
||||
22
crates/ruff_python_parser/README.md
Normal file
22
crates/ruff_python_parser/README.md
Normal file
@@ -0,0 +1,22 @@
|
||||
# Ruff Python Parser
|
||||
|
||||
Ruff's Python parser is a hand-written [recursive descent parser] which can parse
|
||||
Python source code into an Abstract Syntax Tree (AST). It also utilizes the [Pratt
|
||||
parsing](https://matklad.github.io/2020/04/13/simple-but-powerful-pratt-parsing.html)
|
||||
technique to parse expressions with different [precedence](https://docs.python.org/3/reference/expressions.html#operator-precedence).
|
||||
|
||||
Try out the parser in the [playground](https://play.ruff.rs/?secondary=AST).
|
||||
|
||||
## Python version support
|
||||
|
||||
The parser supports the latest Python syntax, which is currently Python 3.12.
|
||||
It does not throw syntax errors if it encounters a syntax feature that is not
|
||||
supported by the [`target-version`](https://docs.astral.sh/ruff/settings/#target-version).
|
||||
This will be fixed in a future release (see <https://github.com/astral-sh/ruff/issues/6591>).
|
||||
|
||||
## Contributing
|
||||
|
||||
Refer to the [contributing guidelines](./CONTRIBUTING.md) to get started and GitHub issues with the
|
||||
[parser label](https://github.com/astral-sh/ruff/issues?q=is:open+is:issue+label:parser) for issues that need help.
|
||||
|
||||
[recursive descent parser]: https://en.wikipedia.org/wiki/Recursive_descent_parser
|
||||
@@ -1,7 +1,7 @@
|
||||
//! This crate can be used to parse Python source code into an Abstract
|
||||
//! Syntax Tree.
|
||||
//!
|
||||
//! ## Overview:
|
||||
//! ## Overview
|
||||
//!
|
||||
//! The process by which source code is parsed into an AST can be broken down
|
||||
//! into two general stages: [lexical analysis] and [parsing].
|
||||
@@ -15,7 +15,7 @@
|
||||
//! Name("print"), LeftParen, String("Hello world"), RightParen
|
||||
//! ```
|
||||
//!
|
||||
//! these tokens are then consumed by the `ruff_python_parser`, which matches them against a set of
|
||||
//! These tokens are then consumed by the `ruff_python_parser`, which matches them against a set of
|
||||
//! grammar rules to verify that the source code is syntactically valid and to construct
|
||||
//! an AST that represents the source code.
|
||||
//!
|
||||
@@ -48,16 +48,16 @@
|
||||
//! },
|
||||
//!```
|
||||
//!
|
||||
//! Note: The Tokens/ASTs shown above are not the exact tokens/ASTs generated by the `ruff_python_parser`.
|
||||
//! **Note:** The Tokens/ASTs shown above are not the exact tokens/ASTs generated by the `ruff_python_parser`.
|
||||
//! Refer to the [playground](https://play.ruff.rs) for the correct representation.
|
||||
//!
|
||||
//! ## Source code layout:
|
||||
//! ## Source code layout
|
||||
//!
|
||||
//! The functionality of this crate is split into several modules:
|
||||
//!
|
||||
//! - token: This module contains the definition of the tokens that are generated by the lexer.
|
||||
//! - [lexer]: This module contains the lexer and is responsible for generating the tokens.
|
||||
//! - `ruff_python_parser`: This module contains an interface to the `ruff_python_parser` and is responsible for generating the AST.
|
||||
//! - Functions and strings have special parsing requirements that are handled in additional files.
|
||||
//! - parser: This module contains an interface to the [Program] and is responsible for generating the AST.
|
||||
//! - mode: This module contains the definition of the different modes that the `ruff_python_parser` can be in.
|
||||
//!
|
||||
//! # Examples
|
||||
@@ -78,14 +78,15 @@
|
||||
//! These tokens can be directly fed into the `ruff_python_parser` to generate an AST:
|
||||
//!
|
||||
//! ```
|
||||
//! use ruff_python_parser::{Mode, parse_tokens, tokenize_all};
|
||||
//! use ruff_python_parser::lexer::lex;
|
||||
//! use ruff_python_parser::{Mode, parse_tokens};
|
||||
//!
|
||||
//! let python_source = r#"
|
||||
//! def is_odd(i):
|
||||
//! return bool(i & 1)
|
||||
//! "#;
|
||||
//! let tokens = tokenize_all(python_source, Mode::Module);
|
||||
//! let ast = parse_tokens(tokens, python_source, Mode::Module);
|
||||
//! let tokens = lex(python_source, Mode::Module);
|
||||
//! let ast = parse_tokens(tokens.collect(), python_source, Mode::Module);
|
||||
//!
|
||||
//! assert!(ast.is_ok());
|
||||
//! ```
|
||||
@@ -138,14 +139,16 @@ pub mod typing;
|
||||
/// For example, parsing a simple function definition and a call to that function:
|
||||
///
|
||||
/// ```
|
||||
/// use ruff_python_parser as parser;
|
||||
/// use ruff_python_parser::parse_program;
|
||||
///
|
||||
/// let source = r#"
|
||||
/// def foo():
|
||||
/// return 42
|
||||
///
|
||||
/// print(foo())
|
||||
/// "#;
|
||||
/// let program = parser::parse_program(source);
|
||||
///
|
||||
/// let program = parse_program(source);
|
||||
/// assert!(program.is_ok());
|
||||
/// ```
|
||||
pub fn parse_program(source: &str) -> Result<ModModule, ParseError> {
|
||||
@@ -156,6 +159,28 @@ pub fn parse_program(source: &str) -> Result<ModModule, ParseError> {
|
||||
}
|
||||
}
|
||||
|
||||
/// Parse a full Python program into a [`Suite`].
|
||||
///
|
||||
/// This function is similar to [`parse_program`] except that it returns the module body
|
||||
/// instead of the module itself.
|
||||
///
|
||||
/// # Example
|
||||
///
|
||||
/// For example, parsing a simple function definition and a call to that function:
|
||||
///
|
||||
/// ```
|
||||
/// use ruff_python_parser::parse_suite;
|
||||
///
|
||||
/// let source = r#"
|
||||
/// def foo():
|
||||
/// return 42
|
||||
///
|
||||
/// print(foo())
|
||||
/// "#;
|
||||
///
|
||||
/// let body = parse_suite(source);
|
||||
/// assert!(body.is_ok());
|
||||
/// ```
|
||||
pub fn parse_suite(source: &str) -> Result<Suite, ParseError> {
|
||||
parse_program(source).map(|m| m.body)
|
||||
}
|
||||
@@ -169,12 +194,11 @@ pub fn parse_suite(source: &str) -> Result<Suite, ParseError> {
|
||||
///
|
||||
/// For example, parsing a single expression denoting the addition of two numbers:
|
||||
///
|
||||
/// ```
|
||||
/// use ruff_python_parser as parser;
|
||||
/// let expr = parser::parse_expression("1 + 2");
|
||||
/// ```
|
||||
/// use ruff_python_parser::parse_expression;
|
||||
///
|
||||
/// let expr = parse_expression("1 + 2");
|
||||
/// assert!(expr.is_ok());
|
||||
///
|
||||
/// ```
|
||||
pub fn parse_expression(source: &str) -> Result<Expr, ParseError> {
|
||||
let lexer = lex(source, Mode::Expression).collect();
|
||||
@@ -195,7 +219,7 @@ pub fn parse_expression(source: &str) -> Result<Expr, ParseError> {
|
||||
/// somewhat silly, location:
|
||||
///
|
||||
/// ```
|
||||
/// use ruff_python_parser::{parse_expression_starts_at};
|
||||
/// use ruff_python_parser::parse_expression_starts_at;
|
||||
/// # use ruff_text_size::TextSize;
|
||||
///
|
||||
/// let expr = parse_expression_starts_at("1 + 2", TextSize::from(400));
|
||||
@@ -262,7 +286,7 @@ pub fn parse(source: &str, mode: Mode) -> Result<Mod, ParseError> {
|
||||
|
||||
/// Parse the given Python source code using the specified [`Mode`] and [`TextSize`].
|
||||
///
|
||||
/// This function allows to specify the location of the the source code, other than
|
||||
/// This function allows to specify the location of the source code, other than
|
||||
/// that, it behaves exactly like [`parse`].
|
||||
///
|
||||
/// # Example
|
||||
@@ -298,10 +322,12 @@ pub fn parse_starts_at(source: &str, mode: Mode, offset: TextSize) -> Result<Mod
|
||||
/// them using the [`lexer::lex`] function:
|
||||
///
|
||||
/// ```
|
||||
/// use ruff_python_parser::{lexer::lex, Mode, parse_tokens};
|
||||
/// use ruff_python_parser::lexer::lex;
|
||||
/// use ruff_python_parser::{Mode, parse_tokens};
|
||||
///
|
||||
/// let source = "1 + 2";
|
||||
/// let expr = parse_tokens(lex(source, Mode::Expression).collect(), source, Mode::Expression);
|
||||
/// let tokens = lex(source, Mode::Expression);
|
||||
/// let expr = parse_tokens(tokens.collect(), source, Mode::Expression);
|
||||
/// assert!(expr.is_ok());
|
||||
/// ```
|
||||
pub fn parse_tokens(tokens: Vec<LexResult>, source: &str, mode: Mode) -> Result<Mod, ParseError> {
|
||||
@@ -370,13 +396,16 @@ pub fn parse_program_tokens(
|
||||
}
|
||||
|
||||
/// Control in the different modes by which a source file can be parsed.
|
||||
///
|
||||
/// The mode argument specifies in what way code must be parsed.
|
||||
#[derive(Clone, Copy, Debug, Hash, PartialEq, Eq)]
|
||||
pub enum Mode {
|
||||
/// The code consists of a sequence of statements.
|
||||
Module,
|
||||
|
||||
/// The code consists of a single expression.
|
||||
Expression,
|
||||
|
||||
/// The code consists of a sequence of statements which can include the
|
||||
/// escape commands that are part of IPython syntax.
|
||||
///
|
||||
@@ -408,6 +437,7 @@ impl std::str::FromStr for Mode {
|
||||
}
|
||||
}
|
||||
|
||||
/// A type that can be represented as [Mode].
|
||||
pub trait AsMode {
|
||||
fn as_mode(&self) -> Mode;
|
||||
}
|
||||
|
||||
@@ -26,6 +26,9 @@ mod statement;
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
|
||||
/// Represents the parsed source code.
|
||||
///
|
||||
/// This includes the AST and all of the errors encountered during parsing.
|
||||
#[derive(Debug)]
|
||||
pub struct Program {
|
||||
ast: ast::Mod,
|
||||
@@ -43,12 +46,12 @@ impl Program {
|
||||
&self.parse_errors
|
||||
}
|
||||
|
||||
/// Consumes the `Program` and returns the parsed AST.
|
||||
/// Consumes the [`Program`] and returns the parsed AST.
|
||||
pub fn into_ast(self) -> ast::Mod {
|
||||
self.ast
|
||||
}
|
||||
|
||||
/// Consumes the `Program` and returns a list of syntax errors found during parsing.
|
||||
/// Consumes the [`Program`] and returns a list of syntax errors found during parsing.
|
||||
pub fn into_errors(self) -> Vec<ParseError> {
|
||||
self.parse_errors
|
||||
}
|
||||
@@ -58,11 +61,13 @@ impl Program {
|
||||
self.parse_errors.is_empty()
|
||||
}
|
||||
|
||||
/// Parse the given Python source code using the specified [`Mode`].
|
||||
pub fn parse_str(source: &str, mode: Mode) -> Program {
|
||||
let tokens = lex(source, mode);
|
||||
Self::parse_tokens(source, tokens.collect(), mode)
|
||||
}
|
||||
|
||||
/// Parse a vector of [`LexResult`]s using the specified [`Mode`].
|
||||
pub fn parse_tokens(source: &str, tokens: Vec<LexResult>, mode: Mode) -> Program {
|
||||
Parser::new(source, mode, TokenSource::new(tokens)).parse_program()
|
||||
}
|
||||
@@ -124,49 +129,11 @@ impl<'src> Parser<'src> {
|
||||
}
|
||||
}
|
||||
|
||||
/// Consumes the [`Parser`] and returns the parsed [`Program`].
|
||||
pub(crate) fn parse_program(mut self) -> Program {
|
||||
let ast = if self.mode == Mode::Expression {
|
||||
let start = self.node_start();
|
||||
let parsed_expr = self.parse_expression_list(ExpressionContext::default());
|
||||
|
||||
// All of the remaining newlines are actually going to be non-logical newlines.
|
||||
self.eat(TokenKind::Newline);
|
||||
|
||||
if !self.at(TokenKind::EndOfFile) {
|
||||
self.add_error(
|
||||
ParseErrorType::UnexpectedExpressionToken,
|
||||
self.current_token_range(),
|
||||
);
|
||||
|
||||
// TODO(dhruvmanila): How should error recovery work here? Just truncate after the expression?
|
||||
let mut progress = ParserProgress::default();
|
||||
loop {
|
||||
progress.assert_progressing(&self);
|
||||
if self.at(TokenKind::EndOfFile) {
|
||||
break;
|
||||
}
|
||||
self.next_token();
|
||||
}
|
||||
}
|
||||
|
||||
self.bump(TokenKind::EndOfFile);
|
||||
|
||||
Mod::Expression(ast::ModExpression {
|
||||
body: Box::new(parsed_expr.expr),
|
||||
range: self.node_range(start),
|
||||
})
|
||||
} else {
|
||||
let body = self.parse_list_into_vec(
|
||||
RecoveryContextKind::ModuleStatements,
|
||||
Parser::parse_statement,
|
||||
);
|
||||
|
||||
self.bump(TokenKind::EndOfFile);
|
||||
|
||||
Mod::Module(ast::ModModule {
|
||||
body,
|
||||
range: self.tokens_range,
|
||||
})
|
||||
let ast = match self.mode {
|
||||
Mode::Expression => Mod::Expression(self.parse_single_expression()),
|
||||
Mode::Module | Mode::Ipython => Mod::Module(self.parse_module()),
|
||||
};
|
||||
|
||||
Program {
|
||||
@@ -175,6 +142,63 @@ impl<'src> Parser<'src> {
|
||||
}
|
||||
}
|
||||
|
||||
/// Parses a single expression.
|
||||
///
|
||||
/// This is to be used for [`Mode::Expression`].
|
||||
///
|
||||
/// ## Recovery
|
||||
///
|
||||
/// After parsing a single expression, an error is reported and all remaining tokens are
|
||||
/// dropped by the parser.
|
||||
fn parse_single_expression(&mut self) -> ast::ModExpression {
|
||||
let start = self.node_start();
|
||||
let parsed_expr = self.parse_expression_list(ExpressionContext::default());
|
||||
|
||||
// All remaining newlines are actually going to be non-logical newlines.
|
||||
self.eat(TokenKind::Newline);
|
||||
|
||||
if !self.at(TokenKind::EndOfFile) {
|
||||
self.add_error(
|
||||
ParseErrorType::UnexpectedExpressionToken,
|
||||
self.current_token_range(),
|
||||
);
|
||||
|
||||
// TODO(dhruvmanila): How should error recovery work here? Just truncate after the expression?
|
||||
let mut progress = ParserProgress::default();
|
||||
loop {
|
||||
progress.assert_progressing(self);
|
||||
if self.at(TokenKind::EndOfFile) {
|
||||
break;
|
||||
}
|
||||
self.next_token();
|
||||
}
|
||||
}
|
||||
|
||||
self.bump(TokenKind::EndOfFile);
|
||||
|
||||
ast::ModExpression {
|
||||
body: Box::new(parsed_expr.expr),
|
||||
range: self.node_range(start),
|
||||
}
|
||||
}
|
||||
|
||||
/// Parses a Python module.
|
||||
///
|
||||
/// This is to be used for [`Mode::Module`] and [`Mode::Ipython`].
|
||||
fn parse_module(&mut self) -> ast::ModModule {
|
||||
let body = self.parse_list_into_vec(
|
||||
RecoveryContextKind::ModuleStatements,
|
||||
Parser::parse_statement,
|
||||
);
|
||||
|
||||
self.bump(TokenKind::EndOfFile);
|
||||
|
||||
ast::ModModule {
|
||||
body,
|
||||
range: self.tokens_range,
|
||||
}
|
||||
}
|
||||
|
||||
fn finish(self) -> Vec<ParseError> {
|
||||
assert_eq!(
|
||||
self.current_token_kind(),
|
||||
|
||||
@@ -3371,34 +3371,15 @@ impl<'src> Parser<'src> {
|
||||
///
|
||||
/// Report errors for all the duplicate names found.
|
||||
fn validate_parameters(&mut self, parameters: &ast::Parameters) {
|
||||
let mut all_arg_names = FxHashSet::with_capacity_and_hasher(
|
||||
parameters.posonlyargs.len()
|
||||
+ parameters.args.len()
|
||||
+ usize::from(parameters.vararg.is_some())
|
||||
+ parameters.kwonlyargs.len()
|
||||
+ usize::from(parameters.kwarg.is_some()),
|
||||
BuildHasherDefault::default(),
|
||||
);
|
||||
let mut all_arg_names =
|
||||
FxHashSet::with_capacity_and_hasher(parameters.len(), BuildHasherDefault::default());
|
||||
|
||||
let posonlyargs = parameters.posonlyargs.iter();
|
||||
let args = parameters.args.iter();
|
||||
let kwonlyargs = parameters.kwonlyargs.iter();
|
||||
|
||||
let vararg = parameters.vararg.as_deref();
|
||||
let kwarg = parameters.kwarg.as_deref();
|
||||
|
||||
for arg in posonlyargs
|
||||
.chain(args)
|
||||
.chain(kwonlyargs)
|
||||
.map(|arg| &arg.parameter)
|
||||
.chain(vararg)
|
||||
.chain(kwarg)
|
||||
{
|
||||
let range = arg.name.range;
|
||||
let arg_name = arg.name.as_str();
|
||||
if !all_arg_names.insert(arg_name) {
|
||||
for parameter in parameters {
|
||||
let range = parameter.name().range();
|
||||
let param_name = parameter.name().as_str();
|
||||
if !all_arg_names.insert(param_name) {
|
||||
self.add_error(
|
||||
ParseErrorType::DuplicateParameter(arg_name.to_string()),
|
||||
ParseErrorType::DuplicateParameter(param_name.to_string()),
|
||||
range,
|
||||
);
|
||||
}
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
//! This module defines the tokens that the lexer recognizes. The tokens are
|
||||
//! loosely based on the token definitions found in the [CPython source].
|
||||
//!
|
||||
//! [CPython source]: https://github.com/python/cpython/blob/dfc2e065a2e71011017077e549cd2f9bf4944c54/Include/internal/pycore_token.h;
|
||||
//! [CPython source]: https://github.com/python/cpython/blob/dfc2e065a2e71011017077e549cd2f9bf4944c54/Grammar/Tokens
|
||||
|
||||
use ruff_python_ast::{AnyStringKind, BoolOp, Int, IpyEscapeKind, Operator, UnaryOp};
|
||||
use std::fmt;
|
||||
@@ -352,6 +352,10 @@ impl fmt::Display for Tok {
|
||||
}
|
||||
}
|
||||
|
||||
/// A kind of token.
|
||||
///
|
||||
/// This is a lightweight representation of [`Tok`] which doesn't contain any information
|
||||
/// about the token itself.
|
||||
#[derive(Copy, Clone, PartialEq, Eq, Hash, Debug)]
|
||||
pub enum TokenKind {
|
||||
/// Token value for a name, commonly known as an identifier.
|
||||
|
||||
@@ -1,10 +1,13 @@
|
||||
use crate::{parse_expression, parse_expression_starts_at};
|
||||
//! This module takes care of parsing a type annotation.
|
||||
|
||||
use anyhow::Result;
|
||||
|
||||
use ruff_python_ast::relocate::relocate_expr;
|
||||
use ruff_python_ast::str;
|
||||
use ruff_python_ast::Expr;
|
||||
use ruff_python_ast::{str, Expr};
|
||||
use ruff_text_size::{TextLen, TextRange};
|
||||
|
||||
use crate::{parse_expression, parse_expression_starts_at};
|
||||
|
||||
#[derive(is_macro::Is, Copy, Clone, Debug)]
|
||||
pub enum AnnotationKind {
|
||||
/// The annotation is defined as part a simple string literal,
|
||||
|
||||
@@ -139,6 +139,12 @@ Module(
|
||||
|
|
||||
|
||||
|
||||
|
|
||||
1 | def foo(a, a=10, *a, a, a: str, **a): ...
|
||||
| ^ Syntax Error: Duplicate parameter "a"
|
||||
|
|
||||
|
||||
|
||||
|
|
||||
1 | def foo(a, a=10, *a, a, a: str, **a): ...
|
||||
| ^ Syntax Error: Duplicate parameter "a"
|
||||
@@ -151,12 +157,6 @@ Module(
|
||||
|
|
||||
|
||||
|
||||
|
|
||||
1 | def foo(a, a=10, *a, a, a: str, **a): ...
|
||||
| ^ Syntax Error: Duplicate parameter "a"
|
||||
|
|
||||
|
||||
|
||||
|
|
||||
1 | def foo(a, a=10, *a, a, a: str, **a): ...
|
||||
| ^ Syntax Error: Duplicate parameter "a"
|
||||
|
||||
@@ -736,10 +736,7 @@ fn find_parameter<'a>(
|
||||
binding: &Binding,
|
||||
) -> Option<&'a ParameterWithDefault> {
|
||||
parameters
|
||||
.args
|
||||
.iter()
|
||||
.chain(parameters.posonlyargs.iter())
|
||||
.chain(parameters.kwonlyargs.iter())
|
||||
.iter_non_variadic_params()
|
||||
.find(|arg| arg.parameter.name.range() == binding.range())
|
||||
}
|
||||
|
||||
|
||||
@@ -26,7 +26,7 @@ ruff_text_size = { path = "../ruff_text_size" }
|
||||
ruff_workspace = { path = "../ruff_workspace" }
|
||||
|
||||
anyhow = { workspace = true }
|
||||
crossbeam-channel = { workspace = true }
|
||||
crossbeam = { workspace = true }
|
||||
jod-thread = { workspace = true }
|
||||
libc = { workspace = true }
|
||||
lsp-server = { workspace = true }
|
||||
|
||||
61
crates/ruff_server/docs/setup/HELIX.md
Normal file
61
crates/ruff_server/docs/setup/HELIX.md
Normal file
@@ -0,0 +1,61 @@
|
||||
## Helix Setup Guide for `ruff server`
|
||||
|
||||
First, open the language configuration file for Helix. On Linux and macOS, this will be at `~/.config/helix/languages.toml`,
|
||||
and on Windows this will be at `%AppData%\helix\languages.toml`.
|
||||
|
||||
Add the language server by adding:
|
||||
|
||||
```toml
|
||||
[language-server.ruff]
|
||||
command = "ruff"
|
||||
args = ["server", "--preview"]
|
||||
```
|
||||
|
||||
Then, you'll register the language server as the one to use with Python.
|
||||
If you don't already have a language server registered to use with Python, add this to `languages.toml`:
|
||||
|
||||
```toml
|
||||
[[language]]
|
||||
name = "python"
|
||||
language-servers = ["ruff"]
|
||||
```
|
||||
|
||||
Otherwise, if you already have `language-servers` defined, you can simply add `"ruff"` to the list. For example,
|
||||
if you already have `pylsp` as a language server, you can modify the language entry as follows:
|
||||
|
||||
```toml
|
||||
[[language]]
|
||||
name = "python"
|
||||
language-servers = ["ruff", "pylsp"]
|
||||
```
|
||||
|
||||
> \[!NOTE\]
|
||||
> Multiple language servers for a single language are only supported in Helix version [`23.10`](https://github.com/helix-editor/helix/blob/master/CHANGELOG.md#2310-2023-10-24) and later.
|
||||
|
||||
Once you've set up the server, you should see diagnostics in your Python files. Code actions and other LSP features should also be available.
|
||||
|
||||

|
||||
*This screenshot is using `select=["ALL]"` for demonstration purposes.*
|
||||
|
||||
If you want to, as an example, turn on auto-formatting, add `auto-format = true`:
|
||||
|
||||
```toml
|
||||
[[language]]
|
||||
name = "python"
|
||||
language-servers = ["ruff", "pylsp"]
|
||||
auto-format = true
|
||||
```
|
||||
|
||||
See the [Helix documentation](https://docs.helix-editor.com/languages.html) for more settings you can use here.
|
||||
|
||||
You can pass settings into `ruff server` using `[language-server.ruff.config.settings]`. For example:
|
||||
|
||||
```toml
|
||||
[language-server.ruff.config.settings]
|
||||
line-length = 80
|
||||
[language-server.ruff.config.settings.lint]
|
||||
select = ["E4", "E7"]
|
||||
preview = false
|
||||
[language-server.ruff.config.settings.format]
|
||||
preview = true
|
||||
```
|
||||
BIN
crates/ruff_server/docs/setup/assets/SuccessfulHelixSetup.png
Normal file
BIN
crates/ruff_server/docs/setup/assets/SuccessfulHelixSetup.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 195 KiB |
@@ -6,7 +6,7 @@ use serde_json::Value;
|
||||
|
||||
use super::schedule::Task;
|
||||
|
||||
pub(crate) type ClientSender = crossbeam_channel::Sender<lsp_server::Message>;
|
||||
pub(crate) type ClientSender = crossbeam::channel::Sender<lsp_server::Message>;
|
||||
|
||||
type ResponseBuilder<'s> = Box<dyn FnOnce(lsp_server::Response) -> Task<'s>>;
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
use std::num::NonZeroUsize;
|
||||
|
||||
use crossbeam_channel::Sender;
|
||||
use crossbeam::channel::Sender;
|
||||
|
||||
use crate::session::Session;
|
||||
|
||||
|
||||
@@ -21,7 +21,7 @@ use std::{
|
||||
},
|
||||
};
|
||||
|
||||
use crossbeam_channel::{Receiver, Sender};
|
||||
use crossbeam::channel::{Receiver, Sender};
|
||||
|
||||
use super::{Builder, JoinHandle, ThreadPriority};
|
||||
|
||||
@@ -52,7 +52,7 @@ impl Pool {
|
||||
let threads = usize::from(threads);
|
||||
|
||||
// Channel buffer capacity is between 2 and 4, depending on the pool size.
|
||||
let (job_sender, job_receiver) = crossbeam_channel::bounded(std::cmp::min(threads * 2, 4));
|
||||
let (job_sender, job_receiver) = crossbeam::channel::bounded(std::cmp::min(threads * 2, 4));
|
||||
let extant_tasks = Arc::new(AtomicUsize::new(0));
|
||||
|
||||
let mut handles = Vec::with_capacity(threads);
|
||||
|
||||
224
playground/api/package-lock.json
generated
224
playground/api/package-lock.json
generated
@@ -16,22 +16,25 @@
|
||||
"@cloudflare/workers-types": "^4.20230801.0",
|
||||
"miniflare": "^3.20230801.1",
|
||||
"typescript": "^5.1.6",
|
||||
"wrangler": "3.51.2"
|
||||
"wrangler": "3.52.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@cloudflare/kv-asset-handler": {
|
||||
"version": "0.3.1",
|
||||
"resolved": "https://registry.npmjs.org/@cloudflare/kv-asset-handler/-/kv-asset-handler-0.3.1.tgz",
|
||||
"integrity": "sha512-lKN2XCfKCmpKb86a1tl4GIwsJYDy9TGuwjhDELLmpKygQhw8X2xR4dusgpC5Tg7q1pB96Eb0rBo81kxSILQMwA==",
|
||||
"version": "0.3.2",
|
||||
"resolved": "https://registry.npmjs.org/@cloudflare/kv-asset-handler/-/kv-asset-handler-0.3.2.tgz",
|
||||
"integrity": "sha512-EeEjMobfuJrwoctj7FA1y1KEbM0+Q1xSjobIEyie9k4haVEBB7vkDvsasw1pM3rO39mL2akxIAzLMUAtrMHZhA==",
|
||||
"dev": true,
|
||||
"dependencies": {
|
||||
"mime": "^3.0.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=16.13"
|
||||
}
|
||||
},
|
||||
"node_modules/@cloudflare/workerd-darwin-64": {
|
||||
"version": "1.20240405.0",
|
||||
"resolved": "https://registry.npmjs.org/@cloudflare/workerd-darwin-64/-/workerd-darwin-64-1.20240405.0.tgz",
|
||||
"integrity": "sha512-ut8kwpHmlz9dNSjoov6v1b6jS50J46Mj9QcMA0t1Hne36InaQk/qqPSd12fN5p2GesZ9OOBJvBdDsTblVdyJ1w==",
|
||||
"version": "1.20240419.0",
|
||||
"resolved": "https://registry.npmjs.org/@cloudflare/workerd-darwin-64/-/workerd-darwin-64-1.20240419.0.tgz",
|
||||
"integrity": "sha512-PGVe9sYWULHfvGhN0IZh8MsskNG/ufnBSqPbgFCxJHCTrVXLPuC35EoVaforyqjKRwj3U35XMyGo9KHcGnTeHQ==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
@@ -45,9 +48,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@cloudflare/workerd-darwin-arm64": {
|
||||
"version": "1.20240405.0",
|
||||
"resolved": "https://registry.npmjs.org/@cloudflare/workerd-darwin-arm64/-/workerd-darwin-arm64-1.20240405.0.tgz",
|
||||
"integrity": "sha512-x3A3Ym+J2DH1uYnw0aedeKOTnUebEo312+Aladv7bFri97pjRJcqVbYhMtOHLkHjwYn7bpKSY2eL5iM+0XT29A==",
|
||||
"version": "1.20240419.0",
|
||||
"resolved": "https://registry.npmjs.org/@cloudflare/workerd-darwin-arm64/-/workerd-darwin-arm64-1.20240419.0.tgz",
|
||||
"integrity": "sha512-z4etQSPiD5Gcjs962LiC7ZdmXnN6SGof5KrYoFiSI9X9kUvpuGH/lnjVVPd+NnVNeDU2kzmcAIgyZjkjTaqVXQ==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
@@ -61,9 +64,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@cloudflare/workerd-linux-64": {
|
||||
"version": "1.20240405.0",
|
||||
"resolved": "https://registry.npmjs.org/@cloudflare/workerd-linux-64/-/workerd-linux-64-1.20240405.0.tgz",
|
||||
"integrity": "sha512-3tYpfjtxEQ0R30Pna7OF3Bz0CTx30hc0QNtH61KnkvXtaeYMkWutSKQKXIuVlPa/7v1MHp+8ViBXMflmS7HquA==",
|
||||
"version": "1.20240419.0",
|
||||
"resolved": "https://registry.npmjs.org/@cloudflare/workerd-linux-64/-/workerd-linux-64-1.20240419.0.tgz",
|
||||
"integrity": "sha512-lBwhg0j3sYTFMsEb4bOClbVje8nqrYOu0H3feQlX+Eks94JIhWPkf8ywK4at/BUc1comPMhCgzDHwc2OMPUGgg==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
@@ -77,9 +80,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@cloudflare/workerd-linux-arm64": {
|
||||
"version": "1.20240405.0",
|
||||
"resolved": "https://registry.npmjs.org/@cloudflare/workerd-linux-arm64/-/workerd-linux-arm64-1.20240405.0.tgz",
|
||||
"integrity": "sha512-NpKZlvmdgcX/m4tP5zM91AfJpZrue2/GRA+Sl3szxAivu2uE5jDVf5SS9dzqzCVfPrdhylqH7yeL4U/cafFNOg==",
|
||||
"version": "1.20240419.0",
|
||||
"resolved": "https://registry.npmjs.org/@cloudflare/workerd-linux-arm64/-/workerd-linux-arm64-1.20240419.0.tgz",
|
||||
"integrity": "sha512-ZMY6wwWkxL+WPq8ydOp/irSYjAnMhBz1OC1+4z+OANtDs2beaZODmq7LEB3hb5WUAaTPY7DIjZh3DfDfty0nYg==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
@@ -93,9 +96,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@cloudflare/workerd-windows-64": {
|
||||
"version": "1.20240405.0",
|
||||
"resolved": "https://registry.npmjs.org/@cloudflare/workerd-windows-64/-/workerd-windows-64-1.20240405.0.tgz",
|
||||
"integrity": "sha512-REBeJMxvUCjwuEVzSSIBtzAyM69QjToab8qBst0S9vdih+9DObym4dw8CevdBQhDbFrHiyL9E6pAZpLPNHVgCw==",
|
||||
"version": "1.20240419.0",
|
||||
"resolved": "https://registry.npmjs.org/@cloudflare/workerd-windows-64/-/workerd-windows-64-1.20240419.0.tgz",
|
||||
"integrity": "sha512-YJjgaJN2yGTkV7Cr4K3i8N4dUwVQTclT3Pr3NpRZCcLjTszwlE53++XXDnHMKGXBbSguIizaVbmcU2EtmIXyeQ==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
@@ -109,9 +112,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@cloudflare/workers-types": {
|
||||
"version": "4.20240419.0",
|
||||
"resolved": "https://registry.npmjs.org/@cloudflare/workers-types/-/workers-types-4.20240419.0.tgz",
|
||||
"integrity": "sha512-UM16sr4HEe0mDj6C5OFcodzdj/CnEp0bfncAq3g7OpDsoZ1sBrfsMrb7Yc4f8J81EemvmQZyE6sSanpURtVkcQ==",
|
||||
"version": "4.20240423.0",
|
||||
"resolved": "https://registry.npmjs.org/@cloudflare/workers-types/-/workers-types-4.20240423.0.tgz",
|
||||
"integrity": "sha512-ssuccb3j+URp6mP2p0PcQE9vmS3YeKBQnALHF9P3yQfUAFozuhTsDTbqmL+zPrJvUcG7SL2xVQkNDF9QJeKDZw==",
|
||||
"dev": true
|
||||
},
|
||||
"node_modules/@cspotcode/source-map-support": {
|
||||
@@ -592,12 +595,6 @@
|
||||
"@types/node": "*"
|
||||
}
|
||||
},
|
||||
"node_modules/@types/json-schema": {
|
||||
"version": "7.0.15",
|
||||
"resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.15.tgz",
|
||||
"integrity": "sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==",
|
||||
"dev": true
|
||||
},
|
||||
"node_modules/@types/node": {
|
||||
"version": "20.4.8",
|
||||
"resolved": "https://registry.npmjs.org/@types/node/-/node-20.4.8.tgz",
|
||||
@@ -646,12 +643,6 @@
|
||||
"printable-characters": "^1.0.42"
|
||||
}
|
||||
},
|
||||
"node_modules/balanced-match": {
|
||||
"version": "1.0.2",
|
||||
"resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz",
|
||||
"integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==",
|
||||
"dev": true
|
||||
},
|
||||
"node_modules/binary-extensions": {
|
||||
"version": "2.2.0",
|
||||
"resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.2.0.tgz",
|
||||
@@ -667,15 +658,6 @@
|
||||
"integrity": "sha512-F1+K8EbfOZE49dtoPtmxUQrpXaBIl3ICvasLh+nJta0xkz+9kF/7uet9fLnwKqhDrmj6g+6K3Tw9yQPUg2ka5g==",
|
||||
"dev": true
|
||||
},
|
||||
"node_modules/brace-expansion": {
|
||||
"version": "2.0.1",
|
||||
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz",
|
||||
"integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==",
|
||||
"dev": true,
|
||||
"dependencies": {
|
||||
"balanced-match": "^1.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/braces": {
|
||||
"version": "3.0.2",
|
||||
"resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz",
|
||||
@@ -733,15 +715,6 @@
|
||||
"fsevents": "~2.3.2"
|
||||
}
|
||||
},
|
||||
"node_modules/commander": {
|
||||
"version": "12.0.0",
|
||||
"resolved": "https://registry.npmjs.org/commander/-/commander-12.0.0.tgz",
|
||||
"integrity": "sha512-MwVNWlYjDTtOjX5PiD7o5pK0UrFU/OYgcJfjjK4RaHZETNtjJqrZa9Y9ds88+A+f+d5lv+561eZ+yCKoS3gbAA==",
|
||||
"dev": true,
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
}
|
||||
},
|
||||
"node_modules/cookie": {
|
||||
"version": "0.5.0",
|
||||
"resolved": "https://registry.npmjs.org/cookie/-/cookie-0.5.0.tgz",
|
||||
@@ -888,12 +861,6 @@
|
||||
"node": ">=8"
|
||||
}
|
||||
},
|
||||
"node_modules/fs.realpath": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz",
|
||||
"integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==",
|
||||
"dev": true
|
||||
},
|
||||
"node_modules/fsevents": {
|
||||
"version": "2.3.2",
|
||||
"resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz",
|
||||
@@ -938,25 +905,6 @@
|
||||
"url": "https://github.com/sponsors/sindresorhus"
|
||||
}
|
||||
},
|
||||
"node_modules/glob": {
|
||||
"version": "8.1.0",
|
||||
"resolved": "https://registry.npmjs.org/glob/-/glob-8.1.0.tgz",
|
||||
"integrity": "sha512-r8hpEjiQEYlF2QU0df3dS+nxxSIreXQS1qRhMJM0Q5NDdR386C7jb7Hwwod8Fgiuex+k0GFjgft18yvxm5XoCQ==",
|
||||
"dev": true,
|
||||
"dependencies": {
|
||||
"fs.realpath": "^1.0.0",
|
||||
"inflight": "^1.0.4",
|
||||
"inherits": "2",
|
||||
"minimatch": "^5.0.1",
|
||||
"once": "^1.3.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=12"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/sponsors/isaacs"
|
||||
}
|
||||
},
|
||||
"node_modules/glob-parent": {
|
||||
"version": "5.1.2",
|
||||
"resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz",
|
||||
@@ -995,22 +943,6 @@
|
||||
"node": ">=12.20.0"
|
||||
}
|
||||
},
|
||||
"node_modules/inflight": {
|
||||
"version": "1.0.6",
|
||||
"resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz",
|
||||
"integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==",
|
||||
"dev": true,
|
||||
"dependencies": {
|
||||
"once": "^1.3.0",
|
||||
"wrappy": "1"
|
||||
}
|
||||
},
|
||||
"node_modules/inherits": {
|
||||
"version": "2.0.4",
|
||||
"resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz",
|
||||
"integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==",
|
||||
"dev": true
|
||||
},
|
||||
"node_modules/is-binary-path": {
|
||||
"version": "2.1.0",
|
||||
"resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-2.1.0.tgz",
|
||||
@@ -1081,18 +1013,6 @@
|
||||
"resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz",
|
||||
"integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw=="
|
||||
},
|
||||
"node_modules/json5": {
|
||||
"version": "2.2.3",
|
||||
"resolved": "https://registry.npmjs.org/json5/-/json5-2.2.3.tgz",
|
||||
"integrity": "sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==",
|
||||
"dev": true,
|
||||
"bin": {
|
||||
"json5": "lib/cli.js"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=6"
|
||||
}
|
||||
},
|
||||
"node_modules/kleur": {
|
||||
"version": "4.1.5",
|
||||
"resolved": "https://registry.npmjs.org/kleur/-/kleur-4.1.5.tgz",
|
||||
@@ -1150,9 +1070,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/miniflare": {
|
||||
"version": "3.20240405.2",
|
||||
"resolved": "https://registry.npmjs.org/miniflare/-/miniflare-3.20240405.2.tgz",
|
||||
"integrity": "sha512-n/V5m9GVMN37U5gWdrNXKx2d1icLXtcIKcxWtLslH4RTaebZJdSRmp12UHyuQsKlaSpTkNqyzLVtCEgt2bhRSA==",
|
||||
"version": "3.20240419.0",
|
||||
"resolved": "https://registry.npmjs.org/miniflare/-/miniflare-3.20240419.0.tgz",
|
||||
"integrity": "sha512-fIev1PP4H+fQp5FtvzHqRY2v5s+jxh/a0xAhvM5fBNXvxWX7Zod1OatXfXwYbse3hqO3KeVMhb0osVtrW0NwJg==",
|
||||
"dev": true,
|
||||
"dependencies": {
|
||||
"@cspotcode/source-map-support": "0.8.1",
|
||||
@@ -1163,7 +1083,7 @@
|
||||
"glob-to-regexp": "^0.4.1",
|
||||
"stoppable": "^1.1.0",
|
||||
"undici": "^5.28.2",
|
||||
"workerd": "1.20240405.0",
|
||||
"workerd": "1.20240419.0",
|
||||
"ws": "^8.11.0",
|
||||
"youch": "^3.2.2",
|
||||
"zod": "^3.20.6"
|
||||
@@ -1175,18 +1095,6 @@
|
||||
"node": ">=16.13"
|
||||
}
|
||||
},
|
||||
"node_modules/minimatch": {
|
||||
"version": "5.1.6",
|
||||
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.1.6.tgz",
|
||||
"integrity": "sha512-lKwV/1brpG6mBUFHtb7NUmtABCb2WZZmm2wNiOA5hAb8VdCS4B3dtMWyvcoViccwAW/COERjXLt0zP1zXUN26g==",
|
||||
"dev": true,
|
||||
"dependencies": {
|
||||
"brace-expansion": "^2.0.1"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=10"
|
||||
}
|
||||
},
|
||||
"node_modules/ms": {
|
||||
"version": "2.1.2",
|
||||
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
|
||||
@@ -1274,15 +1182,6 @@
|
||||
"validate-npm-package-name": "^4.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/once": {
|
||||
"version": "1.4.0",
|
||||
"resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz",
|
||||
"integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==",
|
||||
"dev": true,
|
||||
"dependencies": {
|
||||
"wrappy": "1"
|
||||
}
|
||||
},
|
||||
"node_modules/onetime": {
|
||||
"version": "6.0.0",
|
||||
"resolved": "https://registry.npmjs.org/onetime/-/onetime-6.0.0.tgz",
|
||||
@@ -1407,15 +1306,6 @@
|
||||
"estree-walker": "^0.6.1"
|
||||
}
|
||||
},
|
||||
"node_modules/safe-stable-stringify": {
|
||||
"version": "2.4.3",
|
||||
"resolved": "https://registry.npmjs.org/safe-stable-stringify/-/safe-stable-stringify-2.4.3.tgz",
|
||||
"integrity": "sha512-e2bDA2WJT0wxseVd4lsDP4+3ONX6HpMXQa1ZhFQ7SU+GjvORCmShbCMltrtIDfkYhVHrOcPtj+KhmDBdPdZD1g==",
|
||||
"dev": true,
|
||||
"engines": {
|
||||
"node": ">=10"
|
||||
}
|
||||
},
|
||||
"node_modules/selfsigned": {
|
||||
"version": "2.1.1",
|
||||
"resolved": "https://registry.npmjs.org/selfsigned/-/selfsigned-2.1.1.tgz",
|
||||
@@ -1537,27 +1427,6 @@
|
||||
"node": ">=8.0"
|
||||
}
|
||||
},
|
||||
"node_modules/ts-json-schema-generator": {
|
||||
"version": "1.5.1",
|
||||
"resolved": "https://registry.npmjs.org/ts-json-schema-generator/-/ts-json-schema-generator-1.5.1.tgz",
|
||||
"integrity": "sha512-apX5qG2+NA66j7b4AJm8q/DpdTeOsjfh7A3LpKsUiil0FepkNwtN28zYgjrsiiya2/OPhsr/PSjX5FUYg79rCg==",
|
||||
"dev": true,
|
||||
"dependencies": {
|
||||
"@types/json-schema": "^7.0.15",
|
||||
"commander": "^12.0.0",
|
||||
"glob": "^8.0.3",
|
||||
"json5": "^2.2.3",
|
||||
"normalize-path": "^3.0.0",
|
||||
"safe-stable-stringify": "^2.4.3",
|
||||
"typescript": "~5.4.2"
|
||||
},
|
||||
"bin": {
|
||||
"ts-json-schema-generator": "bin/ts-json-schema-generator"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=10.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/tslib": {
|
||||
"version": "2.6.1",
|
||||
"resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.1.tgz",
|
||||
@@ -1627,9 +1496,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/workerd": {
|
||||
"version": "1.20240405.0",
|
||||
"resolved": "https://registry.npmjs.org/workerd/-/workerd-1.20240405.0.tgz",
|
||||
"integrity": "sha512-AWrOSBh4Ll7sBWHuh0aywm8hDkKqsZmcwnDB0PVGszWZM5mndNBI5iJ/8haXVpdoyqkJQEVdhET9JDi4yU8tRg==",
|
||||
"version": "1.20240419.0",
|
||||
"resolved": "https://registry.npmjs.org/workerd/-/workerd-1.20240419.0.tgz",
|
||||
"integrity": "sha512-9yV98KpkQgG+bdEsKEW8i1AYZgxns6NVSfdOVEB2Ue1pTMtIEYfUyqUE+O2amisRrfaC3Pw4EvjtTmVaoetfeg==",
|
||||
"dev": true,
|
||||
"hasInstallScript": true,
|
||||
"bin": {
|
||||
@@ -1639,33 +1508,32 @@
|
||||
"node": ">=16"
|
||||
},
|
||||
"optionalDependencies": {
|
||||
"@cloudflare/workerd-darwin-64": "1.20240405.0",
|
||||
"@cloudflare/workerd-darwin-arm64": "1.20240405.0",
|
||||
"@cloudflare/workerd-linux-64": "1.20240405.0",
|
||||
"@cloudflare/workerd-linux-arm64": "1.20240405.0",
|
||||
"@cloudflare/workerd-windows-64": "1.20240405.0"
|
||||
"@cloudflare/workerd-darwin-64": "1.20240419.0",
|
||||
"@cloudflare/workerd-darwin-arm64": "1.20240419.0",
|
||||
"@cloudflare/workerd-linux-64": "1.20240419.0",
|
||||
"@cloudflare/workerd-linux-arm64": "1.20240419.0",
|
||||
"@cloudflare/workerd-windows-64": "1.20240419.0"
|
||||
}
|
||||
},
|
||||
"node_modules/wrangler": {
|
||||
"version": "3.51.2",
|
||||
"resolved": "https://registry.npmjs.org/wrangler/-/wrangler-3.51.2.tgz",
|
||||
"integrity": "sha512-8TRUwzPHj6+uPDzY0hBJ9/YwniEF9pqMGe5qbqLP/XsHTCWxIFib5go374zyCkmuVh23AwV7NuTA6gUtSqZ8pQ==",
|
||||
"version": "3.52.0",
|
||||
"resolved": "https://registry.npmjs.org/wrangler/-/wrangler-3.52.0.tgz",
|
||||
"integrity": "sha512-HR06jTym+yr7+CI3Ggld3nfp1OM9vSC7h4B8vwWHwhi5K0sYg8p44rxV514Gmsv9dkFHegkRP70SM3sjuuxxpQ==",
|
||||
"dev": true,
|
||||
"dependencies": {
|
||||
"@cloudflare/kv-asset-handler": "0.3.1",
|
||||
"@cloudflare/kv-asset-handler": "0.3.2",
|
||||
"@esbuild-plugins/node-globals-polyfill": "^0.2.3",
|
||||
"@esbuild-plugins/node-modules-polyfill": "^0.2.2",
|
||||
"blake3-wasm": "^2.1.5",
|
||||
"chokidar": "^3.5.3",
|
||||
"esbuild": "0.17.19",
|
||||
"miniflare": "3.20240405.2",
|
||||
"miniflare": "3.20240419.0",
|
||||
"nanoid": "^3.3.3",
|
||||
"path-to-regexp": "^6.2.0",
|
||||
"resolve": "^1.22.8",
|
||||
"resolve.exports": "^2.0.2",
|
||||
"selfsigned": "^2.0.1",
|
||||
"source-map": "0.6.1",
|
||||
"ts-json-schema-generator": "^1.5.0",
|
||||
"xxhash-wasm": "^1.0.1"
|
||||
},
|
||||
"bin": {
|
||||
@@ -1679,7 +1547,7 @@
|
||||
"fsevents": "~2.3.2"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"@cloudflare/workers-types": "^4.20240405.0"
|
||||
"@cloudflare/workers-types": "^4.20240419.0"
|
||||
},
|
||||
"peerDependenciesMeta": {
|
||||
"@cloudflare/workers-types": {
|
||||
@@ -1687,12 +1555,6 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"node_modules/wrappy": {
|
||||
"version": "1.0.2",
|
||||
"resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",
|
||||
"integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==",
|
||||
"dev": true
|
||||
},
|
||||
"node_modules/ws": {
|
||||
"version": "8.13.0",
|
||||
"resolved": "https://registry.npmjs.org/ws/-/ws-8.13.0.tgz",
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
"@cloudflare/workers-types": "^4.20230801.0",
|
||||
"miniflare": "^3.20230801.1",
|
||||
"typescript": "^5.1.6",
|
||||
"wrangler": "3.51.2"
|
||||
"wrangler": "3.52.0"
|
||||
},
|
||||
"private": true,
|
||||
"scripts": {
|
||||
|
||||
110
playground/package-lock.json
generated
110
playground/package-lock.json
generated
@@ -11,7 +11,7 @@
|
||||
"@monaco-editor/react": "^4.4.6",
|
||||
"classnames": "^2.3.2",
|
||||
"lz-string": "^1.5.0",
|
||||
"monaco-editor": "^0.47.0",
|
||||
"monaco-editor": "^0.48.0",
|
||||
"react": "^18.2.0",
|
||||
"react-dom": "^18.2.0",
|
||||
"react-resizable-panels": "^2.0.0"
|
||||
@@ -1046,9 +1046,9 @@
|
||||
"dev": true
|
||||
},
|
||||
"node_modules/@types/react": {
|
||||
"version": "18.2.79",
|
||||
"resolved": "https://registry.npmjs.org/@types/react/-/react-18.2.79.tgz",
|
||||
"integrity": "sha512-RwGAGXPl9kSXwdNTafkOEuFrTBD5SA2B3iEB96xi8+xu5ddUa/cpvyVCSNn+asgLCTHkb5ZxN8gbuibYJi4s1w==",
|
||||
"version": "18.3.1",
|
||||
"resolved": "https://registry.npmjs.org/@types/react/-/react-18.3.1.tgz",
|
||||
"integrity": "sha512-V0kuGBX3+prX+DQ/7r2qsv1NsdfnCLnTgnRJ1pYnxykBhGMz+qj+box5lq7XsO5mtZsBqpjwwTu/7wszPfMBcw==",
|
||||
"dev": true,
|
||||
"dependencies": {
|
||||
"@types/prop-types": "*",
|
||||
@@ -1056,9 +1056,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@types/react-dom": {
|
||||
"version": "18.2.25",
|
||||
"resolved": "https://registry.npmjs.org/@types/react-dom/-/react-dom-18.2.25.tgz",
|
||||
"integrity": "sha512-o/V48vf4MQh7juIKZU2QGDfli6p1+OOi5oXx36Hffpc9adsHeXjVp8rHuPkjd8VT8sOJ2Zp05HR7CdpGTIUFUA==",
|
||||
"version": "18.3.0",
|
||||
"resolved": "https://registry.npmjs.org/@types/react-dom/-/react-dom-18.3.0.tgz",
|
||||
"integrity": "sha512-EhwApuTmMBmXuFOikhQLIBUn6uFg81SwLMOAUgodJF14SOBOCMdU04gDoYi0WOJJHD144TL32z4yDqCW3dnkQg==",
|
||||
"dev": true,
|
||||
"dependencies": {
|
||||
"@types/react": "*"
|
||||
@@ -1071,16 +1071,16 @@
|
||||
"dev": true
|
||||
},
|
||||
"node_modules/@typescript-eslint/eslint-plugin": {
|
||||
"version": "7.7.0",
|
||||
"resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-7.7.0.tgz",
|
||||
"integrity": "sha512-GJWR0YnfrKnsRoluVO3PRb9r5aMZriiMMM/RHj5nnTrBy1/wIgk76XCtCKcnXGjpZQJQRFtGV9/0JJ6n30uwpQ==",
|
||||
"version": "7.7.1",
|
||||
"resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-7.7.1.tgz",
|
||||
"integrity": "sha512-KwfdWXJBOviaBVhxO3p5TJiLpNuh2iyXyjmWN0f1nU87pwyvfS0EmjC6ukQVYVFJd/K1+0NWGPDXiyEyQorn0Q==",
|
||||
"dev": true,
|
||||
"dependencies": {
|
||||
"@eslint-community/regexpp": "^4.10.0",
|
||||
"@typescript-eslint/scope-manager": "7.7.0",
|
||||
"@typescript-eslint/type-utils": "7.7.0",
|
||||
"@typescript-eslint/utils": "7.7.0",
|
||||
"@typescript-eslint/visitor-keys": "7.7.0",
|
||||
"@typescript-eslint/scope-manager": "7.7.1",
|
||||
"@typescript-eslint/type-utils": "7.7.1",
|
||||
"@typescript-eslint/utils": "7.7.1",
|
||||
"@typescript-eslint/visitor-keys": "7.7.1",
|
||||
"debug": "^4.3.4",
|
||||
"graphemer": "^1.4.0",
|
||||
"ignore": "^5.3.1",
|
||||
@@ -1106,15 +1106,15 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@typescript-eslint/parser": {
|
||||
"version": "7.7.0",
|
||||
"resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-7.7.0.tgz",
|
||||
"integrity": "sha512-fNcDm3wSwVM8QYL4HKVBggdIPAy9Q41vcvC/GtDobw3c4ndVT3K6cqudUmjHPw8EAp4ufax0o58/xvWaP2FmTg==",
|
||||
"version": "7.7.1",
|
||||
"resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-7.7.1.tgz",
|
||||
"integrity": "sha512-vmPzBOOtz48F6JAGVS/kZYk4EkXao6iGrD838sp1w3NQQC0W8ry/q641KU4PrG7AKNAf56NOcR8GOpH8l9FPCw==",
|
||||
"dev": true,
|
||||
"dependencies": {
|
||||
"@typescript-eslint/scope-manager": "7.7.0",
|
||||
"@typescript-eslint/types": "7.7.0",
|
||||
"@typescript-eslint/typescript-estree": "7.7.0",
|
||||
"@typescript-eslint/visitor-keys": "7.7.0",
|
||||
"@typescript-eslint/scope-manager": "7.7.1",
|
||||
"@typescript-eslint/types": "7.7.1",
|
||||
"@typescript-eslint/typescript-estree": "7.7.1",
|
||||
"@typescript-eslint/visitor-keys": "7.7.1",
|
||||
"debug": "^4.3.4"
|
||||
},
|
||||
"engines": {
|
||||
@@ -1134,13 +1134,13 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@typescript-eslint/scope-manager": {
|
||||
"version": "7.7.0",
|
||||
"resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-7.7.0.tgz",
|
||||
"integrity": "sha512-/8INDn0YLInbe9Wt7dK4cXLDYp0fNHP5xKLHvZl3mOT5X17rK/YShXaiNmorl+/U4VKCVIjJnx4Ri5b0y+HClw==",
|
||||
"version": "7.7.1",
|
||||
"resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-7.7.1.tgz",
|
||||
"integrity": "sha512-PytBif2SF+9SpEUKynYn5g1RHFddJUcyynGpztX3l/ik7KmZEv19WCMhUBkHXPU9es/VWGD3/zg3wg90+Dh2rA==",
|
||||
"dev": true,
|
||||
"dependencies": {
|
||||
"@typescript-eslint/types": "7.7.0",
|
||||
"@typescript-eslint/visitor-keys": "7.7.0"
|
||||
"@typescript-eslint/types": "7.7.1",
|
||||
"@typescript-eslint/visitor-keys": "7.7.1"
|
||||
},
|
||||
"engines": {
|
||||
"node": "^18.18.0 || >=20.0.0"
|
||||
@@ -1151,13 +1151,13 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@typescript-eslint/type-utils": {
|
||||
"version": "7.7.0",
|
||||
"resolved": "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-7.7.0.tgz",
|
||||
"integrity": "sha512-bOp3ejoRYrhAlnT/bozNQi3nio9tIgv3U5C0mVDdZC7cpcQEDZXvq8inrHYghLVwuNABRqrMW5tzAv88Vy77Sg==",
|
||||
"version": "7.7.1",
|
||||
"resolved": "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-7.7.1.tgz",
|
||||
"integrity": "sha512-ZksJLW3WF7o75zaBPScdW1Gbkwhd/lyeXGf1kQCxJaOeITscoSl0MjynVvCzuV5boUz/3fOI06Lz8La55mu29Q==",
|
||||
"dev": true,
|
||||
"dependencies": {
|
||||
"@typescript-eslint/typescript-estree": "7.7.0",
|
||||
"@typescript-eslint/utils": "7.7.0",
|
||||
"@typescript-eslint/typescript-estree": "7.7.1",
|
||||
"@typescript-eslint/utils": "7.7.1",
|
||||
"debug": "^4.3.4",
|
||||
"ts-api-utils": "^1.3.0"
|
||||
},
|
||||
@@ -1178,9 +1178,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@typescript-eslint/types": {
|
||||
"version": "7.7.0",
|
||||
"resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-7.7.0.tgz",
|
||||
"integrity": "sha512-G01YPZ1Bd2hn+KPpIbrAhEWOn5lQBrjxkzHkWvP6NucMXFtfXoevK82hzQdpfuQYuhkvFDeQYbzXCjR1z9Z03w==",
|
||||
"version": "7.7.1",
|
||||
"resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-7.7.1.tgz",
|
||||
"integrity": "sha512-AmPmnGW1ZLTpWa+/2omPrPfR7BcbUU4oha5VIbSbS1a1Tv966bklvLNXxp3mrbc+P2j4MNOTfDffNsk4o0c6/w==",
|
||||
"dev": true,
|
||||
"engines": {
|
||||
"node": "^18.18.0 || >=20.0.0"
|
||||
@@ -1191,13 +1191,13 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@typescript-eslint/typescript-estree": {
|
||||
"version": "7.7.0",
|
||||
"resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-7.7.0.tgz",
|
||||
"integrity": "sha512-8p71HQPE6CbxIBy2kWHqM1KGrC07pk6RJn40n0DSc6bMOBBREZxSDJ+BmRzc8B5OdaMh1ty3mkuWRg4sCFiDQQ==",
|
||||
"version": "7.7.1",
|
||||
"resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-7.7.1.tgz",
|
||||
"integrity": "sha512-CXe0JHCXru8Fa36dteXqmH2YxngKJjkQLjxzoj6LYwzZ7qZvgsLSc+eqItCrqIop8Vl2UKoAi0StVWu97FQZIQ==",
|
||||
"dev": true,
|
||||
"dependencies": {
|
||||
"@typescript-eslint/types": "7.7.0",
|
||||
"@typescript-eslint/visitor-keys": "7.7.0",
|
||||
"@typescript-eslint/types": "7.7.1",
|
||||
"@typescript-eslint/visitor-keys": "7.7.1",
|
||||
"debug": "^4.3.4",
|
||||
"globby": "^11.1.0",
|
||||
"is-glob": "^4.0.3",
|
||||
@@ -1243,17 +1243,17 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@typescript-eslint/utils": {
|
||||
"version": "7.7.0",
|
||||
"resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-7.7.0.tgz",
|
||||
"integrity": "sha512-LKGAXMPQs8U/zMRFXDZOzmMKgFv3COlxUQ+2NMPhbqgVm6R1w+nU1i4836Pmxu9jZAuIeyySNrN/6Rc657ggig==",
|
||||
"version": "7.7.1",
|
||||
"resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-7.7.1.tgz",
|
||||
"integrity": "sha512-QUvBxPEaBXf41ZBbaidKICgVL8Hin0p6prQDu6bbetWo39BKbWJxRsErOzMNT1rXvTll+J7ChrbmMCXM9rsvOQ==",
|
||||
"dev": true,
|
||||
"dependencies": {
|
||||
"@eslint-community/eslint-utils": "^4.4.0",
|
||||
"@types/json-schema": "^7.0.15",
|
||||
"@types/semver": "^7.5.8",
|
||||
"@typescript-eslint/scope-manager": "7.7.0",
|
||||
"@typescript-eslint/types": "7.7.0",
|
||||
"@typescript-eslint/typescript-estree": "7.7.0",
|
||||
"@typescript-eslint/scope-manager": "7.7.1",
|
||||
"@typescript-eslint/types": "7.7.1",
|
||||
"@typescript-eslint/typescript-estree": "7.7.1",
|
||||
"semver": "^7.6.0"
|
||||
},
|
||||
"engines": {
|
||||
@@ -1268,12 +1268,12 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@typescript-eslint/visitor-keys": {
|
||||
"version": "7.7.0",
|
||||
"resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-7.7.0.tgz",
|
||||
"integrity": "sha512-h0WHOj8MhdhY8YWkzIF30R379y0NqyOHExI9N9KCzvmu05EgG4FumeYa3ccfKUSphyWkWQE1ybVrgz/Pbam6YA==",
|
||||
"version": "7.7.1",
|
||||
"resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-7.7.1.tgz",
|
||||
"integrity": "sha512-gBL3Eq25uADw1LQ9kVpf3hRM+DWzs0uZknHYK3hq4jcTPqVCClHGDnB6UUUV2SFeBeA4KWHWbbLqmbGcZ4FYbw==",
|
||||
"dev": true,
|
||||
"dependencies": {
|
||||
"@typescript-eslint/types": "7.7.0",
|
||||
"@typescript-eslint/types": "7.7.1",
|
||||
"eslint-visitor-keys": "^3.4.3"
|
||||
},
|
||||
"engines": {
|
||||
@@ -2486,9 +2486,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/eslint-plugin-react-hooks": {
|
||||
"version": "4.6.0",
|
||||
"resolved": "https://registry.npmjs.org/eslint-plugin-react-hooks/-/eslint-plugin-react-hooks-4.6.0.tgz",
|
||||
"integrity": "sha512-oFc7Itz9Qxh2x4gNHStv3BqJq54ExXmfC+a1NjAta66IAN87Wu0R/QArgIS9qKzX3dXKPI9H5crl9QchNMY9+g==",
|
||||
"version": "4.6.2",
|
||||
"resolved": "https://registry.npmjs.org/eslint-plugin-react-hooks/-/eslint-plugin-react-hooks-4.6.2.tgz",
|
||||
"integrity": "sha512-QzliNJq4GinDBcD8gPB5v0wh6g8q3SUi6EFF0x8N/BL9PoVs0atuGc47ozMRyOWAKdwaZ5OnbOEa3WR+dSGKuQ==",
|
||||
"dev": true,
|
||||
"engines": {
|
||||
"node": ">=10"
|
||||
@@ -3689,9 +3689,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/monaco-editor": {
|
||||
"version": "0.47.0",
|
||||
"resolved": "https://registry.npmjs.org/monaco-editor/-/monaco-editor-0.47.0.tgz",
|
||||
"integrity": "sha512-VabVvHvQ9QmMwXu4du008ZDuyLnHs9j7ThVFsiJoXSOQk18+LF89N4ADzPbFenm0W4V2bGHnFBztIRQTgBfxzw=="
|
||||
"version": "0.48.0",
|
||||
"resolved": "https://registry.npmjs.org/monaco-editor/-/monaco-editor-0.48.0.tgz",
|
||||
"integrity": "sha512-goSDElNqFfw7iDHMg8WDATkfcyeLTNpBHQpO8incK6p5qZt5G/1j41X0xdGzpIkGojGXM+QiRQyLjnfDVvrpwA=="
|
||||
},
|
||||
"node_modules/ms": {
|
||||
"version": "2.1.2",
|
||||
|
||||
@@ -18,7 +18,7 @@
|
||||
"@monaco-editor/react": "^4.4.6",
|
||||
"classnames": "^2.3.2",
|
||||
"lz-string": "^1.5.0",
|
||||
"monaco-editor": "^0.47.0",
|
||||
"monaco-editor": "^0.48.0",
|
||||
"react": "^18.2.0",
|
||||
"react-dom": "^18.2.0",
|
||||
"react-resizable-panels": "^2.0.0"
|
||||
|
||||
@@ -54,7 +54,7 @@ mod tests {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
"""
|
||||
""" # noqa: UP031 # Using an f-string here is ugly as all the curly parens need to be escaped
|
||||
% dir_name(plugin),
|
||||
)
|
||||
|
||||
|
||||
@@ -11,8 +11,9 @@ Example invocations of the script:
|
||||
but only reporting bugs that are new on your branch:
|
||||
`python scripts/fuzz-parser/fuzz.py 0-10 --new-bugs-only`
|
||||
- Run the fuzzer concurrently on 10,000 different Python source-code files,
|
||||
and only print a summary at the end:
|
||||
`python scripts/fuzz-parser/fuzz.py 1-10000 --quiet
|
||||
using a random selection of seeds, and only print a summary at the end
|
||||
(the `shuf` command is Unix-specific):
|
||||
`python scripts/fuzz-parser/fuzz.py $(shuf -i 0-1000000 -n 10000) --quiet
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -27,6 +28,7 @@ from typing import NewType
|
||||
|
||||
from pysource_codegen import generate as generate_random_code
|
||||
from pysource_minimize import minimize as minimize_repro
|
||||
from rich_argparse import RawDescriptionRichHelpFormatter
|
||||
from termcolor import colored
|
||||
|
||||
MinimizedSourceCode = NewType("MinimizedSourceCode", str)
|
||||
@@ -67,19 +69,20 @@ class FuzzResult:
|
||||
# required to trigger the bug. If not, it will be `None`.
|
||||
maybe_bug: MinimizedSourceCode | None
|
||||
|
||||
def print_description(self) -> None:
|
||||
def print_description(self, index: int, num_seeds: int) -> None:
|
||||
"""Describe the results of fuzzing the parser with this seed."""
|
||||
progress = f"[{index}/{num_seeds}]"
|
||||
msg = (
|
||||
colored(f"Ran fuzzer on seed {self.seed}", "red")
|
||||
if self.maybe_bug
|
||||
else colored(f"Ran fuzzer successfully on seed {self.seed}", "green")
|
||||
)
|
||||
print(f"{msg:<60} {progress:>15}", flush=True)
|
||||
if self.maybe_bug:
|
||||
print(colored(f"Ran fuzzer on seed {self.seed}", "red"))
|
||||
print(colored("The following code triggers a bug:", "red"))
|
||||
print()
|
||||
print(self.maybe_bug)
|
||||
print(flush=True)
|
||||
else:
|
||||
print(
|
||||
colored(f"Ran fuzzer successfully on seed {self.seed}", "green"),
|
||||
flush=True,
|
||||
)
|
||||
|
||||
|
||||
def fuzz_code(
|
||||
@@ -110,9 +113,10 @@ def fuzz_code(
|
||||
|
||||
|
||||
def run_fuzzer_concurrently(args: ResolvedCliArgs) -> list[FuzzResult]:
|
||||
num_seeds = len(args.seeds)
|
||||
print(
|
||||
f"Concurrently running the fuzzer on "
|
||||
f"{len(args.seeds)} randomly generated source-code files..."
|
||||
f"{num_seeds} randomly generated source-code files..."
|
||||
)
|
||||
bugs: list[FuzzResult] = []
|
||||
with concurrent.futures.ProcessPoolExecutor() as executor:
|
||||
@@ -127,10 +131,12 @@ def run_fuzzer_concurrently(args: ResolvedCliArgs) -> list[FuzzResult]:
|
||||
for seed in args.seeds
|
||||
]
|
||||
try:
|
||||
for future in concurrent.futures.as_completed(fuzz_result_futures):
|
||||
for i, future in enumerate(
|
||||
concurrent.futures.as_completed(fuzz_result_futures), start=1
|
||||
):
|
||||
fuzz_result = future.result()
|
||||
if not args.quiet:
|
||||
fuzz_result.print_description()
|
||||
fuzz_result.print_description(i, num_seeds)
|
||||
if fuzz_result.maybe_bug:
|
||||
bugs.append(fuzz_result)
|
||||
except KeyboardInterrupt:
|
||||
@@ -142,12 +148,13 @@ def run_fuzzer_concurrently(args: ResolvedCliArgs) -> list[FuzzResult]:
|
||||
|
||||
|
||||
def run_fuzzer_sequentially(args: ResolvedCliArgs) -> list[FuzzResult]:
|
||||
num_seeds = len(args.seeds)
|
||||
print(
|
||||
f"Sequentially running the fuzzer on "
|
||||
f"{len(args.seeds)} randomly generated source-code files..."
|
||||
f"{num_seeds} randomly generated source-code files..."
|
||||
)
|
||||
bugs: list[FuzzResult] = []
|
||||
for seed in args.seeds:
|
||||
for i, seed in enumerate(args.seeds, start=1):
|
||||
fuzz_result = fuzz_code(
|
||||
seed,
|
||||
test_executable=args.test_executable,
|
||||
@@ -155,7 +162,7 @@ def run_fuzzer_sequentially(args: ResolvedCliArgs) -> list[FuzzResult]:
|
||||
only_new_bugs=args.only_new_bugs,
|
||||
)
|
||||
if not args.quiet:
|
||||
fuzz_result.print_description()
|
||||
fuzz_result.print_description(i, num_seeds)
|
||||
if fuzz_result.maybe_bug:
|
||||
bugs.append(fuzz_result)
|
||||
return bugs
|
||||
@@ -212,7 +219,7 @@ class ResolvedCliArgs:
|
||||
def parse_args() -> ResolvedCliArgs:
|
||||
"""Parse command-line arguments"""
|
||||
parser = argparse.ArgumentParser(
|
||||
description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter
|
||||
description=__doc__, formatter_class=RawDescriptionRichHelpFormatter
|
||||
)
|
||||
parser.add_argument(
|
||||
"seeds",
|
||||
@@ -291,9 +298,19 @@ def parse_args() -> ResolvedCliArgs:
|
||||
|
||||
if not args.test_executable:
|
||||
print(
|
||||
"Running `cargo build --release` since no test executable was specified..."
|
||||
"Running `cargo build --release` since no test executable was specified...",
|
||||
flush=True,
|
||||
)
|
||||
subprocess.run(["cargo", "build", "--release"], check=True, capture_output=True)
|
||||
try:
|
||||
subprocess.run(
|
||||
["cargo", "build", "--release", "--locked", "--color", "always"],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
except subprocess.CalledProcessError as e:
|
||||
print(e.stderr)
|
||||
raise
|
||||
args.test_executable = os.path.join("target", "release", "ruff")
|
||||
assert os.path.exists(args.test_executable)
|
||||
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
pysource-codegen
|
||||
pysource-minimize
|
||||
rich-argparse
|
||||
ruff
|
||||
termcolor
|
||||
|
||||
@@ -12,11 +12,14 @@ mdurl==0.1.2
|
||||
# via markdown-it-py
|
||||
pygments==2.17.2
|
||||
# via rich
|
||||
pysource-codegen==0.5.1
|
||||
pysource-minimize==0.6.2
|
||||
pysource-codegen==0.5.2
|
||||
pysource-minimize==0.6.3
|
||||
rich==13.7.1
|
||||
# via pysource-minimize
|
||||
ruff==0.4.0
|
||||
# via
|
||||
# pysource-minimize
|
||||
# rich-argparse
|
||||
rich-argparse==1.4.0
|
||||
ruff==0.4.2
|
||||
six==1.16.0
|
||||
# via
|
||||
# asttokens
|
||||
|
||||
Reference in New Issue
Block a user