formatter: multi char tokens in SimpleTokenizer (#5610)

2023-07-10 09:00:59 +01:00
parent 52b22ceb6e
commit 1e894f328c
8 changed files with 209 additions and 33 deletions
--- a/crates/ruff_python_formatter/Cargo.toml
+++ b/crates/ruff_python_formatter/Cargo.toml
@@ -28,6 +28,7 @@ rustpython-parser = { workspace = true }
 serde = { workspace = true, optional = true }
 smallvec = { workspace = true }
 thiserror = { workspace = true }
+unic-ucd-ident = "0.9.0"

 [dev-dependencies]
 ruff_formatter = { path = "../ruff_formatter", features = ["serde"]}