From 2b4779427895fb4148d1f24a29b5ed1d9373e181 Mon Sep 17 00:00:00 2001
From: Dhruv Manilawala <dhruvmanila@gmail.com>
Date: Mon, 11 Mar 2024 09:56:19 +0530
Subject: [PATCH] Do not emit `Unknown` token

---
 crates/ruff_python_parser/src/parser/expression.rs | 6 ------
 crates/ruff_python_parser/src/token_source.rs      | 2 --
 2 files changed, 8 deletions(-)

diff --git a/crates/ruff_python_parser/src/parser/expression.rs b/crates/ruff_python_parser/src/parser/expression.rs
index e3d08482da..42da9a8db8 100644
--- a/crates/ruff_python_parser/src/parser/expression.rs
+++ b/crates/ruff_python_parser/src/parser/expression.rs
@@ -977,12 +977,6 @@ impl<'src> Parser<'src> {
                         });
                     fstring_literal
                 }
-                // `Invalid` tokens are created when there's a lexical error, so
-                // we ignore it here to avoid creating unexpected token errors
-                TokenKind::Unknown => {
-                    self.next_token();
-                    continue;
-                }
                 // Handle an unexpected token
                 _ => {
                     let (tok, range) = self.next_token();
diff --git a/crates/ruff_python_parser/src/token_source.rs b/crates/ruff_python_parser/src/token_source.rs
index e80a2aaeff..9dbbb70f3a 100644
--- a/crates/ruff_python_parser/src/token_source.rs
+++ b/crates/ruff_python_parser/src/token_source.rs
@@ -100,9 +100,7 @@ impl Iterator for TokenSource {
                 }
 
                 Err(error) => {
-                    let location = error.location();
                     self.errors.push(error);
-                    break Some((Tok::Unknown, location));
                 }
             }
         }