diff --git a/crates/ruff/resources/test/fixtures/pyflakes/F821_11.py b/crates/ruff/resources/test/fixtures/pyflakes/F821_11.py
new file mode 100644
index 0000000000..a98f55ae07
--- /dev/null
+++ b/crates/ruff/resources/test/fixtures/pyflakes/F821_11.py
@@ -0,0 +1,23 @@
+"""Test case: strings used within calls within type annotations."""
+
+from typing import Callable
+
+import bpy
+from mypy_extensions import VarArg
+
+from foo import Bar
+
+
+class LightShow(bpy.types.Operator):
+    label = "Create Character"
+    name = "lightshow.letter_creation"
+
+    filepath: bpy.props.StringProperty(subtype="FILE_PATH")  # OK
+
+
+def f(x: Callable[[VarArg("os")], None]):  # F821
+    pass
+
+
+f(Callable[["Bar"], None])
+f(Callable[["Baz"], None])
diff --git a/crates/ruff/resources/test/fixtures/pyflakes/F821_12.py b/crates/ruff/resources/test/fixtures/pyflakes/F821_12.py
new file mode 100644
index 0000000000..6c4384ac54
--- /dev/null
+++ b/crates/ruff/resources/test/fixtures/pyflakes/F821_12.py
@@ -0,0 +1,25 @@
+"""Test case: strings used within calls within type annotations."""
+
+from __future__ import annotations
+
+from typing import Callable
+
+import bpy
+from mypy_extensions import VarArg
+
+from foo import Bar
+
+
+class LightShow(bpy.types.Operator):
+    label = "Create Character"
+    name = "lightshow.letter_creation"
+
+    filepath: bpy.props.StringProperty(subtype="FILE_PATH")  # OK
+
+
+def f(x: Callable[[VarArg("os")], None]):  # F821
+    pass
+
+
+f(Callable[["Bar"], None])
+f(Callable[["Baz"], None])
diff --git a/crates/ruff/src/checkers/ast.rs b/crates/ruff/src/checkers/ast.rs
index d23da1a6d3..365c7aca65 100644
--- a/crates/ruff/src/checkers/ast.rs
+++ b/crates/ruff/src/checkers/ast.rs
@@ -3635,9 +3635,16 @@ where
                         Some(Callable::NamedTuple)
                     } else if self.match_typing_call_path(&call_path, "TypedDict") {
                         Some(Callable::TypedDict)
-                    } else if ["Arg", "DefaultArg", "NamedArg", "DefaultNamedArg"]
-                        .iter()
-                        .any(|target| call_path.as_slice() == ["mypy_extensions", target])
+                    } else if [
+                        "Arg",
+                        "DefaultArg",
+                        "NamedArg",
+                        "DefaultNamedArg",
+                        "VarArg",
+                        "KwArg",
+                    ]
+                    .iter()
+                    .any(|target| call_path.as_slice() == ["mypy_extensions", target])
                     {
                         Some(Callable::MypyExtension)
                     } else {
@@ -3760,7 +3767,17 @@ where
                         }
                     }
                     None => {
-                        visitor::walk_expr(self, expr);
+                        // If we're in a type definition, we need to treat the arguments to any
+                        // other callables as non-type definitions (i.e., we don't want to treat
+                        // any strings as deferred type definitions).
+                        self.visit_expr(func);
+                        for arg in args {
+                            visit_non_type_definition!(self, arg);
+                        }
+                        for keyword in keywords {
+                            let KeywordData { value, .. } = &keyword.node;
+                            visit_non_type_definition!(self, value);
+                        }
                     }
                 }
             }
diff --git a/crates/ruff/src/rules/pyflakes/mod.rs b/crates/ruff/src/rules/pyflakes/mod.rs
index fcaa7888f3..b10a75e338 100644
--- a/crates/ruff/src/rules/pyflakes/mod.rs
+++ b/crates/ruff/src/rules/pyflakes/mod.rs
@@ -104,6 +104,8 @@ mod tests {
     #[test_case(Rule::UndefinedName, Path::new("F821_8.pyi"); "F821_8")]
     #[test_case(Rule::UndefinedName, Path::new("F821_9.py"); "F821_9")]
     #[test_case(Rule::UndefinedName, Path::new("F821_10.py"); "F821_10")]
+    #[test_case(Rule::UndefinedName, Path::new("F821_11.py"); "F821_11")]
+    #[test_case(Rule::UndefinedName, Path::new("F821_12.py"); "F821_12")]
     #[test_case(Rule::UndefinedExport, Path::new("F822_0.py"); "F822_0")]
     #[test_case(Rule::UndefinedExport, Path::new("F822_1.py"); "F822_1")]
     #[test_case(Rule::UndefinedExport, Path::new("F822_2.py"); "F822_2")]
diff --git a/crates/ruff/src/rules/pyflakes/snapshots/ruff__rules__pyflakes__tests__F821_F821_11.py.snap b/crates/ruff/src/rules/pyflakes/snapshots/ruff__rules__pyflakes__tests__F821_F821_11.py.snap
new file mode 100644
index 0000000000..a9177c79f2
--- /dev/null
+++ b/crates/ruff/src/rules/pyflakes/snapshots/ruff__rules__pyflakes__tests__F821_F821_11.py.snap
@@ -0,0 +1,27 @@
+---
+source: crates/ruff/src/rules/pyflakes/mod.rs
+expression: diagnostics
+---
+- kind:
+    UndefinedName:
+      name: os
+  location:
+    row: 18
+    column: 26
+  end_location:
+    row: 18
+    column: 30
+  fix: ~
+  parent: ~
+- kind:
+    UndefinedName:
+      name: Baz
+  location:
+    row: 23
+    column: 12
+  end_location:
+    row: 23
+    column: 17
+  fix: ~
+  parent: ~
+
diff --git a/crates/ruff/src/rules/pyflakes/snapshots/ruff__rules__pyflakes__tests__F821_F821_12.py.snap b/crates/ruff/src/rules/pyflakes/snapshots/ruff__rules__pyflakes__tests__F821_F821_12.py.snap
new file mode 100644
index 0000000000..68a60b3eb5
--- /dev/null
+++ b/crates/ruff/src/rules/pyflakes/snapshots/ruff__rules__pyflakes__tests__F821_F821_12.py.snap
@@ -0,0 +1,27 @@
+---
+source: crates/ruff/src/rules/pyflakes/mod.rs
+expression: diagnostics
+---
+- kind:
+    UndefinedName:
+      name: os
+  location:
+    row: 20
+    column: 26
+  end_location:
+    row: 20
+    column: 30
+  fix: ~
+  parent: ~
+- kind:
+    UndefinedName:
+      name: Baz
+  location:
+    row: 25
+    column: 12
+  end_location:
+    row: 25
+    column: 17
+  fix: ~
+  parent: ~
+
diff --git a/crates/ruff/src/rules/pyupgrade/snapshots/ruff__rules__pyupgrade__tests__UP037.py.snap b/crates/ruff/src/rules/pyupgrade/snapshots/ruff__rules__pyupgrade__tests__UP037.py.snap
index efd2a6ad26..276fbfba31 100644
--- a/crates/ruff/src/rules/pyupgrade/snapshots/ruff__rules__pyupgrade__tests__UP037.py.snap
+++ b/crates/ruff/src/rules/pyupgrade/snapshots/ruff__rules__pyupgrade__tests__UP037.py.snap
@@ -478,21 +478,4 @@ expression: diagnostics
       row: 67
       column: 49
   parent: ~
-- kind:
-    QuotedAnnotation: ~
-  location:
-    row: 69
-    column: 14
-  end_location:
-    row: 69
-    column: 17
-  fix:
-    content: X
-    location:
-      row: 69
-      column: 14
-    end_location:
-      row: 69
-      column: 17
-  parent: ~