Cyclic0007/ruff
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[ty] implement typing.NewType by adding Type::NewTypeInstance

Browse Source
This commit is contained in:
Jack O'Connor
2025-10-23 10:10:10 -07:00
parent 039a69fa8c
commit 5f3e086ee4
25 changed files with 1343 additions and 191 deletions
Download Patch File Download Diff File Expand all files Collapse all files

151
crates/ty_python_semantic/src/types.rs
View File

@@ -66,6 +66,7 @@ use crate::types::generics::{
use crate::types::infer::infer_unpack_types;
use crate::types::mro::{Mro, MroError, MroIterator};
pub(crate) use crate::types::narrow::infer_narrowing_constraint;
use crate::types::newtype::NewType;
use crate::types::signatures::{ParameterForm, walk_signature};
use crate::types::tuple::{TupleSpec, TupleSpecBuilder};
pub(crate) use crate::types::typed_dict::{TypedDictParams, TypedDictType, walk_typed_dict_type};
@@ -98,6 +99,7 @@ mod instance;
mod member;
mod mro;
mod narrow;
mod newtype;
mod protocol_class;
mod signatures;
mod special_form;
@@ -783,6 +785,13 @@ pub enum Type<'db> {
TypedDict(TypedDictType<'db>),
/// An aliased type (lazily not-yet-unpacked to its value type).
TypeAlias(TypeAliasType<'db>),
/// The set of Python objects that belong to a `typing.NewType` subtype. Note that
/// `typing.NewType` itself is a `Type::ClassLiteral` with `KnownClass::NewType`, and the
/// identity callables it returns (which behave like subtypes in type expressions) are of
/// `Type::KnownInstance` with `KnownInstanceType::NewType`. This `Type` refers to the objects
/// wrapped/returned by a specific one of those identity callables, or by another that inherits
/// from it.
NewTypeInstance(NewType<'db>),
}
#[salsa::tracked]
@@ -1420,6 +1429,13 @@ impl<'db> Type<'db> {
self
}
Type::TypeAlias(alias) => alias.value_type(db).normalized_impl(db, visitor),
Type::NewTypeInstance(newtype) => {
visitor.visit(self, || {
Type::NewTypeInstance(newtype.map_base_class_type(db, |class_type| {
class_type.normalized_impl(db, visitor)
}))
})
}
Type::LiteralString
| Type::AlwaysFalsy
| Type::AlwaysTruthy
@@ -1482,7 +1498,8 @@ impl<'db> Type<'db> {
| Type::BoundSuper(_)
| Type::TypeIs(_)
| Type::TypedDict(_)
| Type::TypeAlias(_) => false,
| Type::TypeAlias(_)
| Type::NewTypeInstance(_) => false,
}
}
@@ -1520,6 +1537,10 @@ impl<'db> Type<'db> {
Type::GenericAlias(alias) => Some(ClassType::Generic(alias).into_callable(db)),
Type::NewTypeInstance(newtype) => {
Type::instance(db, newtype.base_class_type(db)).try_upcast_to_callable(db)
}
// TODO: This is unsound so in future we can consider an opt-in option to disable it.
Type::SubclassOf(subclass_of_ty) => match subclass_of_ty.subclass_of() {
SubclassOfInner::Class(class) => Some(class.into_callable(db)),
@@ -1549,6 +1570,15 @@ impl<'db> Type<'db> {
false,
))),
Type::KnownInstance(KnownInstanceType::NewType(newtype)) => Some(CallableType::single(
db,
Signature::new(
Parameters::new([Parameter::positional_only(None)
.with_annotated_type(newtype.base(db).instance_type(db))]),
Some(Type::NewTypeInstance(newtype)),
),
)),
Type::Never
| Type::DataclassTransformer(_)
| Type::AlwaysTruthy
@@ -2429,6 +2459,22 @@ impl<'db> Type<'db> {
})
}
(Type::NewTypeInstance(self_newtype), Type::NewTypeInstance(target_newtype)) => {
self_newtype.has_relation_to_impl(db, target_newtype)
}
(
Type::NewTypeInstance(self_newtype),
Type::NominalInstance(target_nominal_instance),
) => self_newtype.base_class_type(db).has_relation_to_impl(
db,
target_nominal_instance.class(db),
inferable,
relation,
relation_visitor,
disjointness_visitor,
),
(Type::PropertyInstance(_), _) => {
KnownClass::Property.to_instance(db).has_relation_to_impl(
db,
@@ -2448,14 +2494,15 @@ impl<'db> Type<'db> {
disjointness_visitor,
),
// Other than the special cases enumerated above, `Instance` types and typevars are
// never subtypes of any other variants
// Other than the special cases enumerated above, nominal-instance types,
// newtype-instance types, and typevars are never subtypes of any other variants
(Type::TypeVar(bound_typevar), _) => {
// All inferable cases should have been handled above
assert!(!bound_typevar.is_inferable(db, inferable));
ConstraintSet::from(false)
}
(Type::NominalInstance(_), _) => ConstraintSet::from(false),
(Type::NewTypeInstance(_), _) => ConstraintSet::from(false),
}
}
@@ -2529,6 +2576,10 @@ impl<'db> Type<'db> {
})
}
(Type::NewTypeInstance(self_newtype), Type::NewTypeInstance(other_newtype)) => {
ConstraintSet::from(self_newtype.is_equivalent_to_impl(db, other_newtype))
}
(Type::NominalInstance(first), Type::NominalInstance(second)) => {
first.is_equivalent_to_impl(db, second, inferable, visitor)
}
@@ -3288,6 +3339,19 @@ impl<'db> Type<'db> {
)
}),
(Type::NewTypeInstance(left), Type::NewTypeInstance(right)) => {
left.is_disjoint_from_impl(db, right)
}
(Type::NewTypeInstance(newtype), other) | (other, Type::NewTypeInstance(newtype)) => {
Type::instance(db, newtype.base_class_type(db)).is_disjoint_from_impl(
db,
other,
inferable,
disjointness_visitor,
relation_visitor,
)
}
(Type::PropertyInstance(_), other) | (other, Type::PropertyInstance(_)) => {
KnownClass::Property.to_instance(db).is_disjoint_from_impl(
db,
@@ -3432,6 +3496,9 @@ impl<'db> Type<'db> {
Type::TypeIs(type_is) => type_is.is_bound(db),
Type::TypedDict(_) => false,
Type::TypeAlias(alias) => alias.value_type(db).is_singleton(db),
Type::NewTypeInstance(newtype) => {
Type::instance(db, newtype.base_class_type(db)).is_singleton(db)
}
}
}
@@ -3482,6 +3549,9 @@ impl<'db> Type<'db> {
}
Type::NominalInstance(instance) => instance.is_single_valued(db),
Type::NewTypeInstance(newtype) => {
Type::instance(db, newtype.base_class_type(db)).is_single_valued(db)
}
Type::BoundSuper(_) => {
// At runtime two super instances never compare equal, even if their arguments are identical.
@@ -3645,7 +3715,8 @@ impl<'db> Type<'db> {
| Type::ProtocolInstance(_)
| Type::PropertyInstance(_)
| Type::TypeIs(_)
| Type::TypedDict(_) => None,
| Type::TypedDict(_)
| Type::NewTypeInstance(_) => None,
}
}
@@ -3732,6 +3803,7 @@ impl<'db> Type<'db> {
Type::Dynamic(_) | Type::Never => Place::bound(self).into(),
Type::NominalInstance(instance) => instance.class(db).instance_member(db, name),
Type::NewTypeInstance(newtype) => newtype.base_class_type(db).instance_member(db, name),
Type::ProtocolInstance(protocol) => protocol.instance_member(db, name),
@@ -4404,6 +4476,7 @@ impl<'db> Type<'db> {
Type::NominalInstance(..)
| Type::ProtocolInstance(..)
| Type::NewTypeInstance(..)
| Type::BooleanLiteral(..)
| Type::IntLiteral(..)
| Type::StringLiteral(..)
@@ -4842,6 +4915,8 @@ impl<'db> Type<'db> {
.value_type(db)
.try_bool_impl(db, allow_short_circuit, visitor)
})?,
Type::NewTypeInstance(newtype) => Type::instance(db, newtype.base_class_type(db))
.try_bool_impl(db, allow_short_circuit, visitor)?,
};
Ok(truthiness)
@@ -5528,7 +5603,7 @@ impl<'db> Type<'db> {
SubclassOfInner::Class(class) => Type::from(class).bindings(db),
},
Type::NominalInstance(_) | Type::ProtocolInstance(_) => {
Type::NominalInstance(_) | Type::ProtocolInstance(_) | Type::NewTypeInstance(_) => {
// Note that for objects that have a (possibly not callable!) `__call__` attribute,
// we will get the signature of the `__call__` attribute, but will pass in the type
// of the original object as the "callable type". That ensures that we get errors
@@ -5581,6 +5656,16 @@ impl<'db> Type<'db> {
Type::EnumLiteral(enum_literal) => enum_literal.enum_class_instance(db).bindings(db),
Type::KnownInstance(KnownInstanceType::NewType(newtype)) => Binding::single(
self,
Signature::new(
Parameters::new([Parameter::positional_only(None)
.with_annotated_type(newtype.base(db).instance_type(db))]),
Some(Type::NewTypeInstance(newtype)),
),
)
.into(),
Type::KnownInstance(known_instance) => {
known_instance.instance_fallback(db).bindings(db)
}
@@ -5716,6 +5801,7 @@ impl<'db> Type<'db> {
match ty {
Type::NominalInstance(nominal) => nominal.tuple_spec(db),
Type::NewTypeInstance(newtype) => non_async_special_case(db, Type::instance(db, newtype.base_class_type(db))),
Type::GenericAlias(alias) if alias.origin(db).is_tuple(db) => {
Some(Cow::Owned(TupleSpec::homogeneous(todo_type!(
"*tuple[] annotations"
@@ -6346,6 +6432,9 @@ impl<'db> Type<'db> {
Type::ClassLiteral(class) => Some(Type::instance(db, class.default_specialization(db))),
Type::GenericAlias(alias) => Some(Type::instance(db, ClassType::from(alias))),
Type::SubclassOf(subclass_of_ty) => Some(subclass_of_ty.to_instance(db)),
Type::KnownInstance(KnownInstanceType::NewType(newtype)) => {
Some(Type::NewTypeInstance(newtype))
}
Type::Union(union) => union.to_instance(db),
// If there is no bound or constraints on a typevar `T`, `T: object` implicitly, which
// has no instance type. Otherwise, synthesize a typevar with bound or constraints
@@ -6376,7 +6465,8 @@ impl<'db> Type<'db> {
| Type::AlwaysTruthy
| Type::AlwaysFalsy
| Type::TypeIs(_)
| Type::TypedDict(_) => None,
| Type::TypedDict(_)
| Type::NewTypeInstance(_) => None,
}
}
@@ -6455,6 +6545,7 @@ impl<'db> Type<'db> {
Type::KnownInstance(known_instance) => match known_instance {
KnownInstanceType::TypeAliasType(alias) => Ok(Type::TypeAlias(*alias)),
KnownInstanceType::NewType(newtype) => Ok(Type::NewTypeInstance(*newtype)),
KnownInstanceType::TypeVar(typevar) => {
let index = semantic_index(db, scope_id.file(db));
Ok(bind_typevar(
@@ -6669,9 +6760,6 @@ impl<'db> Type<'db> {
Some(KnownClass::TypeVarTuple) => Ok(todo_type!(
"Support for `typing.TypeVarTuple` instances in type expressions"
)),
Some(KnownClass::NewType) => Ok(todo_type!(
"Support for `typing.NewType` instances in type expressions"
)),
Some(KnownClass::GenericAlias) => Ok(todo_type!(
"Support for `typing.GenericAlias` instances in type expressions"
)),
@@ -6690,6 +6778,13 @@ impl<'db> Type<'db> {
.value_type(db)
.in_type_expression(db, scope_id, typevar_binding_context)
}
Type::NewTypeInstance(_) => Err(InvalidTypeExpressionError {
invalid_expressions: smallvec::smallvec_inline![
InvalidTypeExpression::InvalidType(*self, scope_id)
],
fallback_type: Type::unknown(),
}),
}
}
@@ -6764,6 +6859,7 @@ impl<'db> Type<'db> {
// understand a more specific meta type in order to correctly handle `__getitem__`.
Type::TypedDict(typed_dict) => SubclassOfType::from(db, typed_dict.defining_class()),
Type::TypeAlias(alias) => alias.value_type(db).to_meta_type(db),
Type::NewTypeInstance(newtype) => Type::from(newtype.base_class_type(db)),
}
}
@@ -6873,8 +6969,8 @@ impl<'db> Type<'db> {
| TypeMapping::ReplaceParameterDefaults
| TypeMapping::BindLegacyTypevars(_) => self,
TypeMapping::Materialize(materialization_kind) => {
Type::TypeVar(bound_typevar.materialize_impl(db, *materialization_kind, visitor))
}
Type::TypeVar(bound_typevar.materialize_impl(db, *materialization_kind, visitor))
}
}
Type::KnownInstance(KnownInstanceType::TypeVar(typevar)) => match type_mapping {
@@ -6909,6 +7005,12 @@ impl<'db> Type<'db> {
instance.apply_type_mapping_impl(db, type_mapping, tcx, visitor)
},
Type::NewTypeInstance(newtype) => visitor.visit(self, || {
Type::NewTypeInstance(newtype.map_base_class_type(db, |class_type| {
class_type.apply_type_mapping_impl(db, type_mapping, tcx, visitor)
}))
}),
Type::ProtocolInstance(instance) => {
// TODO: Add tests for materialization once subtyping/assignability is implemented for
// protocols. It _might_ require changing the logic here because:
@@ -7150,6 +7252,12 @@ impl<'db> Type<'db> {
instance.find_legacy_typevars_impl(db, binding_context, typevars, visitor);
}
Type::NewTypeInstance(_) => {
// A newtype can never be constructed from an unspecialized generic class, so it is
// impossible that we could ever find any legacy typevars in a newtype instance or
// its underlying class.
}
Type::SubclassOf(subclass_of) => {
subclass_of.find_legacy_typevars_impl(db, binding_context, typevars, visitor);
}
@@ -7305,6 +7413,7 @@ impl<'db> Type<'db> {
},
Self::TypeAlias(alias) => alias.value_type(db).definition(db),
Self::NewTypeInstance(newtype) => Some(TypeDefinition::NewType(newtype.definition(db))),
Self::StringLiteral(_)
| Self::BooleanLiteral(_)
@@ -7528,7 +7637,8 @@ impl<'db> VarianceInferable<'db> for Type<'db> {
| Type::BoundSuper(_)
| Type::TypeVar(_)
| Type::TypedDict(_)
| Type::TypeAlias(_) => TypeVarVariance::Bivariant,
| Type::TypeAlias(_)
| Type::NewTypeInstance(_) => TypeVarVariance::Bivariant,
};
tracing::trace!(
@@ -7726,6 +7836,10 @@ pub enum KnownInstanceType<'db> {
/// A single instance of `typing.Annotated`
Annotated(InternedType<'db>),
/// An identity callable created with `typing.NewType(name, base)`, which behaves like a
/// subtype of `base` in type expressions. See the `struct NewType` payload for an example.
NewType(NewType<'db>),
}
fn walk_known_instance_type<'db, V: visitor::TypeVisitor<'db> + ?Sized>(
@@ -7760,6 +7874,11 @@ fn walk_known_instance_type<'db, V: visitor::TypeVisitor<'db> + ?Sized>(
KnownInstanceType::Literal(ty) | KnownInstanceType::Annotated(ty) => {
visitor.visit_type(db, ty.inner(db));
}
KnownInstanceType::NewType(newtype) => {
if let ClassType::Generic(generic_alias) = newtype.base_class_type(db) {
visitor.visit_generic_alias_type(db, generic_alias);
}
}
}
}
@@ -7799,6 +7918,10 @@ impl<'db> KnownInstanceType<'db> {
Self::UnionType(list) => Self::UnionType(list.normalized_impl(db, visitor)),
Self::Literal(ty) => Self::Literal(ty.normalized_impl(db, visitor)),
Self::Annotated(ty) => Self::Annotated(ty.normalized_impl(db, visitor)),
Self::NewType(newtype) => Self::NewType(
newtype
.map_base_class_type(db, |class_type| class_type.normalized_impl(db, visitor)),
),
}
}
@@ -7819,6 +7942,7 @@ impl<'db> KnownInstanceType<'db> {
Self::UnionType(_) => KnownClass::UnionType,
Self::Literal(_) => KnownClass::GenericAlias,
Self::Annotated(_) => KnownClass::GenericAlias,
Self::NewType(_) => KnownClass::NewType,
}
}
@@ -7903,6 +8027,9 @@ impl<'db> KnownInstanceType<'db> {
KnownInstanceType::Annotated(_) => {
f.write_str("<typing.Annotated special form>")
}
KnownInstanceType::NewType(declaration) => {
write!(f, "<NewType pseudo-class '{}'>", declaration.name(self.db))
}
}
}
}

3
crates/ty_python_semantic/src/types/bound_super.rs
View File

@@ -404,6 +404,9 @@ impl<'db> BoundSuperType<'db> {
.to_specialized_instance(db, [key_builder.build(), value_builder.build()]),
);
}
Type::NewTypeInstance(newtype) => {
return delegate_to(Type::instance(db, newtype.base_class_type(db)));
}
Type::Callable(callable) if callable.is_function_like(db) => {
return delegate_to(KnownClass::FunctionType.to_instance(db));
}

8
crates/ty_python_semantic/src/types/class.rs
View File

@@ -358,6 +358,14 @@ pub enum ClassType<'db> {
#[salsa::tracked]
impl<'db> ClassType<'db> {
/// Return a `ClassType` representing the class `builtins.object`
pub(super) fn object(db: &'db dyn Db) -> Self {
KnownClass::Object
.to_class_literal(db)
.to_class_type(db)
.unwrap()
}
pub(super) const fn is_generic(self) -> bool {
matches!(self, Self::Generic(_))
}

12
crates/ty_python_semantic/src/types/class_base.rs
View File

@@ -137,6 +137,12 @@ impl<'db> ClassBase<'db> {
Type::TypeAlias(alias) => Self::try_from_type(db, alias.value_type(db), subclass),
Type::NewTypeInstance(newtype) => ClassBase::try_from_type(
db,
Type::instance(db, newtype.base_class_type(db)),
subclass,
),
Type::PropertyInstance(_)
| Type::BooleanLiteral(_)
| Type::FunctionLiteral(_)
@@ -169,7 +175,11 @@ impl<'db> ClassBase<'db> {
| KnownInstanceType::Field(_)
| KnownInstanceType::ConstraintSet(_)
| KnownInstanceType::UnionType(_)
| KnownInstanceType::Literal(_) => None,
| KnownInstanceType::Literal(_)
// A class inheriting from a newtype would make intuitive sense, but newtype
// wrappers are just identity callables at runtime, so this sort of inheritance
// doesn't work and isn't allowed.
| KnownInstanceType::NewType(_) => None,
KnownInstanceType::Annotated(ty) => Self::try_from_type(db, ty.inner(db), subclass),
},

7
crates/ty_python_semantic/src/types/definition.rs
View File

@@ -12,6 +12,7 @@ pub enum TypeDefinition<'db> {
Function(Definition<'db>),
TypeVar(Definition<'db>),
TypeAlias(Definition<'db>),
NewType(Definition<'db>),
}
impl TypeDefinition<'_> {
@@ -21,7 +22,8 @@ impl TypeDefinition<'_> {
Self::Class(definition)
| Self::Function(definition)
| Self::TypeVar(definition)
| Self::TypeAlias(definition) => {
| Self::TypeAlias(definition)
| Self::NewType(definition) => {
let module = parsed_module(db, definition.file(db)).load(db);
Some(definition.focus_range(db, &module))
}
@@ -38,7 +40,8 @@ impl TypeDefinition<'_> {
Self::Class(definition)
| Self::Function(definition)
| Self::TypeVar(definition)
| Self::TypeAlias(definition) => {
| Self::TypeAlias(definition)
| Self::NewType(definition) => {
let module = parsed_module(db, definition.file(db)).load(db);
Some(definition.full_range(db, &module))
}

44
crates/ty_python_semantic/src/types/diagnostic.rs
View File

@@ -12,6 +12,7 @@ use crate::semantic_index::definition::{Definition, DefinitionKind};
use crate::semantic_index::place::{PlaceTable, ScopedPlaceId};
use crate::semantic_index::{global_scope, place_table};
use crate::suppression::FileSuppressionId;
use crate::types::KnownInstanceType;
use crate::types::call::CallError;
use crate::types::class::{DisjointBase, DisjointBaseKind, Field};
use crate::types::function::KnownFunction;
@@ -65,6 +66,7 @@ pub(crate) fn register_lints(registry: &mut LintRegistryBuilder) {
registry.register_lint(&INVALID_LEGACY_TYPE_VARIABLE);
registry.register_lint(&INVALID_PARAMSPEC);
registry.register_lint(&INVALID_TYPE_ALIAS_TYPE);
registry.register_lint(&INVALID_NEWTYPE);
registry.register_lint(&INVALID_METACLASS);
registry.register_lint(&INVALID_OVERLOAD);
registry.register_lint(&USELESS_OVERLOAD_BODY);
@@ -926,6 +928,30 @@ declare_lint! {
}
}
declare_lint! {
/// ## What it does
/// Checks for the creation of invalid `NewType`s
///
/// ## Why is this bad?
/// There are several requirements that you must follow when creating a `NewType`.
///
/// ## Examples
/// ```python
/// from typing import NewType
///
/// def get_name() -> str: ...
///
/// Foo = NewType("Foo", int) # okay
/// Bar = NewType(get_name(), int) # error: The first argument to `NewType` must be a string literal
/// Baz = NewType("Baz", int | str) # error: invalid base for `typing.NewType`
/// ```
pub(crate) static INVALID_NEWTYPE = {
summary: "detects invalid NewType definitions",
status: LintStatus::preview("1.0.0"),
default_level: Level::Error,
}
}
declare_lint! {
/// ## What it does
/// Checks for arguments to `metaclass=` that are invalid.
@@ -2898,6 +2924,24 @@ pub(crate) fn report_invalid_or_unsupported_base(
return;
}
if let Type::KnownInstance(KnownInstanceType::NewType(newtype)) = base_type {
let Some(builder) = context.report_lint(&INVALID_BASE, base_node) else {
return;
};
let mut diagnostic = builder.into_diagnostic("Cannot subclass an instance of NewType");
diagnostic.info(format_args!(
"Perhaps you were looking for: `{} = NewType('{}', {})`",
class.name(context.db()),
class.name(context.db()),
newtype.name(context.db()),
));
diagnostic.info(format_args!(
"Definition of class `{}` will raise `TypeError` at runtime",
class.name(context.db())
));
return;
}
let tuple_of_types = Type::homogeneous_tuple(db, instance_of_type);
let explain_mro_entries = |diagnostic: &mut LintDiagnosticGuard| {

1
crates/ty_python_semantic/src/types/display.rs
View File

@@ -618,6 +618,7 @@ impl Display for DisplayRepresentation<'_> {
.fmt(f),
}
}
Type::NewTypeInstance(newtype) => f.write_str(newtype.name(self.db)),
}
}
}

5
crates/ty_python_semantic/src/types/function.rs
View File

@@ -1101,6 +1101,11 @@ fn is_instance_truthiness<'db>(
Type::NominalInstance(..) => always_true_if(is_instance(&ty)),
Type::NewTypeInstance(newtype) => always_true_if(is_instance(&Type::instance(
db,
newtype.base_class_type(db),
))),
Type::BooleanLiteral(..)
| Type::BytesLiteral(..)
| Type::IntLiteral(..)

4
crates/ty_python_semantic/src/types/ide_support.rs
View File

@@ -128,6 +128,10 @@ impl<'db> AllMembers<'db> {
}
}
Type::NewTypeInstance(newtype) => {
self.extend_with_type(db, Type::instance(db, newtype.base_class_type(db)));
}
Type::ClassLiteral(class_literal) if class_literal.is_typed_dict(db) => {
self.extend_with_type(db, KnownClass::TypedDictFallback.to_class_literal(db));
}

176
crates/ty_python_semantic/src/types/infer/builder.rs
View File

@@ -59,8 +59,8 @@ use crate::types::diagnostic::{
DIVISION_BY_ZERO, DUPLICATE_KW_ONLY, INCONSISTENT_MRO, INVALID_ARGUMENT_TYPE,
INVALID_ASSIGNMENT, INVALID_ATTRIBUTE_ACCESS, INVALID_BASE, INVALID_DECLARATION,
INVALID_GENERIC_CLASS, INVALID_KEY, INVALID_LEGACY_TYPE_VARIABLE, INVALID_METACLASS,
INVALID_NAMED_TUPLE, INVALID_OVERLOAD, INVALID_PARAMETER_DEFAULT, INVALID_PARAMSPEC,
INVALID_PROTOCOL, INVALID_TYPE_FORM, INVALID_TYPE_GUARD_CALL,
INVALID_NAMED_TUPLE, INVALID_NEWTYPE, INVALID_OVERLOAD, INVALID_PARAMETER_DEFAULT,
INVALID_PARAMSPEC, INVALID_PROTOCOL, INVALID_TYPE_FORM, INVALID_TYPE_GUARD_CALL,
INVALID_TYPE_VARIABLE_CONSTRAINTS, IncompatibleBases, NON_SUBSCRIPTABLE,
POSSIBLY_MISSING_IMPLICIT_CALL, POSSIBLY_MISSING_IMPORT, SUBCLASS_OF_FINAL_CLASS,
UNDEFINED_REVEAL, UNRESOLVED_ATTRIBUTE, UNRESOLVED_GLOBAL, UNRESOLVED_IMPORT,
@@ -90,6 +90,7 @@ use crate::types::generics::{
use crate::types::infer::nearest_enclosing_function;
use crate::types::instance::SliceLiteral;
use crate::types::mro::MroErrorKind;
use crate::types::newtype::NewType;
use crate::types::signatures::Signature;
use crate::types::subclass_of::SubclassOfInner;
use crate::types::tuple::{Tuple, TupleLength, TupleSpec, TupleType};
@@ -3884,7 +3885,8 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> {
| Type::AlwaysTruthy
| Type::AlwaysFalsy
| Type::TypeIs(_)
| Type::TypedDict(_) => {
| Type::TypedDict(_)
| Type::NewTypeInstance(_) => {
// TODO: We could use the annotated parameter type of `__setattr__` as type context here.
// However, we would still have to perform the first inference without type context.
let value_ty = infer_value_ty(self, TypeContext::default());
@@ -4454,6 +4456,9 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> {
Some(KnownClass::ParamSpec) => {
self.infer_paramspec(target, call_expr, definition)
}
Some(KnownClass::NewType) => {
self.infer_newtype_expression(target, call_expr, definition)
}
Some(_) | None => {
self.infer_call_expression_impl(call_expr, callable_type, tcx)
}
@@ -4892,14 +4897,114 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> {
)))
}
fn infer_newtype_expression(
&mut self,
target: &ast::Expr,
call_expr: &ast::ExprCall,
definition: Definition<'db>,
) -> Type<'db> {
fn error<'db>(
context: &InferContext<'db, '_>,
message: impl std::fmt::Display,
node: impl Ranged,
) -> Type<'db> {
if let Some(builder) = context.report_lint(&INVALID_NEWTYPE, node) {
builder.into_diagnostic(message);
}
Type::unknown()
}
let db = self.db();
let arguments = &call_expr.arguments;
if !arguments.keywords.is_empty() {
return error(
&self.context,
"Keyword arguments are not supported in `NewType` creation",
call_expr,
);
}
if let Some(starred) = arguments.args.iter().find(|arg| arg.is_starred_expr()) {
return error(
&self.context,
"Starred arguments are not supported in `NewType` creation",
starred,
);
}
if arguments.args.len() != 2 {
return error(
&self.context,
format!(
"Wrong number of arguments in `NewType` creation, expected 2, found {}",
arguments.args.len()
),
call_expr,
);
}
let name_param_ty = self.infer_expression(&arguments.args[0], TypeContext::default());
let Some(name) = name_param_ty.as_string_literal().map(|name| name.value(db)) else {
return error(
&self.context,
"The first argument to `NewType` must be a string literal",
call_expr,
);
};
let ast::Expr::Name(ast::ExprName {
id: target_name, ..
}) = target
else {
return error(
&self.context,
"A `NewType` definition must be a simple variable assignment",
target,
);
};
if name != target_name {
return error(
&self.context,
format_args!(
"The name of a `NewType` (`{name}`) must match \
the name of the variable it is assigned to (`{target_name}`)"
),
target,
);
}
// Inference of `tp` must be deferred, to avoid cycles.
self.deferred.insert(definition, self.multi_inference_state);
Type::KnownInstance(KnownInstanceType::NewType(NewType::new(
db,
ast::name::Name::from(name),
definition,
None,
)))
}
fn infer_assignment_deferred(&mut self, value: &ast::Expr) {
// Infer deferred bounds/constraints/defaults of a legacy TypeVar / ParamSpec.
// Infer deferred bounds/constraints/defaults of a legacy TypeVar / ParamSpec / NewType.
let ast::Expr::Call(ast::ExprCall {
func, arguments, ..
}) = value
else {
return;
};
let func_ty = self
.try_expression_type(func)
.unwrap_or_else(|| self.infer_expression(func, TypeContext::default()));
let known_class = func_ty
.as_class_literal()
.and_then(|cls| cls.known(self.db()));
if let Some(KnownClass::NewType) = known_class {
self.infer_newtype_assignment_deferred(arguments);
return;
}
for arg in arguments.args.iter().skip(1) {
self.infer_type_expression(arg);
}
@@ -4907,12 +5012,7 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> {
self.infer_type_expression(&bound.value);
}
if let Some(default) = arguments.find_keyword("default") {
let func_ty = self
.try_expression_type(func)
.unwrap_or_else(|| self.infer_expression(func, TypeContext::default()));
if func_ty.as_class_literal().is_some_and(|class_literal| {
class_literal.is_known(self.db(), KnownClass::ParamSpec)
}) {
if let Some(KnownClass::ParamSpec) = known_class {
self.infer_paramspec_default(&default.value);
} else {
self.infer_type_expression(&default.value);
@@ -4920,6 +5020,34 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> {
}
}
// Infer the deferred base type of a NewType.
fn infer_newtype_assignment_deferred(&mut self, arguments: &ast::Arguments) {
match self.infer_type_expression(&arguments.args[1]) {
Type::NominalInstance(_) | Type::NewTypeInstance(_) => {}
// `Unknown` is likely to be the result of an unresolved import or a typo, which will
// already get a diagnostic, so don't pile on an extra diagnostic here.
Type::Dynamic(DynamicType::Unknown) => {}
other_type => {
if let Some(builder) = self
.context
.report_lint(&INVALID_NEWTYPE, &arguments.args[1])
{
let mut diag = builder.into_diagnostic("invalid base for `typing.NewType`");
diag.set_primary_message(format!("type `{}`", other_type.display(self.db())));
if matches!(other_type, Type::ProtocolInstance(_)) {
diag.info("The base of a `NewType` is not allowed to be a protocol class.");
} else if matches!(other_type, Type::TypedDict(_)) {
diag.info("The base of a `NewType` is not allowed to be a `TypedDict`.");
} else {
diag.info(
"The base of a `NewType` must be a class type or another `NewType`.",
);
}
}
}
}
}
fn infer_annotated_assignment_statement(&mut self, assignment: &ast::StmtAnnAssign) {
if assignment.target.is_name_expr() {
self.infer_definition(assignment);
@@ -7483,11 +7611,11 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> {
.to_class_type(self.db())
.is_none_or(|enum_class| !class.is_subclass_of(self.db(), enum_class))
{
// Inference of correctly-placed `TypeVar` and `ParamSpec` definitions is done in
// `TypeInferenceBuilder::infer_legacy_typevar` and
// `TypeInferenceBuilder::infer_paramspec`, and doesn't use the full
// call-binding machinery. If we reach here, it means that someone is trying to
// instantiate a `typing.TypeVar` and `typing.ParamSpec` in an invalid context.
// Inference of correctly-placed `TypeVar`, `ParamSpec`, and `NewType` definitions
// is done in `infer_legacy_typevar`, `infer_paramspec`, and
// `infer_newtype_expression`, and doesn't use the full call-binding machinery. If
// we reach here, it means that someone is trying to instantiate one of these in an
// invalid context.
match class.known(self.db()) {
Some(KnownClass::TypeVar | KnownClass::ExtensionsTypeVar) => {
if let Some(builder) = self
@@ -7509,6 +7637,15 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> {
);
}
}
Some(KnownClass::NewType) => {
if let Some(builder) =
self.context.report_lint(&INVALID_NEWTYPE, call_expression)
{
builder.into_diagnostic(
"A `NewType` definition must be a simple variable assignment",
);
}
}
_ => {}
}
@@ -8577,7 +8714,8 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> {
| Type::BoundSuper(_)
| Type::TypeVar(_)
| Type::TypeIs(_)
| Type::TypedDict(_),
| Type::TypedDict(_)
| Type::NewTypeInstance(_),
) => {
let unary_dunder_method = match op {
ast::UnaryOp::Invert => "__invert__",
@@ -9025,7 +9163,8 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> {
| Type::BoundSuper(_)
| Type::TypeVar(_)
| Type::TypeIs(_)
| Type::TypedDict(_),
| Type::TypedDict(_)
| Type::NewTypeInstance(_),
Type::FunctionLiteral(_)
| Type::BooleanLiteral(_)
| Type::Callable(..)
@@ -9054,7 +9193,8 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> {
| Type::BoundSuper(_)
| Type::TypeVar(_)
| Type::TypeIs(_)
| Type::TypedDict(_),
| Type::TypedDict(_)
| Type::NewTypeInstance(_),
op,
) => Type::try_call_bin_op(self.db(), left_ty, op, right_ty)
.map(|outcome| outcome.return_type(self.db()))

10
crates/ty_python_semantic/src/types/infer/builder/type_expression.rs
View File

@@ -828,6 +828,16 @@ impl<'db> TypeInferenceBuilder<'db, '_> {
self.infer_type_expression(slice);
todo_type!("Generic specialization of typing.Annotated")
}
KnownInstanceType::NewType(newtype) => {
self.infer_type_expression(&subscript.slice);
if let Some(builder) = self.context.report_lint(&INVALID_TYPE_FORM, subscript) {
builder.into_diagnostic(format_args!(
"`{}` is a `NewType` and cannot be specialized",
newtype.name(self.db())
));
}
Type::unknown()
}
},
Type::Dynamic(DynamicType::Todo(_)) => {
self.infer_type_expression(slice);

3
crates/ty_python_semantic/src/types/narrow.rs
View File

@@ -252,7 +252,8 @@ impl ClassInfoConstraintFunction {
| Type::TypeIs(_)
| Type::WrapperDescriptor(_)
| Type::DataclassTransformer(_)
| Type::TypedDict(_) => None,
| Type::TypedDict(_)
| Type::NewTypeInstance(_) => None,
}
}
}

266
crates/ty_python_semantic/src/types/newtype.rs Normal file
View File

@@ -0,0 +1,266 @@
use std::collections::BTreeSet;
use crate::Db;
use crate::semantic_index::definition::{Definition, DefinitionKind};
use crate::types::constraints::ConstraintSet;
use crate::types::{ClassType, Type, definition_expression_type, visitor};
use ruff_db::parsed::parsed_module;
use ruff_python_ast as ast;
/// A `typing.NewType` declaration, either from the perspective of the
/// identity-callable-that-acts-like-a-subtype-in-type-expressions returned by the call to
/// `typing.NewType(...)`, or from the perspective of instances of that subtype returned by the
/// identity callable. For example:
///
/// ```py
/// import typing
/// Foo = typing.NewType("Foo", int)
/// x = Foo(42)
/// ```
///
/// The revealed types there are:
/// - `typing.NewType`: `Type::ClassLiteral(ClassLiteral)` with `KnownClass::NewType`.
/// - `Foo`: `Type::KnownInstance(KnownInstanceType::NewType(NewType { .. }))`
/// - `x`: `Type::NewTypeInstance(NewType { .. })`
///
/// # Ordering
/// Ordering is based on the newtype's salsa-assigned id and not on its values.
/// The id may change between runs, or when the newtype was garbage collected and recreated.
#[salsa::interned(debug, heap_size=ruff_memory_usage::heap_size)]
#[derive(PartialOrd, Ord)]
pub struct NewType<'db> {
/// The name of this NewType (e.g. `"Foo"`)
#[returns(ref)]
pub name: ast::name::Name,
/// The binding where this NewType is first created.
pub definition: Definition<'db>,
// The base type of this NewType, if it's eagerly specified. This is typically `None` when a
// `NewType` is first encountered, because the base type is lazy/deferred to avoid panics in
// the recursive case. This becomes `Some` when a `NewType` is modified by methods like
// `.normalize()`. Callers should use the `base` method instead of accessing this field
// directly.
eager_base: Option<NewTypeBase<'db>>,
}
impl get_size2::GetSize for NewType<'_> {}
#[salsa::tracked]
impl<'db> NewType<'db> {
pub fn base(self, db: &'db dyn Db) -> NewTypeBase<'db> {
match self.eager_base(db) {
Some(base) => base,
None => self.lazy_base(db),
}
}
#[salsa::tracked(
cycle_initial=lazy_base_cycle_initial,
heap_size=ruff_memory_usage::heap_size
)]
fn lazy_base(self, db: &'db dyn Db) -> NewTypeBase<'db> {
// `TypeInferenceBuilder` emits diagnostics for invalid `NewType` definitions that show up
// in assignments, but invalid definitions still get here, and also `NewType` might show up
// in places that aren't definitions at all. Fall back to `object` in all error cases.
let object_fallback = NewTypeBase::ClassType(ClassType::object(db));
let definition = self.definition(db);
let module = parsed_module(db, definition.file(db)).load(db);
let DefinitionKind::Assignment(assignment) = definition.kind(db) else {
return object_fallback;
};
let Some(call_expr) = assignment.value(&module).as_call_expr() else {
return object_fallback;
};
let Some(second_arg) = call_expr.arguments.args.get(1) else {
return object_fallback;
};
match definition_expression_type(db, definition, second_arg) {
Type::NominalInstance(nominal_instance_type) => {
NewTypeBase::ClassType(nominal_instance_type.class(db))
}
Type::NewTypeInstance(newtype) => NewTypeBase::NewType(newtype),
// This branch includes bases that are other typing constructs besides classes and
// other newtypes, for example unions. `NewType("Foo", int | str)` is not allowed.
_ => object_fallback,
}
}
fn iter_bases(self, db: &'db dyn Db) -> NewTypeBaseIter<'db> {
NewTypeBaseIter {
current: Some(self),
seen_before: BTreeSet::new(),
db,
}
}
// Walk the `NewTypeBase` chain to find the underlying `ClassType`. There might not be a
// `ClassType` if this `NewType` is cyclical, and we fall back to `object` in that case.
pub fn base_class_type(self, db: &'db dyn Db) -> ClassType<'db> {
for base in self.iter_bases(db) {
if let NewTypeBase::ClassType(class_type) = base {
return class_type;
}
}
ClassType::object(db)
}
pub(crate) fn is_equivalent_to_impl(self, db: &'db dyn Db, other: Self) -> bool {
// Two instances of the "same" `NewType` won't compare == if one of them has an eagerly
// evaluated base (or a normalized base, etc.) and the other doesn't, so we only check for
// equality of the `definition`.
self.definition(db) == other.definition(db)
}
// Since a regular class can't inherit from a newtype, the only way for one newtype to be a
// subtype of another is to have the other in its chain of newtype bases. Once we reach the
// base class, we don't have to keep looking.
pub(crate) fn has_relation_to_impl(self, db: &'db dyn Db, other: Self) -> ConstraintSet<'db> {
if self.is_equivalent_to_impl(db, other) {
return ConstraintSet::from(true);
}
for base in self.iter_bases(db) {
if let NewTypeBase::NewType(base_newtype) = base {
if base_newtype.is_equivalent_to_impl(db, other) {
return ConstraintSet::from(true);
}
}
}
ConstraintSet::from(false)
}
pub(crate) fn is_disjoint_from_impl(self, db: &'db dyn Db, other: Self) -> ConstraintSet<'db> {
// Two NewTypes are disjoint if they're not equal and neither inherits from the other.
// NewTypes have single inheritance, and a regular class can't inherit from a NewType, so
// it's not possible for some third type to multiply-inherit from both.
let mut self_not_subtype_of_other = self.has_relation_to_impl(db, other).negate(db);
let other_not_subtype_of_self = other.has_relation_to_impl(db, self).negate(db);
self_not_subtype_of_other.intersect(db, other_not_subtype_of_self)
}
/// Create a new `NewType` by mapping the underlying `ClassType`. This descends through any
/// number of nested `NewType` layers and rebuilds the whole chain. In the rare case of cyclic
/// `NewType`s with no underlying `ClassType`, this has no effect and does not call `f`.
pub(crate) fn map_base_class_type(
self,
db: &'db dyn Db,
f: impl FnOnce(ClassType<'db>) -> ClassType<'db>,
) -> Self {
// Modifying the base class type requires unwrapping and re-wrapping however many base
// newtypes there are between here and there. Normally recursion would be natural for this,
// but the bases iterator does cycle detection, and I think using that with a stack is a
// little cleaner than conjuring up yet another `CycleDetector` visitor and yet another
// layer of "*_impl" nesting. Also if there is no base class type, returning `self`
// unmodified seems more correct than injecting some default type like `object` into the
// cycle, which is what `CycleDetector` would do if we used it here.
let mut inner_newtype_stack = Vec::new();
for base in self.iter_bases(db) {
match base {
// Build up the stack of intermediate newtypes that we'll need to re-wrap after
// we've mapped the `ClassType`.
NewTypeBase::NewType(base_newtype) => inner_newtype_stack.push(base_newtype),
// We've reached the `ClassType`.
NewTypeBase::ClassType(base_class_type) => {
// Call `f`.
let mut mapped_base = NewTypeBase::ClassType(f(base_class_type));
// Re-wrap the mapped base class in however many newtypes we unwrapped.
for inner_newtype in inner_newtype_stack.into_iter().rev() {
mapped_base = NewTypeBase::NewType(NewType::new(
db,
inner_newtype.name(db).clone(),
inner_newtype.definition(db),
Some(mapped_base),
));
}
return NewType::new(
db,
self.name(db).clone(),
self.definition(db),
Some(mapped_base),
);
}
}
}
// If we get here, there is no `ClassType` (because this newtype is cyclic), and we don't
// call `f` at all.
self
}
}
pub(crate) fn walk_newtype_instance_type<'db, V: visitor::TypeVisitor<'db> + ?Sized>(
db: &'db dyn Db,
newtype: NewType<'db>,
visitor: &V,
) {
visitor.visit_type(db, newtype.base(db).instance_type(db));
}
/// `typing.NewType` typically wraps a class type, but it can also wrap another newtype.
#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq, get_size2::GetSize, salsa::Update)]
pub enum NewTypeBase<'db> {
ClassType(ClassType<'db>),
NewType(NewType<'db>),
}
impl<'db> NewTypeBase<'db> {
pub fn instance_type(self, db: &'db dyn Db) -> Type<'db> {
match self {
NewTypeBase::ClassType(class_type) => Type::instance(db, class_type),
NewTypeBase::NewType(newtype) => Type::NewTypeInstance(newtype),
}
}
}
/// An iterator over the transitive bases of a `NewType`. In the most common case, e.g.
/// `Foo = NewType("Foo", int)`, this yields the one `NewTypeBase::ClassType` (e.g. `int`). For
/// newtypes that wrap other newtypes, this iterator yields the `NewTypeBase::NewType`s (not
/// including `self`) before finally yielding the `NewTypeBase::ClassType`. In the pathological
/// case of cyclic newtypes like `Foo = NewType("Foo", "Foo")`, this iterator yields the unique
/// `NewTypeBase::NewType`s (not including `self`), detects the cycle, and then stops.
///
/// Note that this does *not* detect indirect cycles that go through a proper class, like this:
/// ```py
/// Foo = NewType("Foo", list["Foo"])
/// ```
/// As far as this iterator is concerned, that's the "common case", and it yields the one
/// `NewTypeBase::ClassType` for `list[Foo]`. Functions like `normalize` that continue recursing
/// over the base class need to pass down a cycle-detecting visitor as usual.
struct NewTypeBaseIter<'db> {
current: Option<NewType<'db>>,
seen_before: BTreeSet<NewType<'db>>,
db: &'db dyn Db,
}
impl<'db> Iterator for NewTypeBaseIter<'db> {
type Item = NewTypeBase<'db>;
fn next(&mut self) -> Option<Self::Item> {
let current = self.current?;
match current.base(self.db) {
NewTypeBase::ClassType(base_class_type) => {
self.current = None;
Some(NewTypeBase::ClassType(base_class_type))
}
NewTypeBase::NewType(base_newtype) => {
// Doing the insertion only in this branch avoids allocating in the common case.
self.seen_before.insert(current);
if self.seen_before.contains(&base_newtype) {
// Cycle detected. Stop iterating.
self.current = None;
None
} else {
self.current = Some(base_newtype);
Some(NewTypeBase::NewType(base_newtype))
}
}
}
}
}
fn lazy_base_cycle_initial<'db>(
db: &'db dyn Db,
_id: salsa::Id,
_self: NewType<'db>,
) -> NewTypeBase<'db> {
NewTypeBase::ClassType(ClassType::object(db))
}

4
crates/ty_python_semantic/src/types/type_ordering.rs
View File

@@ -213,6 +213,10 @@ pub(super) fn union_or_intersection_elements_ordering<'db>(
(Type::TypedDict(_), _) => Ordering::Less,
(_, Type::TypedDict(_)) => Ordering::Greater,
(Type::NewTypeInstance(left), Type::NewTypeInstance(right)) => left.cmp(right),
(Type::NewTypeInstance(_), _) => Ordering::Less,
(_, Type::NewTypeInstance(_)) => Ordering::Greater,
(Type::Union(_), _) | (_, Type::Union(_)) => {
unreachable!("our type representation does not permit nested unions");
}

12
crates/ty_python_semantic/src/types/visitor.rs
View File

@@ -11,6 +11,7 @@ use crate::{
class::walk_generic_alias,
function::{FunctionType, walk_function_type},
instance::{walk_nominal_instance_type, walk_protocol_instance_type},
newtype::{NewType, walk_newtype_instance_type},
subclass_of::walk_subclass_of_type,
walk_bound_method_type, walk_bound_type_var_type, walk_callable_type,
walk_intersection_type, walk_known_instance_type, walk_method_wrapper_type,
@@ -109,6 +110,10 @@ pub(crate) trait TypeVisitor<'db> {
fn visit_typed_dict_type(&self, db: &'db dyn Db, typed_dict: TypedDictType<'db>) {
walk_typed_dict_type(db, typed_dict, self);
}
fn visit_newtype_instance_type(&self, db: &'db dyn Db, newtype: NewType<'db>) {
walk_newtype_instance_type(db, newtype, self);
}
}
/// Enumeration of types that may contain other types, such as unions, intersections, and generics.
@@ -131,6 +136,7 @@ pub(super) enum NonAtomicType<'db> {
ProtocolInstance(ProtocolInstanceType<'db>),
TypedDict(TypedDictType<'db>),
TypeAlias(TypeAliasType<'db>),
NewTypeInstance(NewType<'db>),
}
pub(super) enum TypeKind<'db> {
@@ -198,6 +204,9 @@ impl<'db> From<Type<'db>> for TypeKind<'db> {
TypeKind::NonAtomic(NonAtomicType::TypedDict(typed_dict))
}
Type::TypeAlias(alias) => TypeKind::NonAtomic(NonAtomicType::TypeAlias(alias)),
Type::NewTypeInstance(newtype) => {
TypeKind::NonAtomic(NonAtomicType::NewTypeInstance(newtype))
}
}
}
}
@@ -239,6 +248,9 @@ pub(super) fn walk_non_atomic_type<'db, V: TypeVisitor<'db> + ?Sized>(
NonAtomicType::TypeAlias(alias) => {
visitor.visit_type_alias_type(db, alias);
}
NonAtomicType::NewTypeInstance(newtype) => {
visitor.visit_newtype_instance_type(db, newtype);
}
}
}