Skip to content

Commit 945299a

Browse files
east825intellij-monorepo-bot
east825
authored and
intellij-monorepo-bot
committed
PY-64359 Handle Any as a generic type argument in PyTypeUtil.convertToType()
Turns out there are still problems with handling explicit Any as a type argument. Namely, after matching `Iterable[T@Iterable]` with `Iterable[Any]`, we don't retain the mapping `T@Iterable -> Any` in PyTypeChecker.MatchContext#mySubstitutions. It doesn't hit during handling of generic calls because in cases like the following ```python from typing import Any def first[T](xs: list[T]) -> T | None: ... xs: list[Any] expr = first(xs) ``` we forcibly replace unbound `T@first` in the result type with Any in PyTypeChecker.getSubstitutionsWithUnresolvedReturnGenerics, hence masking the underlying issue. I temporarily extended this approach to `convertToType`. Specializing all type parameters through CSP should help with that. (cherry picked from commit 111c6c0d0a15fd290de6df7388e9f6ada70cf998) GitOrigin-RevId: 0777fdac0034d04dfec517f970aa91fa7d6eba6a
1 parent 9c75262 commit 945299a

3 files changed

Lines changed: 59 additions & 43 deletions

File tree

‎python/python-psi-impl/src/com/jetbrains/python/psi/types/PyTypeChecker.kt‎

Lines changed: 52 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -10,14 +10,14 @@ import com.intellij.psi.PsiNamedElement
1010
import com.intellij.util.ArrayUtil
1111
import com.intellij.util.containers.ContainerUtil
1212
import com.jetbrains.python.PyNames
13+
import com.jetbrains.python.PyNames.isPrivate
14+
import com.jetbrains.python.PyNames.isProtected
1315
import com.jetbrains.python.PythonRuntimeService
1416
import com.jetbrains.python.ast.PyAstFunction
1517
import com.jetbrains.python.codeInsight.dataflow.scope.ScopeUtil
1618
import com.jetbrains.python.codeInsight.typing.PyTypingTypeProvider
1719
import com.jetbrains.python.codeInsight.typing.inspectProtocolSubclass
1820
import com.jetbrains.python.codeInsight.typing.isProtocol
19-
import com.jetbrains.python.PyNames.isPrivate
20-
import com.jetbrains.python.PyNames.isProtected
2121
import com.jetbrains.python.psi.AccessDirection
2222
import com.jetbrains.python.psi.LanguageLevel
2323
import com.jetbrains.python.psi.PyClass
@@ -1208,51 +1208,62 @@ object PyTypeChecker {
12081208
substitutions: GenericSubstitutions?,
12091209
context: TypeEvalContext,
12101210
): GenericSubstitutions {
1211-
val substitutions = substitutions ?: GenericSubstitutions()
1212-
val typeParamsFromReturnType = returnType.collectGenerics(context)
1213-
// TODO Handle unmatched TypeVarTuples here as well
1214-
if (typeParamsFromReturnType.typeVars.isEmpty() && typeParamsFromReturnType.paramSpecs.isEmpty()) {
1215-
return substitutions
1216-
}
1217-
val typeParamsFromParameterTypes = GenericsImpl()
1218-
for (parameter in parameters) {
1219-
collectGenerics(parameter.getArgumentType(context), context, typeParamsFromParameterTypes)
1211+
val parameterTypes = parameters.map { it.getArgumentType(context) }
1212+
return substituteUnboundTypeVarsWithDefaultOrAny(returnType, parameterTypes, substitutions, context)
1213+
}
1214+
1215+
private fun substituteUnboundTypeVarsWithDefaultOrAny(
1216+
targetType: PyType?,
1217+
typeParameterSources: List<PyType?>,
1218+
substitutions: GenericSubstitutions?,
1219+
context: TypeEvalContext,
1220+
): GenericSubstitutions {
1221+
val resolvableTypeParams = GenericsImpl()
1222+
for (parameterType in typeParameterSources) {
1223+
collectGenerics(parameterType, context, resolvableTypeParams)
12201224
}
12211225

1222-
for (returnTypeParam in typeParamsFromReturnType.typeVars) {
1223-
val canGetBoundFromArguments = returnTypeParam in typeParamsFromParameterTypes.typeVars ||
1224-
returnTypeParam.invert() in typeParamsFromParameterTypes.typeVars
1225-
val isAlreadyBound = returnTypeParam in substitutions.typeVars ||
1226-
returnTypeParam.invert() in substitutions.typeVars
1227-
if (canGetBoundFromArguments && !isAlreadyBound) {
1228-
@Suppress("UNCHECKED_CAST")
1229-
substitutions.putTypeVar(returnTypeParam, returnTypeParam.defaultType as Ref<PyType?>?, KeyImpl)
1226+
val existingSubstitutions = substitutions ?: GenericSubstitutions()
1227+
val requiredTypeParams = targetType.collectGenerics(context)
1228+
// TODO Handle unmatched TypeVarTuples here as well
1229+
if (requiredTypeParams.typeVars.isEmpty() && requiredTypeParams.paramSpecs.isEmpty()) {
1230+
return existingSubstitutions
1231+
}
1232+
1233+
for (typeVar in requiredTypeParams.typeVars) {
1234+
val isResolvable = resolvableTypeParams.typeVars.contains(typeVar) ||
1235+
resolvableTypeParams.typeVars.contains(typeVar.invert())
1236+
val isAlreadyBound = existingSubstitutions.typeVars.containsKey(typeVar) ||
1237+
existingSubstitutions.typeVars.containsKey(typeVar.invert())
1238+
if (isResolvable && !isAlreadyBound) {
1239+
if (typeVar.defaultType != null) {
1240+
@Suppress("UNCHECKED_CAST")
1241+
existingSubstitutions.putTypeVar(typeVar, typeVar.defaultType as Ref<PyType?>?, KeyImpl)
1242+
}
1243+
else {
1244+
existingSubstitutions.putTypeVar(typeVar, Ref.create<PyType?>(null), KeyImpl)
1245+
}
12301246
}
12311247
}
1232-
for (paramSpecType in typeParamsFromReturnType.paramSpecs) {
1233-
val canGetBoundFromArguments = paramSpecType in typeParamsFromParameterTypes.paramSpecs
1234-
val isAlreadyBound = paramSpecType in substitutions.paramSpecs
1235-
if (canGetBoundFromArguments && !isAlreadyBound) {
1248+
for (paramSpecType in requiredTypeParams.paramSpecs) {
1249+
val isResolvable = resolvableTypeParams.paramSpecs.contains(paramSpecType)
1250+
val isAlreadyBound = existingSubstitutions.paramSpecs.containsKey(paramSpecType)
1251+
if (isResolvable && !isAlreadyBound) {
12361252
if (paramSpecType.defaultType != null) {
1237-
substitutions.putParamSpec(paramSpecType, Ref.deref(paramSpecType.defaultType), KeyImpl)
1253+
existingSubstitutions.putParamSpec(paramSpecType, Ref.deref<PyCallableParameterVariadicType?>(paramSpecType.defaultType), KeyImpl)
12381254
}
12391255
else {
1240-
substitutions.putParamSpec(
1241-
paramSpecType,
1242-
PyCallableParameterListTypeImpl(
1243-
listOf(
1244-
PyCallableParameterImpl.positionalNonPsi("args", null),
1245-
PyCallableParameterImpl.keywordNonPsi("kwargs", null),
1246-
)
1247-
),
1248-
KeyImpl,
1256+
existingSubstitutions.putParamSpec(paramSpecType, PyCallableParameterListTypeImpl(
1257+
listOf(PyCallableParameterImpl.positionalNonPsi("args", null),
1258+
PyCallableParameterImpl.keywordNonPsi("kwargs", null))), KeyImpl
12491259
)
12501260
}
12511261
}
12521262
}
1253-
return substitutions
1263+
return existingSubstitutions
12541264
}
12551265

1266+
12561267
private fun <T : PyInstantiableType<T>> PyInstantiableType<T>.invert(): T {
12571268
return if (isDefinition) toInstance() else toClass()
12581269
}
@@ -1947,8 +1958,13 @@ object PyTypeChecker {
19471958
fun convertToType(type: PyType?, superType: PyClassType, context: TypeEvalContext): PyType? {
19481959
val matchContext = MatchContext(context, GenericSubstitutions(), false)
19491960
val matched = match(superType, type, matchContext)
1950-
if (matched.orElse(false)!!) {
1951-
return substitute(superType, matchContext.mySubstitutions, context)
1961+
if (matched.orElse(false)) {
1962+
// There is a tricky problem with handling type parameter binds to Any. Namely, during matching list[Any] to Iterable[T@Iterable],
1963+
// we don't keep the bind T@Iterable -> Any (see the implementation of `match(PyTypeVarType, PyType, MatchContext)`).
1964+
// As a workaround, until we migrate to type checking with CSP, we consider that
1965+
// all parameter of the super types should be bound, and if they aren't, we fall back them to Any.
1966+
val substitutions = substituteUnboundTypeVarsWithDefaultOrAny(superType, listOf(superType), matchContext.mySubstitutions, context)
1967+
return substitute(superType, substitutions, context)
19521968
}
19531969
return null
19541970
}

‎python/testSrc/com/jetbrains/python/Py3TypeTest.java‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -477,7 +477,7 @@ yield from get_tuple()
477477
}
478478

479479
public void testYieldFromUnknownTuple() {
480-
doTest("_T_co",
480+
doTest("Any",
481481
"""
482482
def get_tuple() -> tuple:
483483
pass
@@ -487,7 +487,7 @@ yield from get_tuple()
487487
}
488488

489489
public void testYieldFromUnknownList() {
490-
doTest("_T_co",
490+
doTest("Any",
491491
"""
492492
def get_list() -> list:
493493
pass
@@ -497,7 +497,7 @@ yield from get_list()
497497
}
498498

499499
public void testYieldFromUnknownDict() {
500-
doTest("_T_co",
500+
doTest("Any",
501501
"""
502502
def get_dict() -> dict:
503503
pass
@@ -507,7 +507,7 @@ yield from get_dict()
507507
}
508508

509509
public void testYieldFromUnknownSet() {
510-
doTest("_T_co",
510+
doTest("Any",
511511
"""
512512
def get_set() -> set:
513513
pass

‎python/testSrc/com/jetbrains/python/PyTypeTest.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1693,7 +1693,7 @@ def test():
16931693

16941694
// PY-20794
16951695
public void testIterateOverPureList() {
1696-
doTest("_T_co",
1696+
doTest("Any",
16971697
"""
16981698
l = None # type: list
16991699
for expr in l:
@@ -1703,7 +1703,7 @@ public void testIterateOverPureList() {
17031703

17041704
// PY-20794
17051705
public void testIterateOverDictValueWithDefaultValue() {
1706-
doTest("Union[_T_co, Any]",
1706+
doTest("Any",
17071707
"""
17081708
d = None # type: dict
17091709
for expr in d.get('field', []):
@@ -2300,7 +2300,7 @@ public void testSubscriptionOnWeakType() {
23002300

23012301
// PY-24364
23022302
public void testReassignedParameter() {
2303-
doTest("(entries: Any) -> Generator[_T_co, Any, None]",
2303+
doTest("(entries: Any) -> Generator[Any, Any, None]",
23042304
"""
23052305
def resort(entries):
23062306
entries = list(entries)

0 commit comments

Comments
 (0)