Skip to content

Commit e121a78

Browse files
committed
Implemented code that applies type narrowing to local variables and parameters when they are captured by an inner-scoped lambda or function and the variable or parameter is not reassigned after the lambda or function along any code flow paths.
1 parent d303a0d commit e121a78

5 files changed

Lines changed: 253 additions & 4 deletions

File tree

‎packages/pyright-internal/src/analyzer/binder.ts‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -411,6 +411,8 @@ export class Binder extends ParseTreeWalker {
411411
}
412412

413413
override visitFunction(node: FunctionNode): boolean {
414+
AnalyzerNodeInfo.setFlowNode(node, this._currentFlowNode!);
415+
414416
const symbol = this._bindNameToScope(this._currentScope, node.name.value);
415417
const containingClassNode = ParseTreeUtils.getEnclosingClass(node, true);
416418
const functionDeclaration: FunctionDeclaration = {
@@ -539,6 +541,8 @@ export class Binder extends ParseTreeWalker {
539541
}
540542

541543
override visitLambda(node: LambdaNode): boolean {
544+
AnalyzerNodeInfo.setFlowNode(node, this._currentFlowNode!);
545+
542546
// Analyze the parameter defaults in the context of the parent's scope
543547
// before we add any names from the function's scope.
544548
node.parameters.forEach((param) => {

‎packages/pyright-internal/src/analyzer/scopeUtils.ts‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,3 +30,52 @@ export function getScopeForNode(node: ParseNode): Scope | undefined {
3030
const scopeNode = getEvaluationScopeNode(node);
3131
return getScope(scopeNode);
3232
}
33+
34+
// Returns a list of scopes associated with the node and its ancestor nodes.
35+
// If stopScope is provided, the search will stop at that scope.
36+
// Returns undefined if stopScope is not found.
37+
export function getScopeHierarchy(node: ParseNode, stopScope?: Scope): Scope[] | undefined {
38+
const scopeHierarchy: Scope[] = [];
39+
let curNode: ParseNode | undefined = node;
40+
41+
while (curNode) {
42+
const curScope = getScopeForNode(curNode);
43+
44+
if (!curScope) {
45+
return undefined;
46+
}
47+
48+
if (scopeHierarchy.length === 0 || scopeHierarchy[scopeHierarchy.length - 1] !== curScope) {
49+
scopeHierarchy.push(curScope);
50+
}
51+
52+
if (curScope === stopScope) {
53+
return scopeHierarchy;
54+
}
55+
56+
curNode = curNode.parent;
57+
}
58+
59+
return stopScope ? undefined : scopeHierarchy;
60+
}
61+
62+
// Walks up the parse tree from the specified node to find the top-most node
63+
// that is within specified scope.
64+
export function findTopNodeInScope(node: ParseNode, scope: Scope): ParseNode | undefined {
65+
let curNode: ParseNode | undefined = node;
66+
let prevNode: ParseNode | undefined;
67+
let foundScope = false;
68+
69+
while (curNode) {
70+
if (getScope(curNode) === scope) {
71+
foundScope = true;
72+
} else if (foundScope) {
73+
return prevNode;
74+
}
75+
76+
prevNode = curNode;
77+
curNode = curNode.parent;
78+
}
79+
80+
return undefined;
81+
}

‎packages/pyright-internal/src/analyzer/typeEvaluator.ts‎

Lines changed: 102 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3214,6 +3214,22 @@ export function createTypeEvaluator(importLookup: ImportLookup, evaluatorOptions
32143214
setSymbolAccessed(fileInfo, outerScopeSymbol.symbol, node);
32153215
}
32163216
}
3217+
3218+
if (!codeFlowTypeResult.type && symbolWithScope.isBeyondExecutionScope) {
3219+
const outerScopeTypeResult = getCodeFlowTypeForCapturedVariable(
3220+
node,
3221+
symbolWithScope,
3222+
effectiveType
3223+
);
3224+
3225+
if (outerScopeTypeResult?.type) {
3226+
type = outerScopeTypeResult.type;
3227+
}
3228+
3229+
if (outerScopeTypeResult?.isIncomplete) {
3230+
isIncomplete = true;
3231+
}
3232+
}
32173233
}
32183234

32193235
// Detect, report, and fill in missing type arguments if appropriate.
@@ -3325,6 +3341,84 @@ export function createTypeEvaluator(importLookup: ImportLookup, evaluatorOptions
33253341
return { type, node, isIncomplete };
33263342
}
33273343

3344+
// Handles the case where a variable or parameter is defined in an outer
3345+
// scope and captured by an inner scope (either a function or a lambda).
3346+
function getCodeFlowTypeForCapturedVariable(
3347+
node: NameNode,
3348+
symbolWithScope: SymbolWithScope,
3349+
effectiveType: Type
3350+
): FlowNodeTypeResult | undefined {
3351+
// This function applies only to variables and parameters, not to other
3352+
// types of symbols.
3353+
if (
3354+
!symbolWithScope.symbol
3355+
.getDeclarations()
3356+
.every((decl) => decl.type === DeclarationType.Variable || decl.type === DeclarationType.Parameter)
3357+
) {
3358+
return undefined;
3359+
}
3360+
3361+
// If the symbol is a variable captured by an inner function
3362+
// or lambda, see if we can infer the type from the outer scope.
3363+
const scopeHierarchy = ScopeUtils.getScopeHierarchy(node, symbolWithScope.scope);
3364+
3365+
// Handle the case where all of the nested scopes are functions,
3366+
// lambdas and modules. Don't allow other types of scopes.
3367+
if (
3368+
scopeHierarchy &&
3369+
scopeHierarchy.length >= 2 &&
3370+
scopeHierarchy.every((s) => s.type === ScopeType.Function || s.type === ScopeType.Module)
3371+
) {
3372+
// Find the parse node associated with the scope that is just inside of the
3373+
// scope that declares the captured variable.
3374+
const innerScopeNode = ScopeUtils.findTopNodeInScope(node, scopeHierarchy[scopeHierarchy.length - 2]);
3375+
if (
3376+
innerScopeNode &&
3377+
(innerScopeNode.nodeType === ParseNodeType.Function || innerScopeNode.nodeType === ParseNodeType.Lambda)
3378+
) {
3379+
const innerScopeCodeFlowNode = AnalyzerNodeInfo.getFlowNode(innerScopeNode);
3380+
if (innerScopeCodeFlowNode) {
3381+
// See if any of the assignments of the symbol are reachable
3382+
// from this node. If so, we cannot apply any narrowing because
3383+
// the type could change after the capture.
3384+
if (
3385+
symbolWithScope.symbol.getDeclarations().every((decl) => {
3386+
// Parameter declarations always start life at the beginning
3387+
// of the execution scope, so they are always safe to narrow.
3388+
if (decl.type === DeclarationType.Parameter) {
3389+
return true;
3390+
}
3391+
3392+
const declCodeFlowNode = AnalyzerNodeInfo.getFlowNode(decl.node);
3393+
if (!declCodeFlowNode) {
3394+
return false;
3395+
}
3396+
3397+
// Functions and lambdas do not create a new flow node, so it's
3398+
// possible that they share the flow node of the declaration. In this
3399+
// case, the declaration must come before, so it's safe.
3400+
if (declCodeFlowNode === innerScopeCodeFlowNode) {
3401+
return true;
3402+
}
3403+
3404+
return !isFlowNodeReachable(declCodeFlowNode, innerScopeCodeFlowNode);
3405+
})
3406+
) {
3407+
return getFlowTypeOfReference(
3408+
node,
3409+
symbolWithScope.symbol.id,
3410+
effectiveType,
3411+
/* isInitialTypeIncomplete */ false,
3412+
innerScopeNode
3413+
);
3414+
}
3415+
}
3416+
}
3417+
}
3418+
3419+
return undefined;
3420+
}
3421+
33283422
// Validates that a TypeVar is valid in this context. If so, it clones it
33293423
// and provides a scope ID defined by its containing scope (class, function
33303424
// or type alias). If not, it emits errors indicating why the TypeVar
@@ -14756,16 +14850,20 @@ export function createTypeEvaluator(importLookup: ImportLookup, evaluatorOptions
1475614850

1475714851
// Attempts to determine the type of the reference expression at the
1475814852
// point in the code. If the code flow analysis has nothing to say
14759-
// about that expression, it return undefined.
14853+
// about that expression, it return undefined. Normally flow analysis
14854+
// starts from the reference node, but startNode can be specified to
14855+
// override this in a few special cases (functions and lambdas) to
14856+
// support analysis of captured variables.
1476014857
function getFlowTypeOfReference(
1476114858
reference: CodeFlowReferenceExpressionNode,
1476214859
targetSymbolId: number,
1476314860
initialType: Type | undefined,
14764-
isInitialTypeIncomplete: boolean
14861+
isInitialTypeIncomplete: boolean,
14862+
startNode?: FunctionNode | LambdaNode
1476514863
): FlowNodeTypeResult {
1476614864
// See if this execution scope requires code flow for this reference expression.
1476714865
const referenceKey = createKeyForReference(reference);
14768-
const executionNode = ParseTreeUtils.getExecutionScopeNode(reference);
14866+
const executionNode = ParseTreeUtils.getExecutionScopeNode(startNode?.parent ?? reference);
1476914867
const codeFlowExpressions = AnalyzerNodeInfo.getCodeFlowExpressions(executionNode);
1477014868

1477114869
if (!codeFlowExpressions || !codeFlowExpressions.has(referenceKey)) {
@@ -14785,7 +14883,7 @@ export function createTypeEvaluator(importLookup: ImportLookup, evaluatorOptions
1478514883
analyzer = getCodeFlowAnalyzerForNode(executionNode.id);
1478614884
}
1478714885

14788-
const flowNode = AnalyzerNodeInfo.getFlowNode(reference);
14886+
const flowNode = AnalyzerNodeInfo.getFlowNode(startNode ?? reference);
1478914887
if (flowNode === undefined) {
1479014888
return { type: undefined, usedOuterScopeAlias: false, isIncomplete: false };
1479114889
}
Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
# This sample tests the code flow analysis used to determine
2+
# whether it is safe to narrow the type of a captured variable.
3+
4+
from typing import Optional
5+
6+
7+
def get_optional_int() -> Optional[int]:
8+
...
9+
10+
11+
v1 = get_optional_int()
12+
if v1 is not None:
13+
lambda: v1 + 5
14+
15+
v2 = get_optional_int()
16+
if v2 is not None:
17+
# This should generate an error because v2
18+
# is reassigned after capture.
19+
lambda: v2 + 5
20+
v2 = None
21+
22+
v3 = get_optional_int()
23+
if v3 is not None:
24+
lambda: v3 + 5
25+
else:
26+
v3 = None
27+
28+
# This should generate an error because v4 is
29+
# not bound prior to the capture.
30+
lambda: v4 + 5
31+
v4 = get_optional_int()
32+
33+
34+
def func1(v1: Optional[int]):
35+
if v1 is not None:
36+
lambda: v1 + 5
37+
38+
39+
def func2(v1: Optional[int]):
40+
if v1 is not None:
41+
42+
def func2_inner1():
43+
v1 + 5
44+
45+
def func2_inner2():
46+
lambda: v1 + 5
47+
48+
func2_inner2()
49+
50+
func2_inner1()
51+
52+
53+
def func3():
54+
v1: Optional[int] = 3
55+
lambda: v1 + 5
56+
57+
58+
def func4():
59+
v1: Optional[int] = 3
60+
# This should generate an error because v1
61+
# is reassigned after capture.
62+
lambda: v1 + 5
63+
v1 = None
64+
65+
66+
def func5():
67+
v1: Optional[int] = 3
68+
69+
while True:
70+
lambda: v1 + 5
71+
72+
73+
def func6():
74+
v1: Optional[int] = 3
75+
76+
while True:
77+
if v1 is not None:
78+
# This should generate an error because
79+
# v1 is reassigned on a code path that is
80+
# reachable from the lambda.
81+
lambda: v1 + 5
82+
else:
83+
v1 = None
84+
85+
86+
def func7():
87+
while True:
88+
# This should generate an error because v1 is
89+
# potentially unbound prior to capture.
90+
lambda: v1 + 5
91+
92+
v1: Optional[int] = 3

‎packages/pyright-internal/src/tests/typeEvaluator1.test.ts‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -710,6 +710,12 @@ test('CodeFlow5', () => {
710710
TestUtils.validateResults(analysisResults, 0);
711711
});
712712

713+
test('CapturedVariable1', () => {
714+
const analysisResults = TestUtils.typeAnalyzeSampleFiles(['capturedVariable1.py']);
715+
716+
TestUtils.validateResults(analysisResults, 5);
717+
});
718+
713719
test('Properties1', () => {
714720
const analysisResults = TestUtils.typeAnalyzeSampleFiles(['properties1.py']);
715721

0 commit comments

Comments
 (0)