Skip to content

Commit 76adffb

Browse files
authored
Remove locks from Construct/Call in python servers (#24081)
Allow python provider server Call and Construct methods to run concurrently.
1 parent 3b6e0db commit 76adffb

5 files changed

Lines changed: 157 additions & 25 deletions

File tree

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
component: sdk/python
2+
kind: fix
3+
body: Allow Construct and Call methods to run concurrently in provider servers
4+
time: 2026-07-27T13:47:17.383419+01:00

‎sdk/python/lib/pulumi/provider/experimental/server.py‎

Lines changed: 0 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -85,7 +85,6 @@ class ProviderServicer(ResourceProviderServicer):
8585
_args: list[str]
8686
_provider: provider.Provider
8787
_engine_address: str
88-
_lock: asyncio.Lock
8988

9089
def __init__(
9190
self,
@@ -99,7 +98,6 @@ def __init__(
9998
self._version = version
10099
self._provider = provider
101100
self._engine_address = engine_address
102-
self._lock = asyncio.Lock()
103101

104102
async def GetPluginInfo(self, request, context) -> proto.PluginInfo:
105103
return proto.PluginInfo(version=self._version)
@@ -142,9 +140,6 @@ def create_grpc_invalid_properties_status(
142140
async def Construct(
143141
self, request: proto.ConstructRequest, context
144142
) -> proto.ConstructResponse:
145-
# Calls to `Construct` and `Call` are serialized because they currently modify globals. When we are able to
146-
# avoid modifying globals, we can remove the locking.
147-
await self._lock.acquire()
148143
try:
149144
return await self._construct(request, context)
150145
except Exception as e: # noqa
@@ -169,8 +164,6 @@ async def Construct(
169164
stack = traceback.extract_tb(e.__traceback__)[:]
170165
pretty_stack = "".join(traceback.format_list(stack))
171166
raise Exception(f"{str(e)}:\n{pretty_stack}")
172-
finally:
173-
self._lock.release()
174167

175168
async def _construct(
176169
self, request: proto.ConstructRequest, context
@@ -290,9 +283,6 @@ def _construct_response(
290283
)
291284

292285
async def Call(self, request: proto.CallRequest, context):
293-
# Calls to `Construct` and `Call` are serialized because they currently modify globals. When we are able to
294-
# avoid modifying globals, we can remove the locking.
295-
await self._lock.acquire()
296286
try:
297287
return await self._call(request, context)
298288
except InputPropertiesError as e:
@@ -307,8 +297,6 @@ async def Call(self, request: proto.CallRequest, context):
307297
await context.abort_with_status(status)
308298
# We already aborted at this point
309299
raise
310-
finally:
311-
self._lock.release()
312300

313301
async def _call(self, request: proto.CallRequest, context):
314302
assert isinstance(request, proto.CallRequest), (

‎sdk/python/lib/pulumi/provider/server.py‎

Lines changed: 0 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -85,7 +85,6 @@ class ProviderServicer(ResourceProviderServicer):
8585
engine_address: str
8686
provider: Provider
8787
args: list[str]
88-
lock: asyncio.Lock
8988

9089
def create_grpc_invalid_properties_status(
9190
self, message: str, errors: Optional[list[InputPropertyErrorDetails]]
@@ -121,9 +120,6 @@ def create_grpc_invalid_properties_status(
121120
async def Construct(
122121
self, request: proto.ConstructRequest, context
123122
) -> proto.ConstructResponse:
124-
# Calls to `Construct` and `Call` are serialized because they currently modify globals. When we are able to
125-
# avoid modifying globals, we can remove the locking.
126-
await self.lock.acquire()
127123
try:
128124
return await self._construct(request, context)
129125
except Exception as e: # noqa
@@ -150,8 +146,6 @@ async def Construct(
150146
stack = traceback.extract_tb(e.__traceback__)[:]
151147
pretty_stack = "".join(traceback.format_list(stack))
152148
raise Exception(f"{str(e)}:\n{pretty_stack}")
153-
finally:
154-
self.lock.release()
155149

156150
async def _construct(
157151
self, request: proto.ConstructRequest, context
@@ -349,9 +343,6 @@ async def _construct_response(
349343
return proto.ConstructResponse(urn=urn, state=state, stateDependencies=deps)
350344

351345
async def Call(self, request: proto.CallRequest, context):
352-
# Calls to `Construct` and `Call` are serialized because they currently modify globals. When we are able to
353-
# avoid modifying globals, we can remove the locking.
354-
await self.lock.acquire()
355346
try:
356347
return await self._call(request, context)
357348
except InputPropertiesError as e:
@@ -366,8 +357,6 @@ async def Call(self, request: proto.CallRequest, context):
366357
await context.abort_with_status(status)
367358
# We already aborted at this point
368359
raise
369-
finally:
370-
self.lock.release()
371360

372361
async def _call(self, request: proto.CallRequest, context):
373362
assert isinstance(request, proto.CallRequest), (
@@ -510,7 +499,6 @@ def __init__(
510499
self.provider = provider
511500
self.args = args
512501
self.engine_address = engine_address
513-
self.lock = asyncio.Lock()
514502

515503

516504
def main(provider: Provider, args: list[str]) -> None: # args not in use?
Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
# Copyright 2026, Pulumi Corporation.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
import asyncio
16+
from typing import Optional
17+
18+
import pytest
19+
20+
from pulumi.provider.experimental import provider
21+
from pulumi.provider.experimental.property_value import PropertyValue
22+
from pulumi.provider.experimental.server import ProviderServicer
23+
from pulumi.runtime import config, proto, settings
24+
from test.grpc_stubs import provider_servicer_stub
25+
26+
27+
class ConcurrentConstructCallProvider(provider.Provider):
28+
def __init__(self):
29+
super().__init__()
30+
self.construct_waiting = asyncio.Event()
31+
self.call_ran = asyncio.Event()
32+
self.construct_stack_after_call: Optional[str] = None
33+
self.construct_config_after_call: Optional[str] = None
34+
35+
async def construct(
36+
self, request: provider.ConstructRequest
37+
) -> provider.ConstructResponse:
38+
assert settings.get_stack() == "construct-stack"
39+
assert config.get_config("test:key") == "construct-config"
40+
41+
self.construct_waiting.set()
42+
await self.call_ran.wait()
43+
self.construct_stack_after_call = settings.get_stack()
44+
self.construct_config_after_call = config.get_config("test:key")
45+
46+
return provider.ConstructResponse(
47+
urn=f"urn:pulumi:{request.name}::{request.resource_type}::test-resource",
48+
state={"result": PropertyValue("construct")},
49+
state_dependencies={},
50+
)
51+
52+
async def call(self, request: provider.CallRequest) -> provider.CallResponse:
53+
assert settings.get_stack() == "call-stack"
54+
assert config.get_config("test:key") == "call-config"
55+
self.call_ran.set()
56+
return provider.CallResponse({"result": PropertyValue("call")})
57+
58+
59+
@pytest.mark.asyncio
60+
async def test_construct_and_call_can_run_concurrently():
61+
test_provider = ConcurrentConstructCallProvider()
62+
servicer = ProviderServicer([], "1.0.0", test_provider, "")
63+
64+
async with provider_servicer_stub(servicer) as stub:
65+
construct_request = proto.ConstructRequest(
66+
name="construct",
67+
type="test:index:Component",
68+
stack="construct-stack",
69+
config={"test:key": "construct-config"},
70+
)
71+
construct_task = asyncio.ensure_future(stub.Construct(construct_request))
72+
73+
await asyncio.wait_for(test_provider.construct_waiting.wait(), timeout=2)
74+
75+
call_request = proto.CallRequest(
76+
tok="test:index:call",
77+
stack="call-stack",
78+
config={"test:key": "call-config"},
79+
)
80+
call_response = await asyncio.wait_for(stub.Call(call_request), timeout=2)
81+
construct_response = await asyncio.wait_for(construct_task, timeout=2)
82+
83+
assert getattr(call_response, "return")["result"] == "call"
84+
assert construct_response.urn.endswith("::test-resource")
85+
assert test_provider.construct_stack_after_call == "construct-stack"
86+
assert test_provider.construct_config_after_call == "construct-config"

‎sdk/python/lib/test/provider/test_server.py‎

Lines changed: 67 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
# See the License for the specific language governing permissions and
1313
# limitations under the License.
1414

15+
import asyncio
1516
import functools
1617
from typing import Any, Awaitable, Callable, Dict, List, Optional, Set, Tuple
1718

@@ -25,7 +26,7 @@
2526
)
2627
from pulumi.provider.server import ComponentInitError
2728
import pulumi.output
28-
from pulumi.provider import ConstructResult
29+
from pulumi.provider import CallResult, ConstructResult
2930
from pulumi.provider.provider import Provider
3031
import pytest
3132

@@ -36,6 +37,7 @@
3637
from pulumi.runtime.proto.provider_pb2 import ConstructRequest
3738
from pulumi.runtime.proto import status_pb2, errors_pb2
3839
from pulumi.runtime.settings import Settings, configure
40+
from pulumi.runtime import config, settings
3941
from semver import VersionInfo as Version
4042
from ..grpc_stubs import provider_servicer_stub
4143

@@ -661,6 +663,40 @@ def construct(
661663
)
662664

663665

666+
class ConcurrentConstructCallProvider(Provider):
667+
def __init__(self):
668+
super().__init__("1.0.0")
669+
self.construct_waiting = asyncio.Event()
670+
self.call_ran = asyncio.Event()
671+
self.construct_stack_after_call: Optional[str] = None
672+
self.construct_config_after_call: Optional[str] = None
673+
674+
def construct(
675+
self,
676+
name: str,
677+
resource_type: str,
678+
inputs: Inputs,
679+
options: Optional[ResourceOptions] = None,
680+
) -> ConstructResult:
681+
assert settings.get_stack() == "construct-stack"
682+
assert config.get_config("test:key") == "construct-config"
683+
684+
async def urn() -> str:
685+
self.construct_waiting.set()
686+
await self.call_ran.wait()
687+
self.construct_stack_after_call = settings.get_stack()
688+
self.construct_config_after_call = config.get_config("test:key")
689+
return f"urn:pulumi:{name}::{resource_type}::test-resource"
690+
691+
return ConstructResult(urn=urn(), state={"result": "construct"})
692+
693+
def call(self, token: str, args: Inputs) -> CallResult:
694+
assert settings.get_stack() == "call-stack"
695+
assert config.get_config("test:key") == "call-config"
696+
self.call_ran.set()
697+
return CallResult({"result": "call"})
698+
699+
664700
@pytest.mark.asyncio
665701
async def test_construct_success():
666702
provider = MockProvider()
@@ -674,6 +710,36 @@ async def test_construct_success():
674710
assert "test-resource" in response.urn
675711

676712

713+
@pytest.mark.asyncio
714+
async def test_construct_and_call_can_run_concurrently():
715+
provider = ConcurrentConstructCallProvider()
716+
servicer = ProviderServicer(provider, [], "")
717+
718+
async with provider_servicer_stub(servicer) as stub:
719+
construct_request = proto.ConstructRequest(
720+
name="construct",
721+
type="test:index:Component",
722+
stack="construct-stack",
723+
config={"test:key": "construct-config"},
724+
)
725+
construct_task = asyncio.ensure_future(stub.Construct(construct_request))
726+
727+
await asyncio.wait_for(provider.construct_waiting.wait(), timeout=2)
728+
729+
call_request = proto.CallRequest(
730+
tok="test:index:call",
731+
stack="call-stack",
732+
config={"test:key": "call-config"},
733+
)
734+
call_response = await asyncio.wait_for(stub.Call(call_request), timeout=2)
735+
construct_response = await asyncio.wait_for(construct_task, timeout=2)
736+
737+
assert getattr(call_response, "return")["result"] == "call"
738+
assert construct_response.urn.endswith("::test-resource")
739+
assert provider.construct_stack_after_call == "construct-stack"
740+
assert provider.construct_config_after_call == "construct-config"
741+
742+
677743
@pytest.mark.asyncio
678744
async def test_construct_run_error():
679745
provider = MockProvider(error_type="run_error", error_message="monitor shut down")

0 commit comments

Comments
 (0)