Skip to content

Commit 10b00bc

Browse files
authored
fix parallel pulumi install resulting in provider missing (#24080)
In #23330 we started using `InstallPluginSet` instead of `EnsurePluginsAreInstalled` for installing missing plugins. These two functions have a small but significant semantic difference. The newer function always sets "reinstall" to true, while the old one set it to false. This means we now re-install providers even when they are already successfully installed. This is both unnecessary, and triggers an issue when multiple `pulumi install` processes are running in parallel. Namely an install process can delete an already installed provider, which a different process expects to exist already. This causes the issues seen in #24059, but can also affect users if they are running `pulumi install` in parallel. There's no need to always re-install the plugin in this case. We already have a marker for partial downloads that the download process respects, so we can just rely on that. Fixes #24059
1 parent fcd2794 commit 10b00bc

4 files changed

Lines changed: 127 additions & 3 deletions

File tree

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
component: cli
2+
kind: fix
3+
body: Fix parallel `pulumi install` processes sharing a `PULUMI_HOME` intermittently
4+
failing with a missing provider executable error
5+
time: 2026-07-27T00:00:00.000000+00:00

‎pkg/cmd/pulumi/packageworkspace/packageworkspace.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -203,7 +203,7 @@ func (w Workspace) DownloadPlugin(
203203
"Unpacking provider "+pluginSpec.Name, diagutils.GetGlobalColorization())
204204
}
205205
cleanup, err := pluginstorage.UnpackContents(
206-
ctx, pluginSpec, pluginstorage.TarPlugin(unpackStream), true, /* reinstall */
206+
ctx, pluginSpec, pluginstorage.TarPlugin(unpackStream), false, /* reinstall */
207207
)
208208
if err != nil {
209209
return "", nil, err

‎pkg/pluginstorage/download.go‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -104,8 +104,12 @@ func UnpackContents(
104104
}
105105

106106
// Either the partial file exists--meaning a previous attempt at installing the plugin failed--or we're
107-
// deliberately reinstalling the plugin. Delete finalDir so we can try installing again. There's no need to
108-
// delete the partial file since we'd just be recreating it again below anyway.
107+
// deliberately reinstalling the plugin. Delete finalDir so we can try installing again. Create the partial
108+
// file before deleting so concurrent processes never observe the directory mid-deletion as a completed
109+
// install.
110+
if err := os.WriteFile(partialFilePath, nil, 0o600); err != nil {
111+
return nil, err
112+
}
109113
if err := os.RemoveAll(finalDir); err != nil {
110114
return nil, err
111115
}

‎pkg/pluginstorage/download_test.go‎

Lines changed: 115 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,115 @@
1+
// Copyright 2026, Pulumi Corporation.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
package pluginstorage
16+
17+
import (
18+
"os"
19+
"path/filepath"
20+
"testing"
21+
22+
"github.com/blang/semver"
23+
"github.com/stretchr/testify/assert"
24+
"github.com/stretchr/testify/require"
25+
26+
"github.com/pulumi/pulumi/sdk/v3/go/common/apitype"
27+
"github.com/pulumi/pulumi/sdk/v3/go/common/workspace"
28+
)
29+
30+
func testPluginSpec(t *testing.T) workspace.PluginDescriptor {
31+
t.Setenv("PULUMI_HOME", t.TempDir())
32+
version := semver.MustParse("1.0.0")
33+
return workspace.PluginDescriptor{
34+
Name: "test",
35+
Kind: apitype.ResourcePlugin,
36+
Version: &version,
37+
}
38+
}
39+
40+
func testPluginContent(t *testing.T, contents string) Content {
41+
src := t.TempDir()
42+
// The plugin binary must be executable.
43+
require.NoError(t, os.WriteFile(filepath.Join(src, "pulumi-resource-test"), []byte(contents), 0o700)) //nolint:gosec
44+
return DirPlugin(src)
45+
}
46+
47+
func binaryContents(t *testing.T, spec workspace.PluginDescriptor) string {
48+
dir, err := spec.DirPath()
49+
require.NoError(t, err)
50+
contents, err := os.ReadFile(filepath.Join(dir, "pulumi-resource-test"))
51+
require.NoError(t, err)
52+
return string(contents)
53+
}
54+
55+
//nolint:paralleltest // t.Setenv is incompatible with t.Parallel
56+
func TestUnpackContentsFreshInstall(t *testing.T) {
57+
spec := testPluginSpec(t)
58+
59+
cleanup, err := UnpackContents(t.Context(), spec, testPluginContent(t, "new"), false)
60+
require.NoError(t, err)
61+
cleanup(true)
62+
63+
assert.Equal(t, "new", binaryContents(t, spec))
64+
assert.True(t, workspace.HasPlugin(spec))
65+
}
66+
67+
//nolint:paralleltest // t.Setenv is incompatible with t.Parallel
68+
func TestUnpackContentsKeepsCompleteInstall(t *testing.T) {
69+
spec := testPluginSpec(t)
70+
71+
cleanup, err := UnpackContents(t.Context(), spec, testPluginContent(t, "original"), false)
72+
require.NoError(t, err)
73+
cleanup(true)
74+
75+
cleanup, err = UnpackContents(t.Context(), spec, testPluginContent(t, "new"), false)
76+
require.NoError(t, err)
77+
cleanup(true)
78+
79+
assert.Equal(t, "original", binaryContents(t, spec))
80+
assert.True(t, workspace.HasPlugin(spec))
81+
}
82+
83+
//nolint:paralleltest // t.Setenv is incompatible with t.Parallel
84+
func TestUnpackContentsReinstallOverwrites(t *testing.T) {
85+
spec := testPluginSpec(t)
86+
87+
cleanup, err := UnpackContents(t.Context(), spec, testPluginContent(t, "original"), false)
88+
require.NoError(t, err)
89+
cleanup(true)
90+
91+
cleanup, err = UnpackContents(t.Context(), spec, testPluginContent(t, "new"), true)
92+
require.NoError(t, err)
93+
cleanup(true)
94+
95+
assert.Equal(t, "new", binaryContents(t, spec))
96+
assert.True(t, workspace.HasPlugin(spec))
97+
}
98+
99+
//nolint:paralleltest // t.Setenv is incompatible with t.Parallel
100+
func TestUnpackContentsRecoversFailedInstall(t *testing.T) {
101+
spec := testPluginSpec(t)
102+
103+
cleanup, err := UnpackContents(t.Context(), spec, testPluginContent(t, "broken"), false)
104+
require.NoError(t, err)
105+
cleanup(false)
106+
107+
assert.False(t, workspace.HasPlugin(spec))
108+
109+
cleanup, err = UnpackContents(t.Context(), spec, testPluginContent(t, "new"), false)
110+
require.NoError(t, err)
111+
cleanup(true)
112+
113+
assert.Equal(t, "new", binaryContents(t, spec))
114+
assert.True(t, workspace.HasPlugin(spec))
115+
}

0 commit comments

Comments
 (0)