Skip to content

Commit 06562f3

Browse files
authored
Harden watcher consumption and adopt bounded Tilt watch delivery (#292)
* Harden DCP watcher consumption and lifecycle cleanup Keep the V2 Namespace watch draining during snapshot requests, coalesce refresh notifications, and join the reader on shutdown. Initialize LogStorage disposal signaling and avoid allocating resource watchers before parent watch setup succeeds. Co-authored-by: Copilot App <[email protected]> Copilot-Session: 4667f368-688b-4dbb-bea1-af8c03f8438a * Temporarily pin bounded Tilt watch delivery fixes Use the immutable fork revision from tilt-dev/tilt-apiserver#117 until the fixes are available upstream. Refresh dependency checksums and notices, remove the test that relies on synchronous watch publication blocking deletion, and update comments for asynchronous delivery. Co-authored-by: Copilot App <[email protected]> Copilot-Session: 4667f368-688b-4dbb-bea1-af8c03f8438a * Preserve package-based Aspire scaffold directives Keep all scaffolded build directives when inserting the regression AppHost body, supporting both SDK-based and package-based daily templates. Require an SDK or package directive rather than assuming Aspire.AppHost.Sdk is present. Co-authored-by: Copilot App <[email protected]> Copilot-Session: 4667f368-688b-4dbb-bea1-af8c03f8438a * Leave Aspire scaffold validation to the CLI Preserve scaffolded build directives without requiring a particular directive type or any directives at all. Let Aspire and .NET interpret the generated AppHost configuration. Co-authored-by: Copilot App <[email protected]> Copilot-Session: 4667f368-688b-4dbb-bea1-af8c03f8438a * Pin merged upstream Tilt watch delivery commit Reference tilt-dev/tilt-apiserver commit d35b179a41c077755089e03ce5a7b40b922b75ce directly through its upstream pseudo-version. Remove the temporary fork replacement and update checksums and dependency notices. Co-authored-by: Copilot App <[email protected]> Copilot-Session: 4667f368-688b-4dbb-bea1-af8c03f8438a * Update tilt-apiserver to v0.19.2 Use the released bounded watch delivery fixes instead of the upstream pseudo-version. Refresh module checksums and dependency notices. Co-authored-by: Copilot App <[email protected]> Copilot-Session: 4667f368-688b-4dbb-bea1-af8c03f8438a --------- Copilot-Session: 4667f368-688b-4dbb-bea1-af8c03f8438a
1 parent 795c80f commit 06562f3

8 files changed

Lines changed: 394 additions & 115 deletions

‎NOTICE‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8127,8 +8127,8 @@ OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
81278127

81288128
----------------------------------------------------------
81298129

8130-
github.com/tilt-dev/tilt-apiserver/pkg v0.19.1 - Apache-2.0
8131-
https://github.com/tilt-dev/tilt-apiserver/blob/v0.19.1/LICENSE
8130+
github.com/tilt-dev/tilt-apiserver/pkg v0.19.2 - Apache-2.0
8131+
https://github.com/tilt-dev/tilt-apiserver/blob/v0.19.2/LICENSE
81328132

81338133
Apache License
81348134
Version 2.0, January 2004

‎api/v1/log_storage.go‎

Lines changed: 12 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -181,6 +181,7 @@ func NewLogStorage(parentKindStorage registry_rest.StandardStorage, logStreamer
181181
resourceWatchers: &syncmap.Map[types.UID, *syncmap.Map[*resourceWatcher, bool]]{},
182182
mutex: &sync.Mutex{},
183183
watcher: nil,
184+
disposeCh: make(chan struct{}),
184185
}, nil
185186
}
186187

@@ -193,6 +194,13 @@ func (ls *LogStorage) watchResource(uid types.UID, log logr.Logger) (*resourceWa
193194
return nil, fmt.Errorf("log storage has been disposed")
194195
}
195196

197+
if ls.watcher == nil {
198+
watchErr := ls.watchResourceEvents(log)
199+
if watchErr != nil {
200+
return nil, watchErr
201+
}
202+
}
203+
196204
resourceWatchers, _ := ls.resourceWatchers.LoadOrStoreNew(uid, func() *syncmap.Map[*resourceWatcher, bool] {
197205
return &syncmap.Map[*resourceWatcher, bool]{}
198206
})
@@ -201,16 +209,8 @@ func (ls *LogStorage) watchResource(uid types.UID, log logr.Logger) (*resourceWa
201209
rw.OnStop(func() {
202210
resourceWatchers.Delete(rw)
203211
})
204-
205-
if ls.watcher == nil {
206-
watchErr := ls.watchResourceEvents(log)
207-
if watchErr != nil {
208-
return nil, watchErr
209-
}
210-
} else {
211-
if event, found := ls.mostRecentResourceEvents.Load(uid); found {
212-
rw.Queue(event)
213-
}
212+
if event, found := ls.mostRecentResourceEvents.Load(uid); found {
213+
rw.Queue(event)
214214
}
215215

216216
_, _ = resourceWatchers.LoadOrStore(rw, true)
@@ -266,11 +266,7 @@ func (ls *LogStorage) watchResourceEvents(log logr.Logger) error {
266266
return
267267
}
268268

269-
// This has to be initiated before making a call to create a new watcher.
270-
// If not, we can end up in a deadlock situation where the storage is trying to deliver
271-
// a watch event (holding the watch set lock in shared mode) while at the same time
272-
// we are making a call to create a new watcher (which tries to take the watch set lock
273-
// in exclusive mode).
269+
// Stop the old watcher before registering its replacement so generations do not overlap.
274270
log.V(1).Info("Stopping old parent resource watcher...")
275271
stopWatcher(ls.watcher)
276272
log.V(1).Info("Old parent resource watcher stopped")
@@ -542,15 +538,7 @@ func (ls *LogStorage) NewGetOptions() (runtime.Object, bool, string) {
542538
}
543539

544540
func stopWatcher(w watch.Interface) {
545-
// We need to drain the watcher result channel because it is an unbuffered channel
546-
// that the Tilt API server storage layer synchronously writes to it as part of handling object changes.
547-
// If we simply abandon the channel with some pending writes, the corresponding update will block forever.
548-
// Even calling Stop() on the watcher will NOT help, because the Stop() method tries to take an exclusive lock
549-
// on the "watch set" that the watcher belongs to before closing the watcher channel,
550-
// and lock is held (in shared mode) by the goroutine that is writing to the channel
551-
// to broadcast object update events.
552-
// By calling drain() on a separate goroutine we ensure that the channel
553-
// will continue to be drained until the call to Stop() succeeds.
541+
// Keep draining until the result channel closes so pending deliveries are not abandoned during shutdown.
554542
go drain(w.ResultChan())
555543

556544
w.Stop()

‎api/v1/log_storage_test.go‎

Lines changed: 104 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,104 @@
1+
/*---------------------------------------------------------------------------------------------
2+
* Copyright (c) Microsoft Corporation. All rights reserved.
3+
* Licensed under the MIT License. See LICENSE in the project root for license information.
4+
*--------------------------------------------------------------------------------------------*/
5+
6+
package v1
7+
8+
import (
9+
"context"
10+
"errors"
11+
"io"
12+
"testing"
13+
14+
"github.com/go-logr/logr"
15+
"github.com/stretchr/testify/require"
16+
apiserver_resource "github.com/tilt-dev/tilt-apiserver/pkg/server/builder/resource"
17+
metainternalversion "k8s.io/apimachinery/pkg/apis/meta/internalversion"
18+
"k8s.io/apimachinery/pkg/watch"
19+
registry_rest "k8s.io/apiserver/pkg/registry/rest"
20+
21+
"github.com/microsoft/dcp/pkg/testutil"
22+
)
23+
24+
// Verifies that LogStorage.watchResource does not allocate a resource watcher when parent
25+
// watch setup fails and can establish the watch on a subsequent request.
26+
func TestLogStorageWatchSetupFailureCanBeRetried(t *testing.T) {
27+
ctx, cancel := testutil.GetTestContext(t, 0)
28+
defer cancel()
29+
setupErr := errors.New("test watch setup failure")
30+
parentWatcher := watch.NewRaceFreeFake()
31+
parentStorage := &logStorageWatchSource{watcher: parentWatcher, watchErr: setupErr}
32+
storage, storageErr := NewLogStorage(parentStorage, logStorageTestStreamer{})
33+
require.NoError(t, storageErr)
34+
t.Cleanup(storage.Destroy)
35+
36+
resourceWatcher, watchErr := storage.watchResource("resource", logr.Discard())
37+
require.ErrorIs(t, watchErr, setupErr)
38+
require.Nil(t, resourceWatcher)
39+
require.Nil(t, storage.watcher)
40+
require.True(t, storage.resourceWatchers.Empty())
41+
42+
parentStorage.watchErr = nil
43+
retryWatcher, retryErr := storage.watchResource("resource", logr.Discard())
44+
require.NoError(t, retryErr)
45+
retryWatcher.Stop()
46+
select {
47+
case _, open := <-retryWatcher.ResultChan():
48+
require.False(t, open)
49+
case <-ctx.Done():
50+
t.Fatal(ctx.Err())
51+
}
52+
53+
storage.Destroy()
54+
require.True(t, parentWatcher.IsStopped())
55+
}
56+
57+
// Verifies that LogStorage.Destroy closes its disposal signal without a parent watcher,
58+
// remains idempotent, and rejects subsequent resource-watch requests.
59+
func TestLogStorageDestroyWithoutParentWatcher(t *testing.T) {
60+
storage, storageErr := NewLogStorage(&logStorageWatchSource{}, logStorageTestStreamer{})
61+
require.NoError(t, storageErr)
62+
t.Cleanup(storage.Destroy)
63+
64+
storage.Destroy()
65+
storage.Destroy()
66+
select {
67+
case <-storage.disposeCh:
68+
default:
69+
t.Fatal("log storage disposal signal was not closed")
70+
}
71+
resourceWatcher, watchErr := storage.watchResource("resource", logr.Discard())
72+
require.ErrorContains(t, watchErr, "disposed")
73+
require.Nil(t, resourceWatcher)
74+
require.True(t, storage.resourceWatchers.Empty())
75+
}
76+
77+
type logStorageWatchSource struct {
78+
registry_rest.StandardStorage
79+
watcher watch.Interface
80+
watchErr error
81+
}
82+
83+
func (source *logStorageWatchSource) Watch(context.Context, *metainternalversion.ListOptions) (watch.Interface, error) {
84+
if source.watchErr != nil {
85+
return nil, source.watchErr
86+
}
87+
return source.watcher, nil
88+
}
89+
90+
type logStorageTestStreamer struct{}
91+
92+
func (logStorageTestStreamer) Preflight(apiserver_resource.Object, *LogOptions) error {
93+
return nil
94+
}
95+
96+
func (logStorageTestStreamer) StreamLogs(context.Context, io.Writer, apiserver_resource.Object, *LogOptions, logr.Logger) (ResourceStreamStatus, <-chan struct{}, error) {
97+
return ResourceStreamStatusDone, nil, nil
98+
}
99+
100+
func (logStorageTestStreamer) OnResourceUpdated(ResourceWatcherEvent, logr.Logger) {}
101+
102+
func (logStorageTestStreamer) Dispose() error {
103+
return nil
104+
}

‎go.mod‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ require (
1717
github.com/spf13/cobra v1.10.2
1818
github.com/spf13/pflag v1.0.10
1919
github.com/stretchr/testify v1.12.1
20-
github.com/tilt-dev/tilt-apiserver v0.19.1
20+
github.com/tilt-dev/tilt-apiserver v0.19.2
2121
github.com/tklauser/go-sysconf v0.4.0
2222
go.opentelemetry.io/otel v1.46.0
2323
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.46.0

‎go.sum‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -562,8 +562,8 @@ github.com/tenntenn/text/transform v0.0.0-20200319021203-7eef512accb3 h1:f+jULpR
562562
github.com/tenntenn/text/transform v0.0.0-20200319021203-7eef512accb3/go.mod h1:ON8b8w4BN/kE1EOhwT0o+d62W65a6aPw1nouo9LMgyY=
563563
github.com/tetafro/godot v1.5.4 h1:u1ww+gqpRLiIA16yF2PV1CV1n/X3zhyezbNXC3E14Sg=
564564
github.com/tetafro/godot v1.5.4/go.mod h1:eOkMrVQurDui411nBY2FA05EYH01r14LuWY/NrVDVcU=
565-
github.com/tilt-dev/tilt-apiserver v0.19.1 h1:j6B/EpzgbJuMCLo/lvskPowHx5oFgQT7VXTyzTHe7zw=
566-
github.com/tilt-dev/tilt-apiserver v0.19.1/go.mod h1:w6vnH5H5npxMvy+iVj+DBqtiFV9TooSOZh/7shv4Njo=
565+
github.com/tilt-dev/tilt-apiserver v0.19.2 h1:0bujknyw1XYO648mzFbCeG64pwHVLYszHYjIsRiRLas=
566+
github.com/tilt-dev/tilt-apiserver v0.19.2/go.mod h1:w6vnH5H5npxMvy+iVj+DBqtiFV9TooSOZh/7shv4Njo=
567567
github.com/timakin/bodyclose v0.0.0-20241222091800-1db5c5ca4d67 h1:9LPGD+jzxMlnk5r6+hJnar67cgpDIz/iyD+rfl5r2Vk=
568568
github.com/timakin/bodyclose v0.0.0-20241222091800-1db5c5ca4d67/go.mod h1:mkjARE7Yr8qU23YcGMSALbIxTQ9r9QBVahQOBRfU460=
569569
github.com/timonwong/loggercheck v0.11.0 h1:jdaMpYBl+Uq9mWPXv1r8jc5fC3gyXx4/WGwTnnNKn4M=

‎internal/apiserver/v2_namespace_lifecycle_storage_test.go‎

Lines changed: 0 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,6 @@ package apiserver
88
import (
99
"context"
1010
"errors"
11-
"sync"
1211
"testing"
1312

1413
"github.com/stretchr/testify/require"
@@ -81,51 +80,6 @@ func TestV2NamespaceLifecycleStorageHoldsChildLeaseAfterHTTPTimeout(t *testing.T
8180
requireNoV2NamespaceMutation(t, fixture.gate, "test")
8281
}
8382

84-
func TestV2NamespaceLifecycleStorageHoldsDeleteLeaseAfterHTTPTimeout(t *testing.T) {
85-
fixture := newV2NamespaceStorageFixture(t)
86-
fixture.createNamespace(t)
87-
namespaceWatch, watchErr := fixture.namespaces.Watch(fixture.ctx, nil)
88-
require.NoError(t, watchErr)
89-
stopWatch := sync.OnceFunc(func() { stopV2NamespaceWatcher(namespaceWatch) })
90-
defer stopWatch()
91-
persisted := make(chan struct{})
92-
fixture.fs.afterWrite = func(obj runtime.Object) {
93-
if namespace, namespaceOK := obj.(*apiv2.Namespace); namespaceOK && namespace.DeletionTimestamp != nil {
94-
close(persisted)
95-
}
96-
}
97-
requestCtx, cancelRequest := context.WithCancel(fixture.ctx)
98-
defer cancelRequest()
99-
httpResult := make(chan error, 1)
100-
storageDone := make(chan struct{})
101-
go func() {
102-
_, finishErr := finisher.FinishRequest(requestCtx, func() (runtime.Object, error) {
103-
defer close(storageDone)
104-
deleted, _, deleteErr := fixture.namespaces.Delete(requestCtx, "test", nil, nil)
105-
return deleted, deleteErr
106-
})
107-
httpResult <- finishErr
108-
}()
109-
waitForSignal(t, fixture.ctx, persisted)
110-
cancelRequest()
111-
require.True(t, apierrors.IsTimeout(waitForError(t, fixture.ctx, httpResult)))
112-
fixture.requireTerminating(t)
113-
waitV2NamespaceMutationReferences(t, fixture.ctx, fixture.gate, "test", 1)
114-
require.Empty(t, fixture.gate.closedNamespaceStates())
115-
_, createErr := fixture.children.Create(fixture.ctx, newV2LifecycleChild(), nil, nil)
116-
require.True(t, apierrors.IsForbidden(createErr))
117-
118-
stopWatch()
119-
waitForSignal(t, fixture.ctx, storageDone)
120-
requireNoV2NamespaceMutation(t, fixture.gate, "test")
121-
fixture.gate.lock.Lock()
122-
defer fixture.gate.lock.Unlock()
123-
state := fixture.gate.namespaces["test"]
124-
require.True(t, state.closed)
125-
require.True(t, state.deleteAccepted)
126-
require.Zero(t, state.activeDeletes)
127-
}
128-
12983
func TestV2NamespaceLifecycleStorageSerializesNamespaceCreateAndDelete(t *testing.T) {
13084
fixture := newV2NamespaceStorageFixture(t)
13185
writeStarted, releaseWrite := make(chan struct{}), make(chan struct{})

0 commit comments

Comments
 (0)