Skip to content

Commit 5747747

Browse files
committed
fix(manifestutil): report one message when the CRD wait is cancelled
WaitForCRDsEstablished reported a cancelled context two different ways. The check at the top of the poll loop returned a message with no CRD name; the select at the bottom named the CRD the poll had stopped on. Which one ran was left to the scheduler: when the deadline and a tick come ready in the same instant, a select picks between the ready cases at random, so one cancellation reached the operator either way. Build the message in one place and return it from both exits, so the two are indistinguishable by construction. The name is carried across iterations, since a cancellation noticed at the top of the loop has no poll of its own to take one from, and starts as the first CRD of the set for a wait cancelled before any poll observed anything. Keeping the check at the top is what makes carrying the name worthwhile. A client refuses Get on a dead context at the first name in the list whichever CRD is actually lagging, so a loop that re-entered and polled anyway would overwrite the name its last poll settled on. The check closes that window and no more than that: a context dying part-way through a poll still records the first Get to fail, because this branch cannot tell a refused request from a CRD status. Assisted-By: Claude <[email protected]> Signed-off-by: Aleksei Sviridkin <[email protected]>
1 parent ccd8a28 commit 5747747

2 files changed

Lines changed: 296 additions & 7 deletions

File tree

‎internal/manifestutil/crd.go‎

Lines changed: 25 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -45,19 +45,38 @@ func WaitForCRDsEstablished(ctx context.Context, k8sClient client.Client, crdNam
4545
ticker := time.NewTicker(500 * time.Millisecond)
4646
defer ticker.Stop()
4747

48+
// The CRD named if the wait is cut short. Until a poll observes the set
49+
// nothing is known about which one lags, so the first of it stands in;
50+
// after that it holds the CRD the last poll stopped on.
51+
pendingCRD := crdNames[0]
52+
53+
// Cancellation leaves the loop from two places and reads the same from
54+
// both, which is the point rather than an accident. When the deadline and
55+
// a tick come ready together, select picks between them at random, so any
56+
// difference between the two exits would reach the operator at random too.
57+
cancelled := func() error {
58+
return fmt.Errorf("context cancelled while waiting for CRD %q to be established: %w", pendingCRD, ctx.Err())
59+
}
60+
4861
for {
49-
select {
50-
case <-ctx.Done():
51-
return fmt.Errorf("context cancelled while waiting for CRDs to be established: %w", ctx.Err())
52-
default:
62+
// Polling a dead context spends a Get to learn nothing: a client
63+
// refuses it, and the refusal would land on the first name in the list
64+
// whichever CRD is actually lagging, renaming what gets reported.
65+
if ctx.Err() != nil {
66+
return cancelled()
5367
}
5468

5569
allEstablished := true
56-
var pendingCRD string
5770
for _, name := range crdNames {
5871
crd := &unstructured.Unstructured{}
5972
crd.SetGroupVersionKind(crdGVK)
6073
if err := k8sClient.Get(ctx, types.NamespacedName{Name: name}, crd); err != nil {
74+
// The poll was entered on a live context, so this is the CRD
75+
// it stopped on and not whichever name a blanket refusal
76+
// reached first. A context dying inside the poll still lands
77+
// here and does record the first name to fail: this branch
78+
// cannot tell a refused request from a CRD status, which is
79+
// the wider problem it has for RBAC and connection errors too.
6180
allEstablished = false
6281
pendingCRD = name
6382
break
@@ -97,7 +116,7 @@ func WaitForCRDsEstablished(ctx context.Context, k8sClient client.Client, crdNam
97116

98117
select {
99118
case <-ctx.Done():
100-
return fmt.Errorf("context cancelled while waiting for CRD %q to be established: %w", pendingCRD, ctx.Err())
119+
return cancelled()
101120
case <-ticker.C:
102121
}
103122
}

‎internal/manifestutil/crd_test.go‎

Lines changed: 271 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,10 @@ package manifestutil
1818

1919
import (
2020
"context"
21+
"errors"
22+
"fmt"
2123
"strings"
24+
"sync/atomic"
2225
"testing"
2326
"time"
2427

@@ -187,6 +190,274 @@ func TestWaitForCRDsEstablished_timeout(t *testing.T) {
187190
if !strings.Contains(err.Error(), "packages.cozystack.io") {
188191
t.Errorf("error should mention stuck CRD name, got: %v", err)
189192
}
193+
// The only deadline-driven test here, so it is the only one that can catch
194+
// the cause being reported as something other than what ended the wait.
195+
if !errors.Is(err, context.DeadlineExceeded) {
196+
t.Errorf("error should wrap the deadline cause, got: %v", err)
197+
}
198+
}
199+
200+
// crdObject builds a CustomResourceDefinition for the fake client. Passing nil
201+
// conditions leaves status unset, which is one of the two shapes the wait reads
202+
// as "not established yet"; the other is a status carrying conditions that do
203+
// not include an established one.
204+
func crdObject(name string, conditions []interface{}) *unstructured.Unstructured {
205+
obj := map[string]interface{}{
206+
"apiVersion": "apiextensions.k8s.io/v1",
207+
"kind": "CustomResourceDefinition",
208+
"metadata": map[string]interface{}{"name": name},
209+
}
210+
if conditions != nil {
211+
obj["status"] = map[string]interface{}{"conditions": conditions}
212+
}
213+
return &unstructured.Unstructured{Object: obj}
214+
}
215+
216+
func establishedConditions() []interface{} {
217+
return []interface{}{map[string]interface{}{"type": "Established", "status": "True"}}
218+
}
219+
220+
func unestablishedConditions() []interface{} {
221+
return []interface{}{map[string]interface{}{"type": "NamesAccepted", "status": "True"}}
222+
}
223+
224+
// cancellationScheme returns the scheme the cancellation tests share.
225+
func cancellationScheme(t *testing.T) *runtime.Scheme {
226+
t.Helper()
227+
scheme := runtime.NewScheme()
228+
if err := apiextensionsv1.AddToScheme(scheme); err != nil {
229+
t.Fatalf("failed to add apiextensions to scheme: %v", err)
230+
}
231+
return scheme
232+
}
233+
234+
// assertCancelled checks the whole message, not just the CRD in it. The point
235+
// of the fix is that every way out of the wait reports cancellation the same
236+
// way, so a second message that happens to carry a name is still the defect;
237+
// comparing the text is what makes that observable to a test.
238+
func assertCancelled(t *testing.T, err error, wantCRD string) {
239+
t.Helper()
240+
if err == nil {
241+
t.Fatal("WaitForCRDsEstablished() expected error on cancelled context, got nil")
242+
}
243+
want := fmt.Sprintf("context cancelled while waiting for CRD %q to be established: %v", wantCRD, context.Canceled)
244+
if err.Error() != want {
245+
t.Errorf("error = %q, want %q", err.Error(), want)
246+
}
247+
// Callers wrap this error again, so the cause has to survive for anyone
248+
// matching on it rather than on the text.
249+
if !errors.Is(err, context.Canceled) {
250+
t.Errorf("error should wrap the context cause, got: %v", err)
251+
}
252+
}
253+
254+
// TestWaitForCRDsEstablished_cancelledBeforeFirstPoll covers the interleaving
255+
// the timeout test above cannot reach: the context is already cancelled when
256+
// the function is entered, so no poll ever observes the set and the name in
257+
// the message can only be the one the wait starts with. The client answers
258+
// regardless of the context, so a wait that polled anyway would settle on the
259+
// second CRD and be visible here rather than passing by accident.
260+
func TestWaitForCRDsEstablished_cancelledBeforeFirstPoll(t *testing.T) {
261+
log.SetLogger(zap.New(zap.UseDevMode(true)))
262+
263+
fakeClient := fake.NewClientBuilder().
264+
WithScheme(cancellationScheme(t)).
265+
WithObjects(
266+
crdObject("established.cozystack.io", establishedConditions()),
267+
crdObject("packages.cozystack.io", nil),
268+
).
269+
Build()
270+
271+
ctx, cancel := context.WithCancel(context.Background())
272+
ctx = log.IntoContext(ctx, log.FromContext(context.Background()))
273+
cancel()
274+
275+
err := WaitForCRDsEstablished(ctx, fakeClient, []string{"established.cozystack.io", "packages.cozystack.io"})
276+
assertCancelled(t, err, "established.cozystack.io")
277+
}
278+
279+
// TestWaitForCRDsEstablished_cancelledMidPollNamesUncheckedCRD pins which CRD
280+
// the message names when the deadline lands inside a poll. The poll walks past
281+
// one CRD it confirms established and is cut off on the next, which is the one
282+
// whose state is genuinely unknown and the one worth reporting. A third name
283+
// behind it keeps "the first Get to fail" apart from "the last one tried",
284+
// which two CRDs cannot tell apart.
285+
func TestWaitForCRDsEstablished_cancelledMidPollNamesUncheckedCRD(t *testing.T) {
286+
log.SetLogger(zap.New(zap.UseDevMode(true)))
287+
288+
ctx, cancel := context.WithCancel(context.Background())
289+
defer cancel()
290+
ctx = log.IntoContext(ctx, log.FromContext(context.Background()))
291+
292+
// The first Get is served, so the first CRD is confirmed established. The
293+
// deadline then lands on the second, the way it would against a live
294+
// client: cancelled first, refused second.
295+
gets := 0
296+
fakeClient := fake.NewClientBuilder().
297+
WithScheme(cancellationScheme(t)).
298+
WithObjects(
299+
crdObject("established.cozystack.io", establishedConditions()),
300+
crdObject("packages.cozystack.io", nil),
301+
crdObject("later.cozystack.io", nil),
302+
).
303+
WithInterceptorFuncs(interceptor.Funcs{
304+
Get: func(ctx context.Context, c client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error {
305+
gets++
306+
if gets > 1 {
307+
cancel()
308+
return context.Canceled
309+
}
310+
if err := ctx.Err(); err != nil {
311+
return err
312+
}
313+
return c.Get(ctx, key, obj, opts...)
314+
},
315+
}).
316+
Build()
317+
318+
err := WaitForCRDsEstablished(ctx, fakeClient, []string{"established.cozystack.io", "packages.cozystack.io", "later.cozystack.io"})
319+
assertCancelled(t, err, "packages.cozystack.io")
320+
}
321+
322+
// TestWaitForCRDsEstablished_cancelledAfterPollKeepsName pins that the name a
323+
// completed poll settled on survives the wait that follows it. The two cases
324+
// are the two shapes of "not established yet", which the loop records from
325+
// different branches.
326+
func TestWaitForCRDsEstablished_cancelledAfterPollKeepsName(t *testing.T) {
327+
log.SetLogger(zap.New(zap.UseDevMode(true)))
328+
329+
for _, tc := range []struct {
330+
name string
331+
conditions []interface{}
332+
}{
333+
{name: "status absent", conditions: nil},
334+
{name: "no established condition", conditions: unestablishedConditions()},
335+
} {
336+
t.Run(tc.name, func(t *testing.T) {
337+
ctx, cancel := context.WithCancel(context.Background())
338+
defer cancel()
339+
ctx = log.IntoContext(ctx, log.FromContext(context.Background()))
340+
341+
// Both Gets are served, so the poll completes and settles on the
342+
// second CRD. The context then dies while the loop waits for a tick.
343+
gets := 0
344+
fakeClient := fake.NewClientBuilder().
345+
WithScheme(cancellationScheme(t)).
346+
WithObjects(
347+
crdObject("established.cozystack.io", establishedConditions()),
348+
crdObject("packages.cozystack.io", tc.conditions),
349+
).
350+
WithInterceptorFuncs(interceptor.Funcs{
351+
Get: func(ctx context.Context, c client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error {
352+
if err := ctx.Err(); err != nil {
353+
return err
354+
}
355+
err := c.Get(ctx, key, obj, opts...)
356+
gets++
357+
if gets == 2 {
358+
cancel()
359+
}
360+
return err
361+
},
362+
}).
363+
Build()
364+
365+
err := WaitForCRDsEstablished(ctx, fakeClient, []string{"established.cozystack.io", "packages.cozystack.io"})
366+
assertCancelled(t, err, "packages.cozystack.io")
367+
})
368+
}
369+
}
370+
371+
// reenteringContext forces the loop to start an iteration on a context that is
372+
// already cancelled. Against a real context that needs the bottom select to
373+
// have taken the tick while both cases were ready, which is a coin toss no test
374+
// can hold still. Done stays open for the single read that select makes in the
375+
// iteration where cancellation lands, so the tick wins by construction; every
376+
// read after that returns a closed channel, so the next iteration sees a
377+
// context cancelled through both Done and Err, the way a real one is. The
378+
// standard library guarantees that pairing rather than merely happening to
379+
// provide it: cancelCtx.Err reads <-c.Done() before returning a non-nil error.
380+
type reenteringContext struct {
381+
context.Context
382+
dead *atomic.Bool
383+
tickWon *atomic.Bool
384+
open chan struct{}
385+
closed chan struct{}
386+
}
387+
388+
func newReenteringContext(parent context.Context) reenteringContext {
389+
closed := make(chan struct{})
390+
close(closed)
391+
return reenteringContext{
392+
Context: parent,
393+
dead: &atomic.Bool{},
394+
tickWon: &atomic.Bool{},
395+
open: make(chan struct{}),
396+
closed: closed,
397+
}
398+
}
399+
400+
func (c reenteringContext) Done() <-chan struct{} {
401+
if c.dead.Load() && !c.tickWon.CompareAndSwap(false, true) {
402+
return c.closed
403+
}
404+
return c.open
405+
}
406+
407+
func (c reenteringContext) Err() error {
408+
if c.dead.Load() && c.tickWon.Load() {
409+
return context.Canceled
410+
}
411+
return nil
412+
}
413+
414+
// TestWaitForCRDsEstablished_cancelledOnLoopReentryKeepsName pins that the CRD
415+
// name survives into the next iteration. One poll completes and settles on the
416+
// second CRD, the context dies during the wait, and the loop re-enters and
417+
// leaves through the check at the top. That check has no poll of its own to
418+
// take a name from, so the one it reports has to be the one carried over.
419+
func TestWaitForCRDsEstablished_cancelledOnLoopReentryKeepsName(t *testing.T) {
420+
log.SetLogger(zap.New(zap.UseDevMode(true)))
421+
422+
base := log.IntoContext(context.Background(), log.FromContext(context.Background()))
423+
reCtx := newReenteringContext(base)
424+
425+
gets := 0
426+
fakeClient := fake.NewClientBuilder().
427+
WithScheme(cancellationScheme(t)).
428+
WithObjects(
429+
crdObject("established.cozystack.io", establishedConditions()),
430+
crdObject("packages.cozystack.io", nil),
431+
).
432+
WithInterceptorFuncs(interceptor.Funcs{
433+
Get: func(ctx context.Context, c client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error {
434+
if err := ctx.Err(); err != nil {
435+
return err
436+
}
437+
err := c.Get(ctx, key, obj, opts...)
438+
gets++
439+
if gets == 2 {
440+
reCtx.dead.Store(true)
441+
}
442+
return err
443+
},
444+
}).
445+
Build()
446+
447+
// The double answers Done from a fixed script, so a loop that stops reading
448+
// it the expected number of times never leaves. Bound the call: a hang here
449+
// costs the whole package's CI timeout, a failure costs one line.
450+
done := make(chan error, 1)
451+
go func() {
452+
done <- WaitForCRDsEstablished(reCtx, fakeClient, []string{"established.cozystack.io", "packages.cozystack.io"})
453+
}()
454+
455+
select {
456+
case err := <-done:
457+
assertCancelled(t, err, "packages.cozystack.io")
458+
case <-time.After(5 * time.Second):
459+
t.Fatal("WaitForCRDsEstablished() never returned: the loop stopped taking the cancellation exits this fixture scripts")
460+
}
190461
}
191462

192463
func TestWaitForCRDsEstablished_empty(t *testing.T) {
@@ -199,4 +470,3 @@ func TestWaitForCRDsEstablished_empty(t *testing.T) {
199470
t.Fatalf("WaitForCRDsEstablished() with empty names should return nil, got: %v", err)
200471
}
201472
}
202-

0 commit comments

Comments
 (0)