Skip to content

Commit 3be0f42

Browse files
committed
kubectl-ate logs: resume follow from the last line read, validate flags first
A reconnect in follow mode resumed from the last line displayed, so under a sparse filter such as --source=lifecycle it replayed every line since the last match, which can be hours of output. The stream is one pod log in write order, so resuming from the last line read, displayed or not, skips nothing; the cursor now advances on every line that parses. The --source and --container combination is validated before the client dials ate-api-server, so a bad flag is reported as such rather than as an unreachable server, and the runner is given the finished filter instead of building it in Run. The kubeconfig error path closes the API client it had already opened. The docs no longer claim the lifecycle view shows migrations: it reads the current worker's log, which holds none of the earlier workers' events.
1 parent 141fbff commit 3be0f42

3 files changed

Lines changed: 74 additions & 71 deletions

File tree

‎cmd/kubectl-ate/internal/cmd/actor.go‎

Lines changed: 27 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -305,14 +305,13 @@ func (s *k8sPodLogsStreamer) StreamLogs(ctx context.Context, namespace, podName
305305

306306
// LogsActorRunner executes the log printing or streaming.
307307
type LogsActorRunner struct {
308-
apiClient AteAPIClient
309-
streamer PodLogsStreamer
310-
actorRef resources.ActorRef
311-
stdout io.Writer
312-
stderr io.Writer
313-
follow bool
314-
source string
315-
container string
308+
apiClient AteAPIClient
309+
streamer PodLogsStreamer
310+
actorRef resources.ActorRef
311+
stdout io.Writer
312+
stderr io.Writer
313+
follow bool
314+
// filter selects the lines to display; see newLogLineFilter.
316315
filter logLineFilter
317316
pollInterval time.Duration
318317
reconnectInterval time.Duration
@@ -323,12 +322,6 @@ type LogsActorRunner struct {
323322
func (r *LogsActorRunner) Run(ctx context.Context) error {
324323
defer r.apiClient.Close()
325324

326-
filter, err := newLogLineFilter(r.actorRef, r.source, r.container)
327-
if err != nil {
328-
return err
329-
}
330-
r.filter = filter
331-
332325
if r.pollInterval <= 0 {
333326
r.pollInterval = 2 * time.Second
334327
}
@@ -515,13 +508,22 @@ func (r *LogsActorRunner) startMigrationMonitor(
515508
func runLogsActor(cmd *cobra.Command, args []string) error {
516509
ctx := cmd.Context()
517510

511+
// Before any connection: a bad flag combination is answered as such, not
512+
// as an unreachable server.
513+
actorRef := resources.ActorRef{Atespace: logsActorAtespaceFlag, Name: args[0]}
514+
filter, err := newLogLineFilter(actorRef, logsActorSourceFlag, logsActorContainerFlag)
515+
if err != nil {
516+
return err
517+
}
518+
518519
apiClient, err := ateclient.NewClient(ctx, kubeconfig, k8sContext, endpoint, tokenFile, traceEnabled)
519520
if err != nil {
520521
return fmt.Errorf("failed to connect to ate-api-server: %w", err)
521522
}
522523

523524
config, err := ateclient.LoadKubeConfig(kubeconfig, k8sContext)
524525
if err != nil {
526+
apiClient.Close()
525527
return fmt.Errorf("while loading kubeconfig: %w", err)
526528
}
527529
k8sClient, err := kubernetes.NewForConfig(config)
@@ -533,12 +535,11 @@ func runLogsActor(cmd *cobra.Command, args []string) error {
533535
runner := &LogsActorRunner{
534536
apiClient: apiClient,
535537
streamer: &k8sPodLogsStreamer{clientset: k8sClient},
536-
actorRef: resources.ActorRef{Atespace: logsActorAtespaceFlag, Name: args[0]},
538+
actorRef: actorRef,
537539
stdout: os.Stdout,
538540
stderr: os.Stderr,
539541
follow: logsActorFollowFlag,
540-
source: logsActorSourceFlag,
541-
container: logsActorContainerFlag,
542+
filter: filter,
542543
pollInterval: 2 * time.Second,
543544
reconnectInterval: 1 * time.Second,
544545
tickerInterval: 2 * time.Second,
@@ -602,6 +603,14 @@ func (f logLineFilter) matches(emitter resources.ActorRef, containerName string)
602603
return f.container == "" || containerName == f.container
603604
}
604605

606+
// filterAndDisplayLogLine writes line to w if filter selects it. It returns
607+
// the line's time whenever the line parses, displayed or not, so follow mode
608+
// can resume from the last line it read: the stream is one pod log in write
609+
// order, so a later line never precedes an earlier one, and resuming from an
610+
// undisplayed line skips nothing. Resuming only from displayed lines would
611+
// make a reconnect under a sparse filter, such as --source=lifecycle, replay
612+
// everything since the last match. The bool reports whether line was
613+
// displayed.
605614
func filterAndDisplayLogLine(line string, filter logLineFilter, w io.Writer) (time.Time, bool) {
606615
var m map[string]any
607616
dec := json.NewDecoder(strings.NewReader(line))
@@ -635,7 +644,7 @@ func filterAndDisplayLogLine(line string, filter logLineFilter, w io.Writer) (ti
635644
}
636645

637646
if !filter.matches(emitter, emitterContainer) {
638-
return time.Time{}, false
647+
return logTime, false
639648
}
640649

641650
// Remove substrate's labels from CLI output. Stripping the whole reserved

‎cmd/kubectl-ate/internal/cmd/actor_test.go‎

Lines changed: 46 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ import (
3232
"google.golang.org/grpc/status"
3333
"google.golang.org/protobuf/testing/protocmp"
3434
corev1 "k8s.io/api/core/v1"
35+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
3536
)
3637

3738
func TestActorCommandArgs(t *testing.T) {
@@ -155,39 +156,39 @@ func TestFilterAndDisplayLogLine(t *testing.T) {
155156
line: `{"time":"2026-05-16T01:03:38Z","message":"Hello world","logging.googleapis.com/labels":{"ate.atespace":"space-1","ate.actor.name":"act-2"}}`,
156157
target: resources.ActorRef{Atespace: "space-1", Name: "act-1"},
157158
wantMatched: false,
158-
wantTime: "", // zero: another actor's line must not advance follow's resume cursor
159+
wantTime: "2026-05-16T01:03:38Z", // undisplayed, but still the follow resume cursor
159160
wantOutput: "",
160161
},
161162
{
162163
name: "same actor name in a different atespace",
163164
line: `{"time":"2026-05-16T01:03:38Z","message":"Hello world","logging.googleapis.com/labels":{"ate.atespace":"space-2","ate.actor.name":"act-1"}}`,
164165
target: resources.ActorRef{Atespace: "space-1", Name: "act-1"},
165166
wantMatched: false,
166-
wantTime: "",
167+
wantTime: "2026-05-16T01:03:38Z", // undisplayed, but still the follow resume cursor
167168
wantOutput: "",
168169
},
169170
{
170171
name: "matching actor name without atespace label",
171172
line: `{"time":"2026-05-16T01:03:38Z","message":"Hello world","logging.googleapis.com/labels":{"ate.actor.name":"act-1"}}`,
172173
target: resources.ActorRef{Atespace: "space-1", Name: "act-1"},
173174
wantMatched: false,
174-
wantTime: "",
175+
wantTime: "2026-05-16T01:03:38Z", // undisplayed, but still the follow resume cursor
175176
wantOutput: "",
176177
},
177178
{
178179
name: "empty target atespace does not match empty atespace label",
179180
line: `{"time":"2026-05-16T01:03:38Z","message":"Hello world","logging.googleapis.com/labels":{"ate.atespace":"","ate.actor.name":"act-1"}}`,
180181
target: resources.ActorRef{Atespace: "", Name: "act-1"},
181182
wantMatched: false,
182-
wantTime: "",
183+
wantTime: "2026-05-16T01:03:38Z", // undisplayed, but still the follow resume cursor
183184
wantOutput: "",
184185
},
185186
{
186187
name: "empty target actor name does not match empty name label",
187188
line: `{"time":"2026-05-16T01:03:38Z","message":"Hello world","logging.googleapis.com/labels":{"ate.atespace":"space-1","ate.actor.name":""}}`,
188189
target: resources.ActorRef{Atespace: "space-1", Name: ""},
189190
wantMatched: false,
190-
wantTime: "",
191+
wantTime: "2026-05-16T01:03:38Z", // undisplayed, but still the follow resume cursor
191192
wantOutput: "",
192193
},
193194
{
@@ -264,7 +265,7 @@ func TestFilterAndDisplayLogLine(t *testing.T) {
264265
target: resources.ActorRef{Atespace: "space-1", Name: "act-1"},
265266
container: "counter",
266267
wantMatched: false,
267-
wantTime: "", // filtered out: only displayed lines may advance follow's resume cursor
268+
wantTime: "2026-05-16T01:03:38Z", // undisplayed, but still the follow resume cursor
268269
wantOutput: "",
269270
},
270271
{
@@ -273,7 +274,7 @@ func TestFilterAndDisplayLogLine(t *testing.T) {
273274
target: resources.ActorRef{Atespace: "space-1", Name: "act-1"},
274275
container: "counter",
275276
wantMatched: false,
276-
wantTime: "",
277+
wantTime: "2026-05-16T01:03:38Z", // undisplayed, but still the follow resume cursor
277278
wantOutput: "",
278279
},
279280
{
@@ -300,7 +301,7 @@ func TestFilterAndDisplayLogLine(t *testing.T) {
300301
target: resources.ActorRef{Atespace: "space-1", Name: "act-1"},
301302
source: "lifecycle",
302303
wantMatched: false,
303-
wantTime: "",
304+
wantTime: "2026-05-16T01:03:38Z", // undisplayed, but still the follow resume cursor
304305
wantOutput: "",
305306
},
306307
{
@@ -309,7 +310,7 @@ func TestFilterAndDisplayLogLine(t *testing.T) {
309310
target: resources.ActorRef{Atespace: "space-1", Name: "act-1"},
310311
source: "lifecycle",
311312
wantMatched: false,
312-
wantTime: "",
313+
wantTime: "2026-05-16T01:03:38Z", // undisplayed, but still the follow resume cursor
313314
wantOutput: "",
314315
},
315316
{
@@ -327,7 +328,7 @@ func TestFilterAndDisplayLogLine(t *testing.T) {
327328
target: resources.ActorRef{Atespace: "space-1", Name: "act-1"},
328329
source: "containers",
329330
wantMatched: false,
330-
wantTime: "",
331+
wantTime: "2026-05-16T01:03:38Z", // undisplayed, but still the follow resume cursor
331332
wantOutput: "",
332333
},
333334
{
@@ -337,7 +338,7 @@ func TestFilterAndDisplayLogLine(t *testing.T) {
337338
source: "containers",
338339
container: "counter",
339340
wantMatched: false,
340-
wantTime: "",
341+
wantTime: "2026-05-16T01:03:38Z", // undisplayed, but still the follow resume cursor
341342
wantOutput: "",
342343
},
343344
}
@@ -480,6 +481,7 @@ func TestLogsActorRunner_Run_OneShotSuccess(t *testing.T) {
480481
runner := &LogsActorRunner{
481482
apiClient: mockAPI,
482483
actorRef: resources.ActorRef{Atespace: "space-1", Name: actorName},
484+
filter: logLineFilter{target: resources.ActorRef{Atespace: "space-1", Name: actorName}},
483485
streamer: mockStreamer,
484486
stdout: &stdout,
485487
stderr: &stderr,
@@ -519,7 +521,6 @@ func TestLogsActorRunner_Run_OneShot_SourceFilter(t *testing.T) {
519521
source string
520522
container string
521523
wantOutput []string
522-
wantErr string
523524
}{
524525
{
525526
name: "no filter shows everything",
@@ -551,17 +552,6 @@ func TestLogsActorRunner_Run_OneShot_SourceFilter(t *testing.T) {
551552
source: "lifecycle",
552553
wantOutput: []string{"Actor started", "Actor checkpointing"},
553554
},
554-
{
555-
name: "source=lifecycle with container filter is rejected before streaming",
556-
source: "lifecycle",
557-
container: "counter",
558-
wantErr: "--container cannot be combined with --source=lifecycle",
559-
},
560-
{
561-
name: "unknown source is rejected before streaming",
562-
source: "supervisor",
563-
wantErr: `invalid --source "supervisor"`,
564-
},
565555
}
566556

567557
for _, tc := range tests {
@@ -595,29 +585,19 @@ func TestLogsActorRunner_Run_OneShot_SourceFilter(t *testing.T) {
595585
streamer: mockStreamer,
596586
stdout: &stdout,
597587
stderr: &stderr,
598-
source: tc.source,
599-
container: tc.container,
588+
filter: logLineFilter{target: resources.ActorRef{Atespace: "space-1", Name: actorName}, source: logSource(tc.source), container: tc.container},
600589
}
601590

602591
err := runner.Run(context.Background())
603592
if mockAPI.CloseCalls != 1 {
604593
t.Errorf("api client Close called %d times, want 1", mockAPI.CloseCalls)
605594
}
606-
if tc.wantErr != "" {
607-
if err == nil || !strings.Contains(err.Error(), tc.wantErr) {
608-
t.Fatalf("got err %v, want it to contain %q", err, tc.wantErr)
609-
}
610-
if streamCalls != 0 {
611-
t.Errorf("streamed %d times, want 0: an invalid flag combination must fail before any logs are fetched", streamCalls)
612-
}
613-
if stdout.Len() != 0 {
614-
t.Errorf("unexpected output %q", stdout.String())
615-
}
616-
return
617-
}
618595
if err != nil {
619596
t.Fatalf("unexpected error: %v", err)
620597
}
598+
if streamCalls != 1 {
599+
t.Errorf("streamed %d times, want 1", streamCalls)
600+
}
621601

622602
var gotLines []string
623603
if got := strings.TrimSpace(stdout.String()); got != "" {
@@ -657,6 +637,7 @@ func TestLogsActorRunner_Run_OneShot_ActorNotRunning(t *testing.T) {
657637
runner := &LogsActorRunner{
658638
apiClient: mockAPI,
659639
actorRef: resources.ActorRef{Atespace: "space-1", Name: actorName},
640+
filter: logLineFilter{target: resources.ActorRef{Atespace: "space-1", Name: actorName}},
660641
streamer: mockStreamer,
661642
stdout: &stdout,
662643
stderr: &stderr,
@@ -746,6 +727,7 @@ func TestLogsActorRunner_Run_Follow_SuspendedToRunning(t *testing.T) {
746727
runner := &LogsActorRunner{
747728
apiClient: mockAPI,
748729
actorRef: resources.ActorRef{Atespace: "space-1", Name: actorName},
730+
filter: logLineFilter{target: resources.ActorRef{Atespace: "space-1", Name: actorName}},
749731
streamer: mockStreamer,
750732
stdout: &stdout,
751733
stderr: &stderr,
@@ -796,6 +778,7 @@ func TestLogsActorRunner_Run_Follow_NotFoundActor(t *testing.T) {
796778
runner := &LogsActorRunner{
797779
apiClient: mockAPI,
798780
actorRef: resources.ActorRef{Atespace: "space-1", Name: actorName},
781+
filter: logLineFilter{target: resources.ActorRef{Atespace: "space-1", Name: actorName}},
799782
streamer: mockStreamer,
800783
stdout: &stdout,
801784
stderr: &stderr,
@@ -911,6 +894,7 @@ func TestLogsActorRunner_Run_Follow_ActorMigration(t *testing.T) {
911894
runner := &LogsActorRunner{
912895
apiClient: mockAPI,
913896
actorRef: resources.ActorRef{Atespace: "space-1", Name: actorName},
897+
filter: logLineFilter{target: resources.ActorRef{Atespace: "space-1", Name: actorName}},
914898
streamer: mockStreamer,
915899
stdout: &stdout,
916900
stderr: &stderr,
@@ -1032,6 +1016,7 @@ func TestLogsActorRunner_Run_Follow_ActorSuspendedMidStream(t *testing.T) {
10321016
runner := &LogsActorRunner{
10331017
apiClient: mockAPI,
10341018
actorRef: resources.ActorRef{Atespace: "space-1", Name: actorName},
1019+
filter: logLineFilter{target: resources.ActorRef{Atespace: "space-1", Name: actorName}},
10351020
streamer: mockStreamer,
10361021
stdout: &stdout,
10371022
stderr: &stderr,
@@ -1055,10 +1040,12 @@ func TestLogsActorRunner_Run_Follow_ActorSuspendedMidStream(t *testing.T) {
10551040
}
10561041
}
10571042

1058-
// A worker pod's stream carries lines that are not displayed (other actors,
1059-
// filtered-out containers). Those must not advance the follow resume cursor
1060-
// (SinceTime), or a reconnect could skip the lines the user asked for.
1061-
func TestLogsActorRunner_Run_Follow_UndisplayedLineDoesNotAdvanceCursor(t *testing.T) {
1043+
// The follow resume cursor (SinceTime) advances on every line the stream
1044+
// carries, displayed or not. The stream is one pod log in write order, so
1045+
// resuming from the last undisplayed line skips nothing; resuming only from
1046+
// displayed lines would make a reconnect under a sparse filter, such as
1047+
// --source=lifecycle, replay everything since the last match.
1048+
func TestLogsActorRunner_Run_Follow_CursorAdvancesOnUndisplayedLines(t *testing.T) {
10621049
actorName := "act-123"
10631050

10641051
mockAPI := &mockAteAPIClient{
@@ -1076,17 +1063,20 @@ func TestLogsActorRunner_Run_Follow_UndisplayedLineDoesNotAdvanceCursor(t *testi
10761063
},
10771064
}
10781065

1079-
// A different actor's line and a filtered-out sidecar line of our own actor:
1080-
// both carry parseable timestamps but neither is displayed.
1081-
foreignLine := `{"time":"2026-05-16T01:03:38Z","message":"not mine","logging.googleapis.com/labels":{"ate.atespace":"space-1","ate.actor.name":"other"}}`
1082-
sidecarLine := `{"time":"2026-05-16T01:03:39Z","level":"info","msg":"from sidecar","logging.googleapis.com/labels":{"ate.atespace":"space-1","ate.actor.name":"act-123","ate.actor.container.name":"sidecar"}}`
1066+
// Under --source=lifecycle only the first line is displayed. The two
1067+
// after it, a different actor's and our own container's, are not, and the
1068+
// last of them is where a reconnect must resume.
1069+
lifecycleLine := `{"time":"2026-05-16T01:03:38Z","message":"Actor started","logging.googleapis.com/labels":{"ate.atespace":"space-1","ate.actor.name":"act-123"}}`
1070+
foreignLine := `{"time":"2026-05-16T01:03:39Z","message":"not mine","logging.googleapis.com/labels":{"ate.atespace":"space-1","ate.actor.name":"other"}}`
1071+
counterLine := `{"time":"2026-05-16T01:03:40Z","level":"info","msg":"from counter","logging.googleapis.com/labels":{"ate.atespace":"space-1","ate.actor.name":"act-123","ate.actor.container.name":"counter"}}`
1072+
wantSince := time.Date(2026, 5, 16, 1, 3, 40, 0, time.UTC)
10831073

10841074
ctx, cancel := context.WithCancel(context.Background())
10851075
defer cancel()
10861076

10871077
var mu sync.Mutex
10881078
var streamCalls int
1089-
var reconnectHadSinceTime bool
1079+
var reconnectSince *metav1.Time
10901080
secondCall := make(chan struct{})
10911081

10921082
mockStreamer := &mockPodLogsStreamer{
@@ -1096,11 +1086,12 @@ func TestLogsActorRunner_Run_Follow_UndisplayedLineDoesNotAdvanceCursor(t *testi
10961086
streamCalls++
10971087

10981088
if streamCalls == 1 {
1099-
// First connection: only undisplayed lines, then EOF to force a reconnect.
1100-
return io.NopCloser(strings.NewReader(foreignLine + "\n" + sidecarLine + "\n")), nil
1089+
// First connection: one displayed line, then two undisplayed ones,
1090+
// then EOF to force a reconnect.
1091+
return io.NopCloser(strings.NewReader(lifecycleLine + "\n" + foreignLine + "\n" + counterLine + "\n")), nil
11011092
}
11021093
if streamCalls == 2 {
1103-
reconnectHadSinceTime = opts.SinceTime != nil
1094+
reconnectSince = opts.SinceTime
11041095
close(secondCall)
11051096
}
11061097
return io.NopCloser(strings.NewReader("")), nil
@@ -1114,7 +1105,7 @@ func TestLogsActorRunner_Run_Follow_UndisplayedLineDoesNotAdvanceCursor(t *testi
11141105
stdout: &bytes.Buffer{},
11151106
stderr: &bytes.Buffer{},
11161107
follow: true,
1117-
container: "counter",
1108+
filter: logLineFilter{target: resources.ActorRef{Atespace: "space-1", Name: actorName}, source: logSourceLifecycle},
11181109
pollInterval: 1 * time.Millisecond,
11191110
reconnectInterval: 1 * time.Millisecond,
11201111
tickerInterval: time.Hour, // keep the migration monitor out of the way
@@ -1139,7 +1130,10 @@ func TestLogsActorRunner_Run_Follow_UndisplayedLineDoesNotAdvanceCursor(t *testi
11391130

11401131
mu.Lock()
11411132
defer mu.Unlock()
1142-
if reconnectHadSinceTime {
1143-
t.Error("reconnect carried a SinceTime cursor; undisplayed lines must not advance it")
1133+
if reconnectSince == nil {
1134+
t.Fatal("reconnect carried no SinceTime cursor")
1135+
}
1136+
if !reconnectSince.Time.Equal(wantSince) {
1137+
t.Errorf("reconnect SinceTime = %v, want %v, the last line read, not the last line displayed", reconnectSince.Time, wantSince)
11441138
}
11451139
}

0 commit comments

Comments
 (0)