Skip to content

Commit 7549958

Browse files
danegstaCopilot
andcommitted
Copy finite stdio logs asynchronously
Return a completion channel for non-follow and completed-resource log copies so client backpressure does not hold resource processing locks. Preserve first-EOF behavior without registering lifecycle followers, and let storage cancellation close the response pipe to unblock delivery. Cover finite-copy formatting, EOF, slow readers, lifecycle progress, request cancellation, and storage disposal. Co-authored-by: Copilot App <[email protected]>
1 parent 873d1b8 commit 7549958

2 files changed

Lines changed: 234 additions & 8 deletions

File tree

Lines changed: 222 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,222 @@
1+
/*---------------------------------------------------------------------------------------------
2+
* Copyright (c) Microsoft Corporation. All rights reserved.
3+
* Licensed under the MIT License. See LICENSE in the project root for license information.
4+
*--------------------------------------------------------------------------------------------*/
5+
6+
package stdiologs
7+
8+
import (
9+
"bytes"
10+
"context"
11+
"io"
12+
"path/filepath"
13+
"sync"
14+
"testing"
15+
16+
"github.com/go-logr/logr"
17+
"github.com/stretchr/testify/require"
18+
resource "github.com/tilt-dev/tilt-apiserver/pkg/server/builder/resource"
19+
metainternalversion "k8s.io/apimachinery/pkg/apis/meta/internalversion"
20+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
21+
"k8s.io/apimachinery/pkg/runtime"
22+
"k8s.io/apimachinery/pkg/watch"
23+
"k8s.io/apiserver/pkg/registry/rest"
24+
25+
apiv1 "github.com/microsoft/dcp/api/v1"
26+
"github.com/microsoft/dcp/internal/logs"
27+
usvc_io "github.com/microsoft/dcp/pkg/io"
28+
"github.com/microsoft/dcp/pkg/osutil"
29+
"github.com/microsoft/dcp/pkg/testutil"
30+
)
31+
32+
// Verifies that stdIoLogStreamer finite copies preserve formatting, finish at EOF even when
33+
// empty, and never register lifecycle followers or consume output appended after completion.
34+
func TestFiniteLogCopiesFinishAtEOF(t *testing.T) {
35+
t.Parallel()
36+
one := int64(1)
37+
for _, testCase := range []struct {
38+
name string
39+
contents string
40+
options apiv1.LogOptions
41+
finished bool
42+
expected string
43+
}{
44+
{name: "empty"},
45+
{name: "non-follow", contents: "first\nsecond\n", expected: "first\nsecond\n"},
46+
{name: "completed-follow", contents: "first\nsecond\n", options: apiv1.LogOptions{Follow: true}, finished: true, expected: "first\nsecond\n"},
47+
{name: "tail", contents: "first\nsecond\n", options: apiv1.LogOptions{Tail: &one}, expected: "second\n"},
48+
{name: "limit", contents: "first\nsecond\n", options: apiv1.LogOptions{Limit: &one}, expected: "first\n"},
49+
{name: "skip", contents: "first\nsecond\n", options: apiv1.LogOptions{Skip: &one}, expected: "second\n"},
50+
{name: "numbering", contents: "1 2026-10-30T12:00:00Z first\n", options: apiv1.LogOptions{LineNumbers: true}, expected: "1 first\n"},
51+
} {
52+
t.Run(testCase.name, func(t *testing.T) {
53+
t.Parallel()
54+
ctx, cancel := testutil.GetTestContext(t, 0)
55+
defer cancel()
56+
path := filepath.Join(t.TempDir(), "stdout")
57+
require.NoError(t, usvc_io.WriteFile(path, []byte(testCase.contents), osutil.PermissionOnlyOwnerReadWrite))
58+
streamer := &stdIoLogStreamer{lock: &sync.Mutex{}, activeStreams: make(logs.LogStreamMop)}
59+
defer func() { require.NoError(t, streamer.Dispose()) }()
60+
executable := &apiv1.Executable{
61+
ObjectMeta: metav1.ObjectMeta{Name: "finite", UID: "finite-uid"},
62+
Status: apiv1.ExecutableStatus{StdOutFile: path},
63+
}
64+
if testCase.finished {
65+
executable.Status.FinishTimestamp = metav1.NowMicro()
66+
}
67+
var output bytes.Buffer
68+
status, done, streamErr := streamer.StreamLogs(ctx, &output, executable, &testCase.options, logr.Discard())
69+
require.NoError(t, streamErr)
70+
require.Equal(t, apiv1.ResourceStreamStatusStreaming, status)
71+
select {
72+
case <-done:
73+
case <-ctx.Done():
74+
t.Fatal("finite log copy did not finish")
75+
}
76+
require.Equal(t, testCase.expected, output.String())
77+
streamer.lock.Lock()
78+
followerCount := len(streamer.activeStreams)
79+
streamer.lock.Unlock()
80+
require.Zero(t, followerCount)
81+
appended, appendErr := usvc_io.OpenOrCreateFileForAppending(path, osutil.PermissionOnlyOwnerReadWrite)
82+
require.NoError(t, appendErr)
83+
_, writeErr := appended.Write([]byte("after-eof\n"))
84+
require.NoError(t, writeErr)
85+
require.NoError(t, appended.Close())
86+
require.Equal(t, testCase.expected, output.String())
87+
require.NoError(t, logs.RemoveWithRetry(ctx, path))
88+
})
89+
}
90+
}
91+
92+
type finiteCopyStorage struct {
93+
rest.StandardStorage
94+
executable *apiv1.Executable
95+
watcher *watch.RaceFreeFakeWatcher
96+
}
97+
98+
func (storage *finiteCopyStorage) Get(context.Context, string, *metav1.GetOptions) (runtime.Object, error) {
99+
return storage.executable.DeepCopy(), nil
100+
}
101+
102+
func (storage *finiteCopyStorage) Watch(context.Context, *metainternalversion.ListOptions) (watch.Interface, error) {
103+
return storage.watcher, nil
104+
}
105+
106+
type finiteCopyObserver struct {
107+
*stdIoLogStreamer
108+
writing chan struct{}
109+
started chan (<-chan struct{})
110+
updated chan struct{}
111+
}
112+
113+
func (observer *finiteCopyObserver) StreamLogs(
114+
ctx context.Context, dest io.Writer, obj resource.Object, opts *apiv1.LogOptions, log logr.Logger,
115+
) (apiv1.ResourceStreamStatus, <-chan struct{}, error) {
116+
status, done, streamErr := observer.stdIoLogStreamer.StreamLogs(ctx, &finiteCopyDestination{
117+
Writer: dest, writing: observer.writing,
118+
}, obj, opts, log)
119+
select {
120+
case observer.started <- done:
121+
case <-ctx.Done():
122+
}
123+
return status, done, streamErr
124+
}
125+
126+
func (observer *finiteCopyObserver) OnResourceUpdated(event apiv1.ResourceWatcherEvent, log logr.Logger) {
127+
observer.stdIoLogStreamer.OnResourceUpdated(event, log)
128+
observer.updated <- struct{}{}
129+
}
130+
131+
type finiteCopyDestination struct {
132+
io.Writer
133+
writing chan struct{}
134+
once sync.Once
135+
}
136+
137+
func (destination *finiteCopyDestination) Write(contents []byte) (int, error) {
138+
destination.once.Do(func() { close(destination.writing) })
139+
return destination.Writer.Write(contents)
140+
}
141+
142+
// Verifies that LogStorage releases resource processing during a backpressured finite copy,
143+
// preserves complete delivery, and closes its pipe on cancellation or disposal to unblock the copy.
144+
func TestFiniteLogCopiesReleaseProcessingAndHonorStorageCancellation(t *testing.T) {
145+
t.Parallel()
146+
for _, action := range []string{"drain", "cancel", "dispose"} {
147+
t.Run(action, func(t *testing.T) {
148+
t.Parallel()
149+
ctx, cancel := testutil.GetTestContext(t, 0)
150+
defer cancel()
151+
requestCtx, cancelRequest := context.WithCancel(ctx)
152+
defer cancelRequest()
153+
path := filepath.Join(t.TempDir(), "stdout")
154+
const expected = "finite-output\n"
155+
require.NoError(t, usvc_io.WriteFile(path, []byte(expected), osutil.PermissionOnlyOwnerReadWrite))
156+
parent := &finiteCopyStorage{
157+
executable: &apiv1.Executable{
158+
ObjectMeta: metav1.ObjectMeta{Name: "finite", UID: "finite-uid"},
159+
Status: apiv1.ExecutableStatus{StdOutFile: path},
160+
},
161+
watcher: watch.NewRaceFreeFake(),
162+
}
163+
observer := &finiteCopyObserver{
164+
stdIoLogStreamer: &stdIoLogStreamer{lock: &sync.Mutex{}, activeStreams: make(logs.LogStreamMop)},
165+
writing: make(chan struct{}), started: make(chan (<-chan struct{}), 1), updated: make(chan struct{}, 2),
166+
}
167+
defer func() { require.NoError(t, observer.Dispose()) }()
168+
storage, storageErr := apiv1.NewLogStorage(parent, observer)
169+
require.NoError(t, storageErr)
170+
defer storage.Destroy()
171+
response, getErr := storage.Get(requestCtx, parent.executable.Name, &apiv1.LogOptions{})
172+
require.NoError(t, getErr)
173+
reader, _, _, inputErr := response.(*apiv1.LogStreamer).InputStream(requestCtx, "", "")
174+
require.NoError(t, inputErr)
175+
defer reader.Close()
176+
parent.watcher.Add(parent.executable.DeepCopy())
177+
var done <-chan struct{}
178+
select {
179+
case done = <-observer.started:
180+
case <-ctx.Done():
181+
t.Fatal("stream setup waited for the consumer to read")
182+
}
183+
select {
184+
case <-observer.writing:
185+
case <-ctx.Done():
186+
t.Fatal("finite copy did not attempt delivery")
187+
}
188+
select {
189+
case <-done:
190+
t.Fatal("finite copy finished before the consumer read any output")
191+
default:
192+
}
193+
select {
194+
case <-observer.updated:
195+
case <-ctx.Done():
196+
t.Fatal("initial lifecycle update was not delivered")
197+
}
198+
parent.watcher.Modify(parent.executable.DeepCopy())
199+
select {
200+
case <-observer.updated:
201+
case <-ctx.Done():
202+
t.Fatal("finite copy held the resource processing lock")
203+
}
204+
switch action {
205+
case "drain":
206+
output, readErr := io.ReadAll(reader)
207+
require.NoError(t, readErr)
208+
require.Equal(t, expected, string(output))
209+
case "cancel":
210+
cancelRequest()
211+
case "dispose":
212+
storage.Destroy()
213+
}
214+
select {
215+
case <-done:
216+
case <-ctx.Done():
217+
t.Fatal("finite copy remained blocked on the response pipe")
218+
}
219+
require.NoError(t, logs.RemoveWithRetry(ctx, path))
220+
})
221+
}
222+
}

‎internal/logs/stdiologs/stdio_log_streamer.go‎

Lines changed: 12 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -145,14 +145,18 @@ func (sls stdIoLogStreamer) StreamLogs(
145145
src = usvc_io.NewTimestampAwareReader(src, logs.ToTimestampReaderOptions(opts))
146146

147147
if !opts.Follow || resource.Done() {
148-
defer logFile.Close()
149-
150-
_, copyErr := io.Copy(dest, src)
151-
if copyErr != nil {
152-
log.Error(copyErr, "Failed to copy log file to destination")
153-
}
154-
155-
return apiv1.ResourceStreamStatusDone, nil, nil
148+
done := make(chan struct{})
149+
go func() {
150+
defer close(done)
151+
defer src.Close()
152+
defer func() { _ = resiliency.MakePanicError(recover(), log) }()
153+
154+
_, copyErr := io.Copy(dest, src)
155+
if copyErr != nil && requestCtx.Err() == nil {
156+
log.Error(copyErr, "Failed to copy log file to destination")
157+
}
158+
}()
159+
return apiv1.ResourceStreamStatusStreaming, done, nil
156160
}
157161

158162
followWriter := usvc_io.NewFollowWriter(requestCtx, src, dest, usvc_io.WithCloseSourceOnCancel())

0 commit comments

Comments
 (0)