Skip to content

Commit 3284939

Browse files
fuweidruiwen-zhao
fuweid
authored and
ruiwen-zhao
committed
integration: reproduce #9347
Signed-off-by: Wei Fu <[email protected]>
1 parent d1aab27 commit 3284939

2 files changed

Lines changed: 105 additions & 3 deletions

File tree

‎integration/build_local_containerd_helper_test.go‎

Lines changed: 68 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,14 +21,17 @@ import (
2121
"path/filepath"
2222
"sync"
2323
"testing"
24+
"time"
2425

2526
"github.com/containerd/containerd"
27+
"github.com/containerd/containerd/content"
2628
"github.com/containerd/containerd/log/logtest"
2729
"github.com/containerd/containerd/pkg/cri/constants"
2830
"github.com/containerd/containerd/platforms"
2931
"github.com/containerd/containerd/plugin"
3032
ctrdsrv "github.com/containerd/containerd/services/server"
3133
srvconfig "github.com/containerd/containerd/services/server/config"
34+
"github.com/opencontainers/go-digest"
3235

3336
_ "github.com/containerd/containerd/diff/walking/plugin"
3437
"github.com/containerd/containerd/events/exchange"
@@ -59,9 +62,11 @@ var (
5962
loadedPluginsErr error
6063
)
6164

65+
type tweakPluginInitFunc func(t *testing.T, p *plugin.Registration) *plugin.Registration
66+
6267
// buildLocalContainerdClient is to return containerd client with initialized
6368
// core plugins in local.
64-
func buildLocalContainerdClient(t *testing.T, tmpDir string) *containerd.Client {
69+
func buildLocalContainerdClient(t *testing.T, tmpDir string, tweakInitFn tweakPluginInitFunc) *containerd.Client {
6570
ctx := logtest.WithT(context.Background(), t)
6671

6772
// load plugins
@@ -107,6 +112,10 @@ func buildLocalContainerdClient(t *testing.T, tmpDir string) *containerd.Client
107112
initContext.Config = pc
108113
}
109114

115+
if tweakInitFn != nil {
116+
p = tweakInitFn(t, p)
117+
}
118+
110119
result := p.Init(initContext)
111120
assert.NoError(t, initialized.Add(result))
112121

@@ -126,3 +135,61 @@ func buildLocalContainerdClient(t *testing.T, tmpDir string) *containerd.Client
126135

127136
return client
128137
}
138+
139+
func tweakContentInitFnWithDelayer(commitDelayDuration time.Duration) tweakPluginInitFunc {
140+
return func(t *testing.T, p *plugin.Registration) *plugin.Registration {
141+
if p.URI() != "io.containerd.content.v1.content" {
142+
return p
143+
}
144+
145+
oldInitFn := p.InitFn
146+
p.InitFn = func(ic *plugin.InitContext) (interface{}, error) {
147+
instance, err := oldInitFn(ic)
148+
if err != nil {
149+
return nil, err
150+
}
151+
152+
return &contentStoreDelayer{
153+
t: t,
154+
155+
Store: instance.(content.Store),
156+
commitDelayDuration: commitDelayDuration,
157+
}, nil
158+
}
159+
return p
160+
}
161+
}
162+
163+
type contentStoreDelayer struct {
164+
t *testing.T
165+
166+
content.Store
167+
commitDelayDuration time.Duration
168+
}
169+
170+
func (cs *contentStoreDelayer) Writer(ctx context.Context, opts ...content.WriterOpt) (content.Writer, error) {
171+
w, err := cs.Store.Writer(ctx, opts...)
172+
if err != nil {
173+
return nil, err
174+
}
175+
176+
return &contentWriterDelayer{
177+
t: cs.t,
178+
179+
Writer: w,
180+
commitDelayDuration: cs.commitDelayDuration,
181+
}, nil
182+
}
183+
184+
type contentWriterDelayer struct {
185+
t *testing.T
186+
187+
content.Writer
188+
commitDelayDuration time.Duration
189+
}
190+
191+
func (w *contentWriterDelayer) Commit(ctx context.Context, size int64, expected digest.Digest, opts ...content.Opt) error {
192+
w.t.Logf("[testcase: %s] Commit %v blob after %v", w.t.Name(), expected, w.commitDelayDuration)
193+
time.Sleep(w.commitDelayDuration)
194+
return w.Writer.Commit(ctx, size, expected, opts...)
195+
}

‎integration/image_pull_timeout_test.go‎

Lines changed: 37 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,41 @@ func TestCRIImagePullTimeout(t *testing.T) {
6262

6363
t.Run("HoldingContentOpenWriter", testCRIImagePullTimeoutByHoldingContentOpenWriter)
6464
t.Run("NoDataTransferred", testCRIImagePullTimeoutByNoDataTransferred)
65+
t.Run("SlowCommitWriter", testCRIImagePullTimeoutBySlowCommitWriter)
66+
}
67+
68+
// testCRIImagePullTimeoutBySlowCommitWriter tests that
69+
//
70+
// It should not cancel if the content.Commit takes long time.
71+
//
72+
// After copying all the data from registry, the request should be inactive
73+
// before content.Commit. If the blob is large, for instance, 2 GiB, the fsync
74+
// during content.Commit maybe take long time during IO pressure. The
75+
// content.Commit holds the bolt's writable mutex and blocks other goroutines
76+
// which are going to commit blob as well. If the progress tracker still
77+
// considers these requests active, it maybe file false alert and cancel the
78+
// ImagePull.
79+
//
80+
// It's reproducer for #9347.
81+
func testCRIImagePullTimeoutBySlowCommitWriter(t *testing.T) {
82+
t.Parallel()
83+
84+
tmpDir := t.TempDir()
85+
86+
delayDuration := 2 * defaultImagePullProgressTimeout
87+
cli := buildLocalContainerdClient(t, tmpDir, tweakContentInitFnWithDelayer(delayDuration))
88+
89+
criService, err := initLocalCRIPlugin(cli, tmpDir, criconfig.Registry{})
90+
assert.NoError(t, err)
91+
92+
ctx := namespaces.WithNamespace(logtest.WithT(context.Background(), t), k8sNamespace)
93+
94+
_, err = criService.PullImage(ctx, &runtimeapi.PullImageRequest{
95+
Image: &runtimeapi.ImageSpec{
96+
Image: pullProgressTestImageName,
97+
},
98+
})
99+
assert.NoError(t, err)
65100
}
66101

67102
// testCRIImagePullTimeoutByHoldingContentOpenWriter tests that
@@ -76,7 +111,7 @@ func testCRIImagePullTimeoutByHoldingContentOpenWriter(t *testing.T) {
76111

77112
tmpDir := t.TempDir()
78113

79-
cli := buildLocalContainerdClient(t, tmpDir)
114+
cli := buildLocalContainerdClient(t, tmpDir, nil)
80115

81116
criService, err := initLocalCRIPlugin(cli, tmpDir, criconfig.Registry{})
82117
assert.NoError(t, err)
@@ -214,7 +249,7 @@ func testCRIImagePullTimeoutByNoDataTransferred(t *testing.T) {
214249

215250
tmpDir := t.TempDir()
216251

217-
cli := buildLocalContainerdClient(t, tmpDir)
252+
cli := buildLocalContainerdClient(t, tmpDir, nil)
218253

219254
mirrorSrv := newMirrorRegistryServer(mirrorRegistryServerConfig{
220255
limitedBytesPerConn: 1024 * 1024 * 3, // 3MB

0 commit comments

Comments
 (0)