Skip to content

Commit 37a05d4

Browse files
estespestesp
authored
Merge pull request #2110 from crosbymichael/release-onclose
[release 1.0] Handle shim killed on restore
2 parents 6ddb0bd + d1d3fd0 commit 37a05d4

9 files changed

Lines changed: 47 additions & 28 deletions

File tree

‎linux/bundle.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -84,9 +84,9 @@ func ShimLocal(exchange *exchange.Exchange) ShimOpt {
8484
}
8585

8686
// ShimConnect is a ShimOpt for connecting to an existing remote shim
87-
func ShimConnect() ShimOpt {
87+
func ShimConnect(onClose func()) ShimOpt {
8888
return func(b *bundle, ns string, ropts *runctypes.RuncOptions) (shim.Config, client.Opt) {
89-
return b.shimConfig(ns, ropts), client.WithConnect(b.shimAddress(ns))
89+
return b.shimConfig(ns, ropts), client.WithConnect(b.shimAddress(ns), onClose)
9090
}
9191
}
9292

‎linux/runtime.go‎

Lines changed: 14 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -201,10 +201,7 @@ func (r *Runtime) Create(ctx context.Context, id string, opts runtime.CreateOpts
201201
"id": id,
202202
"namespace": namespace,
203203
}).Warn("cleaning up after killed shim")
204-
err = r.cleanupAfterDeadShim(context.Background(), bundle, namespace, id, lc.pid)
205-
if err == nil {
206-
r.tasks.Delete(ctx, lc)
207-
} else {
204+
if err = r.cleanupAfterDeadShim(context.Background(), bundle, namespace, id, lc.pid); err != nil {
208205
log.G(ctx).WithError(err).WithFields(logrus.Fields{
209206
"id": id,
210207
"namespace": namespace,
@@ -313,7 +310,7 @@ func (r *Runtime) Delete(ctx context.Context, c runtime.Task) (*runtime.Exit, er
313310
}
314311
return nil, errdefs.FromGRPC(err)
315312
}
316-
r.tasks.Delete(ctx, lc)
313+
r.tasks.Delete(ctx, lc.id)
317314
if err := lc.shim.KillShim(ctx); err != nil {
318315
log.G(ctx).WithError(err).Error("failed to kill shim")
319316
}
@@ -383,7 +380,17 @@ func (r *Runtime) loadTasks(ctx context.Context, ns string) ([]*Task, error) {
383380
)
384381
ctx = namespaces.WithNamespace(ctx, ns)
385382
pid, _ := runc.ReadPidFile(filepath.Join(bundle.path, proc.InitPidFile))
386-
s, err := bundle.NewShimClient(ctx, ns, ShimConnect(), nil)
383+
s, err := bundle.NewShimClient(ctx, ns, ShimConnect(func() {
384+
log.G(ctx).WithError(err).WithFields(logrus.Fields{
385+
"id": id,
386+
"namespace": ns,
387+
}).Error("connecting to shim")
388+
err := r.cleanupAfterDeadShim(ctx, bundle, ns, id, pid)
389+
if err != nil {
390+
log.G(ctx).WithError(err).WithField("bundle", bundle.path).
391+
Error("cleaning up after dead shim")
392+
}
393+
}), nil)
387394
if err != nil {
388395
log.G(ctx).WithError(err).WithFields(logrus.Fields{
389396
"id": id,
@@ -433,6 +440,7 @@ func (r *Runtime) cleanupAfterDeadShim(ctx context.Context, bundle *bundle, ns,
433440
ExitedAt: exitedAt,
434441
})
435442

443+
r.tasks.Delete(ctx, id)
436444
if err := bundle.Delete(); err != nil {
437445
log.G(ctx).WithError(err).Error("delete bundle")
438446
}
@@ -448,12 +456,10 @@ func (r *Runtime) cleanupAfterDeadShim(ctx context.Context, bundle *bundle, ns,
448456
}
449457

450458
func (r *Runtime) terminate(ctx context.Context, bundle *bundle, ns, id string) error {
451-
ctx = namespaces.WithNamespace(ctx, ns)
452459
rt, err := r.getRuntime(ctx, ns, id)
453460
if err != nil {
454461
return err
455462
}
456-
457463
if err := rt.Delete(ctx, id, &runc.DeleteOpts{
458464
Force: true,
459465
}); err != nil {

‎linux/shim/client/client.go‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -77,7 +77,7 @@ func WithStart(binary, address, daemonAddress, cgroup string, debug bool, exitHa
7777
if err = sys.SetOOMScore(cmd.Process.Pid, sys.OOMScoreMaxKillable); err != nil {
7878
return nil, nil, errors.Wrap(err, "failed to set OOM Score on shim")
7979
}
80-
c, clo, err := WithConnect(address)(ctx, config)
80+
c, clo, err := WithConnect(address, func() {})(ctx, config)
8181
if err != nil {
8282
return nil, nil, errors.Wrap(err, "failed to connect")
8383
}
@@ -146,13 +146,15 @@ func annonDialer(address string, timeout time.Duration) (net.Conn, error) {
146146
}
147147

148148
// WithConnect connects to an existing shim
149-
func WithConnect(address string) Opt {
149+
func WithConnect(address string, onClose func()) Opt {
150150
return func(ctx context.Context, config shim.Config) (shimapi.ShimService, io.Closer, error) {
151151
conn, err := connect(address, annonDialer)
152152
if err != nil {
153153
return nil, nil, err
154154
}
155-
return shimapi.NewShimClient(ttrpc.NewClient(conn)), conn, nil
155+
client := ttrpc.NewClient(conn)
156+
client.OnClose(onClose)
157+
return shimapi.NewShimClient(client), conn, nil
156158
}
157159
}
158160

‎runtime/task_list.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -92,7 +92,7 @@ func (l *TaskList) AddWithNamespace(namespace string, t Task) error {
9292
}
9393

9494
// Delete a task
95-
func (l *TaskList) Delete(ctx context.Context, t Task) {
95+
func (l *TaskList) Delete(ctx context.Context, id string) {
9696
l.mu.Lock()
9797
defer l.mu.Unlock()
9898
namespace, err := namespaces.NamespaceRequired(ctx)
@@ -101,6 +101,6 @@ func (l *TaskList) Delete(ctx context.Context, t Task) {
101101
}
102102
tasks, ok := l.tasks[namespace]
103103
if ok {
104-
delete(tasks, t.ID())
104+
delete(tasks, id)
105105
}
106106
}

‎vendor.conf‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,4 +40,4 @@ github.com/boltdb/bolt e9cf4fae01b5a8ff89d0ec6b32f0d9c9f79aefdd
4040
google.golang.org/genproto d80a6e20e776b0b17a324d0ba1ab50a39c8e8944
4141
golang.org/x/text 19e51611da83d6be54ddafce4a4af510cb3e9ea4
4242
github.com/dmcgowan/go-tar go1.10
43-
github.com/stevvooe/ttrpc d2710463e497617f16f26d1e715a3308609e7982
43+
github.com/stevvooe/ttrpc d4528379866b0ce7e9d71f3eb96f0582fc374577

‎vendor/github.com/stevvooe/ttrpc/channel.go‎

Lines changed: 6 additions & 3 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎vendor/github.com/stevvooe/ttrpc/client.go‎

Lines changed: 15 additions & 7 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎vendor/github.com/stevvooe/ttrpc/server.go‎

Lines changed: 1 addition & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎windows/runtime.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -194,7 +194,7 @@ func (r *windowsRuntime) Delete(ctx context.Context, t runtime.Task) (*runtime.E
194194
}
195195

196196
wt.cleanup()
197-
r.tasks.Delete(ctx, t)
197+
r.tasks.Delete(ctx, t.ID())
198198

199199
r.publisher.Publish(ctx,
200200
runtime.TaskDeleteEventTopic,

0 commit comments

Comments
 (0)