Add timeout to task state calls

crosbymichael · crosbymichael · commit 101d4b78eb7f · 2019-07-23T19:58:41.000Z
Fixes #3440 This also returns the task that times out or has an error on the state call in an UNKNOWN status so that the user can manually kill and remove the task. Signed-off-by: Michael Crosby <crosbymichael@gmail.com>
diff --git a/services/tasks/local.go b/services/tasks/local.go
@@ -266,12 +266,18 @@ func (l *local) DeleteProcess(ctx context.Context, r *api.DeleteProcessRequest,
 	}, nil
 }
 
-func processFromContainerd(ctx context.Context, p runtime.Process) (*task.Process, error) {
+func getProcessState(ctx context.Context, p runtime.Process) (*task.Process, error) {
+	ctx, cancel := context.WithTimeout(ctx, 2*time.Second)
+	defer cancel()
+
 	state, err := p.State(ctx)
 	if err != nil {
-		return nil, err
+		if errdefs.IsNotFound(err) {
+			return nil, err
+		}
+		log.G(ctx).WithError(err).Errorf("get state for %s", p.ID())
 	}
-	var status task.Status
+	status := task.StatusUnknown
 	switch state.Status {
 	case runtime.CreatedStatus:
 		status = task.StatusCreated
@@ -310,7 +316,7 @@ func (l *local) Get(ctx context.Context, r *api.GetRequest, _ ...grpc.CallOption
 			return nil, errdefs.ToGRPC(err)
 		}
 	}
-	t, err := processFromContainerd(ctx, p)
+	t, err := getProcessState(ctx, p)
 	if err != nil {
 		return nil, errdefs.ToGRPC(err)
 	}
@@ -333,7 +339,7 @@ func (l *local) List(ctx context.Context, r *api.ListTasksRequest, _ ...grpc.Cal
 
 func addTasks(ctx context.Context, r *api.ListTasksResponse, tasks []runtime.Task) {
 	for _, t := range tasks {
-		tt, err := processFromContainerd(ctx, t)
+		tt, err := getProcessState(ctx, t)
 		if err != nil {
 			if !errdefs.IsNotFound(err) { // handle race with deletion
 				log.G(ctx).WithError(err).WithField("id", t.ID()).Error("converting task to protobuf")
