Fix pull fails on unexpected EOF

ktock · Kern Walster · commit aa7c9d9daf58 · 2021-10-12T18:07:52.000Z
Currently, containerd doesn't restart pull when it encounters unexpected EOF of blob strem withtout error codes. There are cases where this lead to pull failure. This commit tries to fix this issue. Signed-off-by: Kohei Tokunaga <ktokunaga.mail@gmail.com> (cherry picked from commit 7bc5aa7) Signed-off-by: Kern Walster <walster@amazon.com>
diff --git a/remotes/docker/httpreadseeker.go b/remotes/docker/httpreadseeker.go
@@ -26,12 +26,16 @@ import (
 	"github.com/pkg/errors"
 )
 
+const maxRetry = 3
+
 type httpReadSeeker struct {
 	size   int64
 	offset int64
 	rc     io.ReadCloser
 	open   func(offset int64) (io.ReadCloser, error)
 	closed bool
+
+	errsWithNoProgress int
 }
 
 func newHTTPReadSeeker(size int64, open func(offset int64) (io.ReadCloser, error)) (io.ReadCloser, error) {
@@ -53,6 +57,27 @@ func (hrs *httpReadSeeker) Read(p []byte) (n int, err error) {
 
 	n, err = rd.Read(p)
 	hrs.offset += int64(n)
+	if n > 0 || err == nil {
+		hrs.errsWithNoProgress = 0
+	}
+	if err == io.ErrUnexpectedEOF {
+		// connection closed unexpectedly. try reconnecting.
+		if n == 0 {
+			hrs.errsWithNoProgress++
+			if hrs.errsWithNoProgress > maxRetry {
+				return // too many retries for this offset with no progress
+			}
+		}
+		if hrs.rc != nil {
+			if clsErr := hrs.rc.Close(); clsErr != nil {
+				log.L.WithError(clsErr).Errorf("httpReadSeeker: failed to close ReadCloser")
+			}
+			hrs.rc = nil
+		}
+		if _, err2 := hrs.reader(); err2 == nil {
+			return n, nil
+		}
+	}
 	return
 }
 
