'docker push' buffers filesystem archives on disk instead of memory.

Solomon Hykes · Solomon Hykes · commit baacae8345fe · 2013-04-21T14:23:55.000-07:00
diff --git a/archive.go b/archive.go
@@ -4,6 +4,7 @@ import (
 	"errors"
 	"io"
 	"io/ioutil"
+	"os"
 	"os/exec"
 )
 
@@ -86,3 +87,38 @@ func CmdStream(cmd *exec.Cmd) (io.Reader, error) {
 	}
 	return pipeR, nil
 }
+
+// NewTempArchive reads the content of src into a temporary file, and returns the contents
+// of that file as an archive. The archive can only be read once - as soon as reading completes,
+// the file will be deleted.
+func NewTempArchive(src Archive, dir string) (*TempArchive, error) {
+	f, err := ioutil.TempFile(dir, "")
+	if err != nil {
+		return nil, err
+	}
+	if _, err := io.Copy(f, src); err != nil {
+		return nil, err
+	}
+	if _, err := f.Seek(0, 0); err != nil {
+		return nil, err
+	}
+	st, err := f.Stat()
+	if err != nil {
+		return nil, err
+	}
+	size := st.Size()
+	return &TempArchive{f, size}, nil
+}
+
+type TempArchive struct {
+	*os.File
+	Size int64 // Pre-computed from Stat().Size() as a convenience
+}
+
+func (archive *TempArchive) Read(data []byte) (int, error) {
+	n, err := archive.File.Read(data)
+	if err != nil {
+		os.Remove(archive.File.Name())
+	}
+	return n, err
+}
diff --git a/graph.go b/graph.go
@@ -129,12 +129,30 @@ func (graph *Graph) Register(layerData Archive, img *Image) error {
 	return nil
 }
 
+// TempLayerArchive creates a temporary archive of the given image's filesystem layer.
+//   The archive is stored on disk and will be automatically deleted as soon as has been read.
+func (graph *Graph) TempLayerArchive(id string, compression Compression) (*TempArchive, error) {
+	image, err := graph.Get(id)
+	if err != nil {
+		return nil, err
+	}
+	tmp, err := graph.tmp()
+	if err != nil {
+		return nil, err
+	}
+	archive, err := image.TarLayer(compression)
+	if err != nil {
+		return nil, err
+	}
+	return NewTempArchive(archive, tmp.Root)
+}
+
 // Mktemp creates a temporary sub-directory inside the graph's filesystem.
 func (graph *Graph) Mktemp(id string) (string, error) {
 	if id == "" {
 		id = GenerateId()
 	}
-	tmp, err := NewGraph(path.Join(graph.Root, ":tmp:"))
+	tmp, err := graph.tmp()
 	if err != nil {
 		return "", fmt.Errorf("Couldn't create temp: %s", err)
 	}
@@ -144,6 +162,10 @@ func (graph *Graph) Mktemp(id string) (string, error) {
 	return tmp.imageRoot(id), nil
 }
 
+func (graph *Graph) tmp() (*Graph, error) {
+	return NewGraph(path.Join(graph.Root, ":tmp:"))
+}
+
 // Check if given error is "not empty".
 // Note: this is the way golang does it internally with os.IsNotExists.
 func isNotEmpty(err error) bool {
diff --git a/image.go b/image.go
@@ -110,6 +110,15 @@ func MountAUFS(ro []string, rw string, target string) error {
 	return nil
 }
 
+// TarLayer returns a tar archive of the image's filesystem layer.
+func (image *Image) TarLayer(compression Compression) (Archive, error) {
+	layerPath, err := image.layer()
+	if err != nil {
+		return nil, err
+	}
+	return Tar(layerPath, compression)
+}
+
 func (image *Image) Mount(root, rw string) error {
 	if mounted, err := Mounted(root); err != nil {
 		return err
diff --git a/registry.go b/registry.go
@@ -7,6 +7,7 @@ import (
 	"io"
 	"io/ioutil"
 	"net/http"
+	"os"
 	"path"
 	"strings"
 )
@@ -269,24 +270,20 @@ func (graph *Graph) PushImage(stdout io.Writer, imgOrig *Image, authConfig *auth
 			return fmt.Errorf("Failed to retrieve layer upload location: %s", err)
 		}
 
-		// FIXME: Don't do this :D. Check the S3 requierement and implement chunks of 5MB
-		// FIXME2: I won't stress it enough, DON'T DO THIS! very high priority
-		layerData2, err := Tar(path.Join(graph.Root, img.Id, "layer"), Xz)
-		tmp, err := ioutil.ReadAll(layerData2)
+		// FIXME: stream the archive directly to the registry instead of buffering it on disk. This requires either:
+		//	a) Implementing S3's proprietary streaming logic, or
+		//	b) Stream directly to the registry instead of S3.
+		// I prefer option b. because it doesn't lock us into a proprietary cloud service.
+		tmpLayer, err := graph.TempLayerArchive(img.Id, Xz)
 		if err != nil {
 			return err
 		}
-		layerLength := len(tmp)
-
-		layerData, err := Tar(path.Join(graph.Root, img.Id, "layer"), Xz)
-		if err != nil {
-			return fmt.Errorf("Failed to generate layer archive: %s", err)
-		}
-		req3, err := http.NewRequest("PUT", url.String(), ProgressReader(layerData.(io.ReadCloser), layerLength, stdout))
+		defer os.Remove(tmpLayer.Name())
+		req3, err := http.NewRequest("PUT", url.String(), ProgressReader(tmpLayer, int(tmpLayer.Size), stdout))
 		if err != nil {
 			return err
 		}
-		req3.ContentLength = int64(layerLength)
+		req3.ContentLength = int64(tmpLayer.Size)
 
 		req3.TransferEncoding = []string{"none"}
 		res3, err := client.Do(req3)
