|
1 | 1 | package main |
2 | 2 |
|
3 | 3 | import ( |
| 4 | + "archive/zip" |
4 | 5 | "fmt" |
| 6 | + "io" |
5 | 7 | "os" |
6 | 8 | "path/filepath" |
7 | 9 |
|
8 | 10 | "github.com/crytic/cloudexec/pkg/config" |
9 | 11 | "github.com/crytic/cloudexec/pkg/s3" |
10 | 12 | ) |
11 | 13 |
|
12 | | -func UploadDirectoryToSpaces(config config.Config, bucketName string, sourcePath string, destPath string) error { |
13 | | - // Walk the directory and upload files recursively |
14 | | - return filepath.Walk(sourcePath, func(path string, info os.FileInfo, err error) error { |
| 14 | +func UploadDirectoryToSpaces(config config.Config, bucket string, sourcePath string, destPath string) error { |
| 15 | + // Compute the path for the zipped archive of sourcePath |
| 16 | + zipFileName := "input.zip" |
| 17 | + zipFilePath := filepath.Join(os.TempDir(), zipFileName) |
| 18 | + |
| 19 | + // Create a file where we will write the zipped archive |
| 20 | + fmt.Printf("Creating zipped archive at %s\n", zipFilePath) |
| 21 | + zipFile, err := os.Create(zipFilePath) |
| 22 | + if err != nil { |
| 23 | + return err |
| 24 | + } |
| 25 | + defer zipFile.Close() |
| 26 | + |
| 27 | + // Create a new zip writer |
| 28 | + zipWriter := zip.NewWriter(zipFile) |
| 29 | + defer zipWriter.Close() |
| 30 | + |
| 31 | + // Walk the directory and recursively add files to the zipped archive |
| 32 | + err = filepath.Walk(sourcePath, func(path string, info os.FileInfo, err error) error { |
| 33 | + target := path |
15 | 34 | if err != nil { |
16 | 35 | return err |
17 | 36 | } |
18 | 37 |
|
19 | | - if !info.IsDir() { |
20 | | - // Read the file |
21 | | - fileBytes, err := os.ReadFile(path) |
| 38 | + // If it's a symbolic link, resolve the target |
| 39 | + if info.Mode()&os.ModeSymlink == os.ModeSymlink { |
| 40 | + target, err = os.Readlink(path) |
| 41 | + fmt.Printf("Resolved link from %s to %s\n", path, target) |
22 | 42 | if err != nil { |
23 | | - return fmt.Errorf("Failed to read file %s: %w", path, err) |
| 43 | + return err |
24 | 44 | } |
| 45 | + } |
25 | 46 |
|
26 | | - // Compute the destination key (path) in the bucket |
27 | | - relativePath, _ := filepath.Rel(sourcePath, path) |
28 | | - destinationKey := filepath.Join(destPath, "input", relativePath) |
| 47 | + // If this is a subdirectory, make sure the path ends with a trailing slash before we create it |
| 48 | + // See https://pkg.go.dev/archive/zip#Writer.Create for details |
| 49 | + targetInfo, err := os.Stat(target) |
| 50 | + if err != nil { |
| 51 | + return err |
| 52 | + } |
29 | 53 |
|
30 | | - err = s3.PutObject(config, bucketName, destinationKey, fileBytes) |
| 54 | + if targetInfo.IsDir() { |
| 55 | + cleanPath := filepath.Clean(path) + string(filepath.Separator) |
| 56 | + fmt.Printf("Creating directory %s in the zipped archive\n", cleanPath) |
| 57 | + _, err = zipWriter.Create(cleanPath) |
31 | 58 | if err != nil { |
32 | 59 | return err |
33 | 60 | } |
| 61 | + return nil |
| 62 | + } |
| 63 | + |
| 64 | + // Don't recursively add this zipped archive |
| 65 | + if filepath.Base(path) == zipFileName { |
| 66 | + return nil |
| 67 | + } |
| 68 | + |
| 69 | + fmt.Printf("Adding %s to the zipped archive\n", target) |
34 | 70 |
|
35 | | - fmt.Printf("Successfully uploaded %s to %s/%s\n", path, bucketName, destinationKey) |
| 71 | + // Create a new file entry in the zipped archive |
| 72 | + zipFileEntry, err := zipWriter.Create(path) |
| 73 | + if err != nil { |
| 74 | + return err |
36 | 75 | } |
| 76 | + |
| 77 | + // Open the file we're adding to the zipped archive |
| 78 | + file, err := os.Open(target) |
| 79 | + if err != nil { |
| 80 | + return err |
| 81 | + } |
| 82 | + |
| 83 | + // Write this file to the zipped archive |
| 84 | + _, err = io.Copy(zipFileEntry, file) |
| 85 | + if err != nil { |
| 86 | + return err |
| 87 | + } |
| 88 | + |
| 89 | + // Explicitly close the file once we're done to prevent a "too many open files" error |
| 90 | + err = file.Close() |
| 91 | + if err != nil { |
| 92 | + return err |
| 93 | + } |
| 94 | + |
37 | 95 | return nil |
38 | 96 | }) |
| 97 | + if err != nil { |
| 98 | + return err |
| 99 | + } |
| 100 | + fmt.Printf("Successfully added all files from %s to zipped archive at %s\n", sourcePath, zipFilePath) |
| 101 | + |
| 102 | + // Make sure all prior writes are sync'd to the filesystem |
| 103 | + // This is necessary bc we're going to read the file right after writing it |
| 104 | + err = zipWriter.Flush() |
| 105 | + if err != nil { |
| 106 | + return err |
| 107 | + } |
| 108 | + err = zipFile.Sync() |
| 109 | + if err != nil { |
| 110 | + return err |
| 111 | + } |
| 112 | + |
| 113 | + // Manually Closing is necessary to prevent zip file corruption during upload |
| 114 | + err = zipWriter.Close() |
| 115 | + if err != nil { |
| 116 | + return err |
| 117 | + } |
| 118 | + err = zipFile.Close() |
| 119 | + if err != nil { |
| 120 | + return err |
| 121 | + } |
| 122 | + |
| 123 | + // Read the zipped archive |
| 124 | + fileBytes, err := os.ReadFile(zipFilePath) |
| 125 | + if err != nil { |
| 126 | + return fmt.Errorf("Failed to read zipped archive %s: %w", zipFilePath, err) |
| 127 | + } |
| 128 | + if len(fileBytes) == 0 { |
| 129 | + return fmt.Errorf("Failed to read zipped archive at %s: read zero bytes of data", zipFilePath) |
| 130 | + } |
| 131 | + |
| 132 | + // Upload the zipped archive |
| 133 | + destKey := filepath.Join(destPath, "input.zip") |
| 134 | + fmt.Printf("Uploading archive (%v bytes) to %s\n", len(fileBytes), destKey) |
| 135 | + err = s3.PutObject(config, bucket, destKey, fileBytes) |
| 136 | + if err != nil { |
| 137 | + return err |
| 138 | + } |
39 | 139 |
|
| 140 | + return nil |
40 | 141 | } |
0 commit comments