Skip to content

Commit 8360f71

Browse files
feat(k3s): pull image opts (#3716)
* fix(k3s): add better error handling to LoadImages * fix(k3s): don't attempt to load all platforms when using LoadImages deprecate LoadImagesWithOpts with a new LoadImagesWithPlatform * feat!: add PullImageWithOpts and PullDockerImageWithPlatform Builds on Laurent's k3s cross-platform image loading fixes. Co-authored-by: Laurent Goderre <[email protected]> * fix(k3s): align tests and document image loading APIs Update platform mismatch tests for current Docker error messages, clarify LoadImagesWithOpts deprecation, and document LoadImages, LoadImagesWithPlatform, and the pull-then-load workflow in the k3s module. * refactor(k3s): fix LoadImagesWithOpts and keep it as extensible API Align containerd import with SaveDockerImageWithPlatforms instead of --all-platforms, add ResolveSaveImageOptions, and make LoadImagesWithPlatform a convenience wrapper. Signed-off-by: Enin <[email protected]> * docs(k3s): document image loading functional options Signed-off-by: Enin <[email protected]> * docs(k3s): update docs Signed-off-by: Enin <[email protected]> --------- Signed-off-by: Enin <[email protected]> Co-authored-by: Laurent Goderre <[email protected]>
1 parent b5e7022 commit 8360f71

8 files changed

Lines changed: 404 additions & 31 deletions

File tree

‎docker.go‎

Lines changed: 18 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1895,6 +1895,7 @@ func (p *DockerProvider) SaveImagesWithOpts(ctx context.Context, output string,
18951895
func SaveDockerImageWithPlatforms(platforms ...specs.Platform) SaveImageOption {
18961896
return func(opts *saveImageOptions) error {
18971897
opts.dockerSaveOpts = append(opts.dockerSaveOpts, client.ImageSaveWithPlatforms(platforms...))
1898+
opts.platforms = append(opts.platforms, platforms...)
18981899

18991900
return nil
19001901
}
@@ -1905,17 +1906,25 @@ func (p *DockerProvider) PullImage(ctx context.Context, img string) error {
19051906
return p.attemptToPullImage(ctx, img, client.ImagePullOptions{})
19061907
}
19071908

1908-
// PullImageWithPlatform pulls an image from the registry for the given platform (e.g. "linux/amd64").
1909-
func (p *DockerProvider) PullImageWithPlatform(ctx context.Context, img, platform string) error {
1910-
pullOpt := client.ImagePullOptions{}
1911-
if platform != "" {
1912-
pf, err := platforms.Parse(platform)
1913-
if err != nil {
1914-
return fmt.Errorf("invalid platform %s: %w", platform, err)
1909+
// PullImageWithOpts pulls image from registry, passing options to the provider.
1910+
func (p *DockerProvider) PullImageWithOpts(ctx context.Context, img string, opts ...PullImageOption) error {
1911+
pullOpts := pullImageOptions{}
1912+
1913+
for _, opt := range opts {
1914+
if err := opt(&pullOpts); err != nil {
1915+
return fmt.Errorf("applying pull image option: %w", err)
19151916
}
1916-
pullOpt.Platforms = append(pullOpt.Platforms, pf)
19171917
}
1918-
return p.attemptToPullImage(ctx, img, pullOpt)
1918+
1919+
return p.attemptToPullImage(ctx, img, pullOpts.dockerPullOpts)
1920+
}
1921+
1922+
func PullDockerImageWithPlatform(platform specs.Platform) PullImageOption {
1923+
return func(opts *pullImageOptions) error {
1924+
opts.dockerPullOpts.Platforms = append(opts.dockerPullOpts.Platforms, platform)
1925+
1926+
return nil
1927+
}
19191928
}
19201929

19211930
var permanentClientErrors = []func(error) bool{

‎docker_test.go‎

Lines changed: 55 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -19,8 +19,10 @@ import (
1919
"time"
2020

2121
"github.com/containerd/errdefs"
22+
"github.com/containerd/platforms"
2223
"github.com/moby/moby/api/types/container"
2324
"github.com/moby/moby/client"
25+
specs "github.com/opencontainers/image-spec/specs-go/v1"
2426
"github.com/stretchr/testify/require"
2527

2628
"github.com/testcontainers/testcontainers-go/internal/core"
@@ -1654,6 +1656,7 @@ type errMockCli struct {
16541656
imageBuildCount int
16551657
containerListCount int
16561658
imagePullCount int
1659+
imagePullOpts []client.ImagePullOptions
16571660
}
16581661

16591662
type fakeStreamResult struct {
@@ -1674,8 +1677,9 @@ func (f *errMockCli) ContainerList(_ context.Context, _ client.ContainerListOpti
16741677
return client.ContainerListResult{Items: []container.Summary{{}}}, f.err
16751678
}
16761679

1677-
func (f *errMockCli) ImagePull(_ context.Context, _ string, _ client.ImagePullOptions) (client.ImagePullResponse, error) {
1680+
func (f *errMockCli) ImagePull(_ context.Context, _ string, opts client.ImagePullOptions) (client.ImagePullResponse, error) {
16781681
f.imagePullCount++
1682+
f.imagePullOpts = append(f.imagePullOpts, opts)
16791683
return fakeStreamResult{ReadCloser: io.NopCloser(&bytes.Buffer{})}, f.err
16801684
}
16811685

@@ -1870,14 +1874,58 @@ func TestDockerProvider_attemptToPullImage_retries(t *testing.T) {
18701874
}
18711875
}
18721876

1873-
func TestDockerProvider_PullImageWithPlatform_invalidPlatform(t *testing.T) {
1874-
p, err := NewDockerProvider()
1877+
func TestPullDockerImageWithPlatform(t *testing.T) {
1878+
t.Run("single platform", func(t *testing.T) {
1879+
platform, err := platforms.Parse("linux/amd64")
1880+
require.NoError(t, err)
1881+
1882+
opts := pullImageOptions{}
1883+
err = PullDockerImageWithPlatform(platform)(&opts)
1884+
require.NoError(t, err)
1885+
require.Equal(t, []specs.Platform{platform}, opts.dockerPullOpts.Platforms)
1886+
})
1887+
1888+
t.Run("multiple platforms via separate options", func(t *testing.T) {
1889+
amd64, err := platforms.Parse("linux/amd64")
1890+
require.NoError(t, err)
1891+
arm64, err := platforms.Parse("linux/arm64")
1892+
require.NoError(t, err)
1893+
1894+
opts := pullImageOptions{}
1895+
require.NoError(t, PullDockerImageWithPlatform(amd64)(&opts))
1896+
require.NoError(t, PullDockerImageWithPlatform(arm64)(&opts))
1897+
require.Equal(t, []specs.Platform{amd64, arm64}, opts.dockerPullOpts.Platforms)
1898+
})
1899+
1900+
t.Run("invalid platform", func(t *testing.T) {
1901+
_, err := platforms.Parse("not-a-valid-platform")
1902+
require.Error(t, err)
1903+
})
1904+
}
1905+
1906+
func TestDockerProvider_PullImageWithOpts_separatePullsWithDifferentPlatforms(t *testing.T) {
1907+
amd64, err := platforms.Parse("linux/amd64")
1908+
require.NoError(t, err)
1909+
arm64, err := platforms.Parse("linux/arm64")
18751910
require.NoError(t, err)
1876-
defer p.Close()
18771911

1878-
err = p.PullImageWithPlatform(context.Background(), "redis:latest", "not-a-valid-platform")
1879-
require.Error(t, err)
1880-
require.Contains(t, err.Error(), "invalid platform")
1912+
mock := &errMockCli{}
1913+
p := &DockerProvider{
1914+
DockerProviderOptions: &DockerProviderOptions{
1915+
GenericProviderOptions: &GenericProviderOptions{
1916+
Logger: log.Default(),
1917+
},
1918+
},
1919+
client: mock,
1920+
}
1921+
1922+
ctx := context.Background()
1923+
require.NoError(t, p.PullImageWithOpts(ctx, "nginx:mainline", PullDockerImageWithPlatform(amd64)))
1924+
require.NoError(t, p.PullImageWithOpts(ctx, "nginx:mainline", PullDockerImageWithPlatform(arm64)))
1925+
1926+
require.Equal(t, 2, mock.imagePullCount)
1927+
require.Equal(t, []specs.Platform{amd64}, mock.imagePullOpts[0].Platforms)
1928+
require.Equal(t, []specs.Platform{arm64}, mock.imagePullOpts[1].Platforms)
18811929
}
18821930

18831931
func TestCustomPrefixTrailingSlashIsProperlyRemovedIfPresent(t *testing.T) {

‎docs/modules/k3s.md‎

Lines changed: 104 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -92,8 +92,109 @@ to the Kubernetes Rest Client API using a Kubernetes client. It'll be returned i
9292

9393
- Since <a href="https://github.com/testcontainers/testcontainers-go/releases/tag/v0.25.0"><span class="tc-version">:material-tag: v0.25.0</span></a>
9494

95-
The `LoadImages` method loads a list of images into the kubernetes cluster and makes them available to pods.
95+
The `LoadImages` method imports images from the local Docker daemon into the k3s cluster and makes them available to pods.
9696

97-
This is useful for testing images generated locally without having to push them to a public docker registry or having to configure `k3s` to [use a private registry](https://docs.k3s.io/installation/private-registry).
97+
```golang
98+
func (c *K3sContainer) LoadImages(ctx context.Context, images ...string) error
99+
```
100+
101+
This is useful for testing images built locally without pushing them to a registry or configuring k3s to [use a private registry](https://docs.k3s.io/installation/private-registry).
102+
103+
Images must already be present on the Docker host running the test. [DockerProvider.PullImage](https://pkg.go.dev/github.com/testcontainers/testcontainers-go#DockerProvider.PullImage) is enough for single-architecture image references. When you need a specific OCI platform, use [DockerProvider.PullImageWithOpts](https://pkg.go.dev/github.com/testcontainers/testcontainers-go#DockerProvider.PullImageWithOpts) with [PullDockerImageWithPlatform](https://pkg.go.dev/github.com/testcontainers/testcontainers-go#PullDockerImageWithPlatform).
104+
105+
`LoadImages` delegates to `LoadImagesWithOpts` without save options. It works best with single-architecture image references (for example `amd64/nginx`). For multi-architecture images, use `LoadImagesWithPlatform` or `LoadImagesWithOpts`.
106+
107+
When creating pods that use loaded images, set `imagePullPolicy: Never` so Kubernetes uses the imported image instead of pulling from a registry.
108+
109+
Example:
110+
111+
```golang
112+
provider, err := testcontainers.ProviderDocker.GetProvider()
113+
if err != nil {
114+
// handle error
115+
}
116+
117+
if err := provider.PullImage(ctx, "amd64/nginx"); err != nil {
118+
// handle error
119+
}
120+
121+
if err := k3sContainer.LoadImages(ctx, "amd64/nginx"); err != nil {
122+
// handle error
123+
}
124+
```
125+
126+
#### LoadImagesWithPlatform
127+
128+
The `LoadImagesWithPlatform` method is a convenience wrapper around `LoadImagesWithOpts` that exports and imports an image for a specific OCI platform.
129+
130+
```golang
131+
func (c *K3sContainer) LoadImagesWithPlatform(ctx context.Context, images []string, platform *v1.Platform) error
132+
```
133+
134+
When `platform` is `nil`, behaviour matches `LoadImages`. When `platform` is set, the image is exported and imported for that OCI platform.
98135

99-
The images must be already present in the node running the test. [DockerProvider](https://pkg.go.dev/github.com/testcontainers/testcontainers-go#DockerProvider) offers a method for pulling images, which can be used from the test code to ensure the image is present locally before loading them to the cluster.
136+
Use this method on multi-architecture hosts or when loading multi-architecture image tags. Pull the same platform into Docker first, then load it into k3s:
137+
138+
```golang
139+
hostPlatform := platforms.DefaultSpec()
140+
hostPlatform.OS = "linux"
141+
142+
provider, err := testcontainers.ProviderDocker.GetProvider()
143+
if err != nil {
144+
// handle error
145+
}
146+
147+
if err := provider.PullImageWithOpts(
148+
ctx,
149+
"amd64/nginx",
150+
testcontainers.PullDockerImageWithPlatform(hostPlatform),
151+
); err != nil {
152+
// handle error
153+
}
154+
155+
if err := k3sContainer.LoadImagesWithPlatform(ctx, []string{"amd64/nginx"}, &hostPlatform); err != nil {
156+
// handle error
157+
}
158+
```
159+
160+
#### LoadImagesWithOpts
161+
162+
- Since <a href="https://github.com/testcontainers/testcontainers-go/releases/tag/v0.25.0"><span class="tc-version">:material-tag: v0.25.0</span></a>
163+
164+
The `LoadImagesWithOpts` method imports local images into the k3s cluster, passing [SaveImageOption](https://pkg.go.dev/github.com/testcontainers/testcontainers-go#SaveImageOption) values through to `docker save`.
165+
166+
```golang
167+
func (c *K3sContainer) LoadImagesWithOpts(ctx context.Context, images []string, opts ...testcontainers.SaveImageOption) error
168+
```
169+
170+
When [SaveDockerImageWithPlatforms](https://pkg.go.dev/github.com/testcontainers/testcontainers-go#SaveDockerImageWithPlatforms) is passed, containerd import uses the same platform. Without platform options, import does not use `--all-platforms`.
171+
172+
For multiple images on different architectures, call `LoadImagesWithOpts` once per image and platform.
173+
174+
Example with platform:
175+
176+
```golang
177+
hostPlatform := platforms.DefaultSpec()
178+
hostPlatform.OS = "linux"
179+
180+
provider, err := testcontainers.ProviderDocker.GetProvider()
181+
if err != nil {
182+
// handle error
183+
}
184+
185+
if err := provider.PullImageWithOpts(
186+
ctx,
187+
"amd64/nginx",
188+
testcontainers.PullDockerImageWithPlatform(hostPlatform),
189+
); err != nil {
190+
// handle error
191+
}
192+
193+
if err := k3sContainer.LoadImagesWithOpts(
194+
ctx,
195+
[]string{"amd64/nginx"},
196+
testcontainers.SaveDockerImageWithPlatforms(hostPlatform),
197+
); err != nil {
198+
// handle error
199+
}
200+
```

‎image.go‎

Lines changed: 31 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,10 @@ package testcontainers
22

33
import (
44
"context"
5+
"fmt"
56

67
"github.com/moby/moby/client"
8+
specs "github.com/opencontainers/image-spec/specs-go/v1"
79
)
810

911
// ImageInfo represents summary information of an image
@@ -14,15 +16,43 @@ type ImageInfo struct {
1416

1517
type saveImageOptions struct {
1618
dockerSaveOpts []client.ImageSaveOption
19+
platforms []specs.Platform
1720
}
1821

1922
type SaveImageOption func(*saveImageOptions) error
2023

24+
type pullImageOptions struct {
25+
dockerPullOpts client.ImagePullOptions
26+
}
27+
28+
type PullImageOption func(*pullImageOptions) error
29+
30+
// ResolveSaveImageOptions applies save image options and returns the platform to
31+
// use for a coordinated containerd import, if exactly one platform was requested.
32+
func ResolveSaveImageOptions(opts ...SaveImageOption) (*specs.Platform, error) {
33+
saveOpts := saveImageOptions{}
34+
35+
for _, opt := range opts {
36+
if err := opt(&saveOpts); err != nil {
37+
return nil, fmt.Errorf("applying save image option: %w", err)
38+
}
39+
}
40+
41+
switch len(saveOpts.platforms) {
42+
case 0:
43+
return nil, nil
44+
case 1:
45+
return &saveOpts.platforms[0], nil
46+
default:
47+
return nil, fmt.Errorf("at most one platform is supported, got %d", len(saveOpts.platforms))
48+
}
49+
}
50+
2151
// ImageProvider allows manipulating images
2252
type ImageProvider interface {
2353
ListImages(context.Context) ([]ImageInfo, error)
2454
SaveImages(context.Context, string, ...string) error
2555
SaveImagesWithOpts(context.Context, string, []string, ...SaveImageOption) error
2656
PullImage(context.Context, string) error
27-
PullImageWithPlatform(context.Context, string, string) error
57+
PullImageWithOpts(context.Context, string, ...PullImageOption) error
2858
}

‎image_test.go‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -109,3 +109,34 @@ func TestSaveImagesWithOpts(t *testing.T) {
109109

110110
require.NotZerof(t, info.Size(), "output file is empty")
111111
}
112+
113+
func TestResolveSaveImageOptions(t *testing.T) {
114+
t.Run("no options", func(t *testing.T) {
115+
platform, err := ResolveSaveImageOptions()
116+
require.NoError(t, err)
117+
require.Nil(t, platform)
118+
})
119+
120+
t.Run("single platform", func(t *testing.T) {
121+
p, err := platforms.Parse("linux/amd64")
122+
require.NoError(t, err)
123+
124+
platform, err := ResolveSaveImageOptions(SaveDockerImageWithPlatforms(p))
125+
require.NoError(t, err)
126+
require.NotNil(t, platform)
127+
require.Equal(t, p, *platform)
128+
})
129+
130+
t.Run("multiple platforms", func(t *testing.T) {
131+
amd64, err := platforms.Parse("linux/amd64")
132+
require.NoError(t, err)
133+
arm64, err := platforms.Parse("linux/arm64")
134+
require.NoError(t, err)
135+
136+
_, err = ResolveSaveImageOptions(
137+
SaveDockerImageWithPlatforms(amd64),
138+
SaveDockerImageWithPlatforms(arm64),
139+
)
140+
require.Error(t, err)
141+
})
142+
}

‎modules/k3s/go.mod‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,9 @@ go 1.25.0
55
toolchain go1.25.9
66

77
require (
8+
github.com/containerd/platforms v0.2.1
89
github.com/moby/moby/api v1.54.2
10+
github.com/opencontainers/image-spec v1.1.1
911
github.com/stretchr/testify v1.11.1
1012
github.com/testcontainers/testcontainers-go v0.42.0
1113
go.yaml.in/yaml/v3 v3.0.4
@@ -23,7 +25,6 @@ require (
2325
github.com/containerd/errdefs v1.0.0 // indirect
2426
github.com/containerd/errdefs/pkg v0.3.0 // indirect
2527
github.com/containerd/log v0.1.0 // indirect
26-
github.com/containerd/platforms v0.2.1 // indirect
2728
github.com/cpuguy83/dockercfg v0.3.2 // indirect
2829
github.com/davecgh/go-spew v1.1.1 // indirect
2930
github.com/distribution/reference v0.6.0 // indirect
@@ -62,7 +63,6 @@ require (
6263
github.com/modern-go/reflect2 v1.0.2 // indirect
6364
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
6465
github.com/opencontainers/go-digest v1.0.0 // indirect
65-
github.com/opencontainers/image-spec v1.1.1 // indirect
6666
github.com/pmezard/go-difflib v1.0.0 // indirect
6767
github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect
6868
github.com/shirou/gopsutil/v4 v4.26.5 // indirect

0 commit comments

Comments
 (0)