|
| 1 | +/* |
| 2 | + Copyright The containerd Authors. |
| 3 | +
|
| 4 | + Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 | + you may not use this file except in compliance with the License. |
| 6 | + You may obtain a copy of the License at |
| 7 | +
|
| 8 | + http://www.apache.org/licenses/LICENSE-2.0 |
| 9 | +
|
| 10 | + Unless required by applicable law or agreed to in writing, software |
| 11 | + distributed under the License is distributed on an "AS IS" BASIS, |
| 12 | + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 | + See the License for the specific language governing permissions and |
| 14 | + limitations under the License. |
| 15 | +*/ |
| 16 | + |
| 17 | +package integration |
| 18 | + |
| 19 | +import ( |
| 20 | + "context" |
| 21 | + "fmt" |
| 22 | + "os" |
| 23 | + "os/exec" |
| 24 | + "path/filepath" |
| 25 | + "runtime" |
| 26 | + "syscall" |
| 27 | + "testing" |
| 28 | + "time" |
| 29 | + |
| 30 | + cri "github.com/containerd/containerd/integration/cri-api/pkg/apis" |
| 31 | + "github.com/containerd/containerd/integration/remote" |
| 32 | + "github.com/stretchr/testify/assert" |
| 33 | + "github.com/stretchr/testify/require" |
| 34 | + criruntime "k8s.io/cri-api/pkg/apis/runtime/v1" |
| 35 | +) |
| 36 | + |
| 37 | +func newPodTCtx(t *testing.T, rSvc cri.RuntimeService, |
| 38 | + name, ns string, opts ...PodSandboxOpts) *podTCtx { |
| 39 | + |
| 40 | + t.Logf("Run a sandbox %s in namespace %s", name, ns) |
| 41 | + sbConfig := PodSandboxConfig(name, ns, opts...) |
| 42 | + sbID, err := rSvc.RunPodSandbox(sbConfig, "") |
| 43 | + require.NoError(t, err) |
| 44 | + |
| 45 | + return &podTCtx{ |
| 46 | + t: t, |
| 47 | + id: sbID, |
| 48 | + name: name, |
| 49 | + ns: ns, |
| 50 | + cfg: sbConfig, |
| 51 | + rSvc: rSvc, |
| 52 | + } |
| 53 | +} |
| 54 | + |
| 55 | +// podTCtx is used to construct pod. |
| 56 | +type podTCtx struct { |
| 57 | + t *testing.T |
| 58 | + id string |
| 59 | + name string |
| 60 | + ns string |
| 61 | + cfg *criruntime.PodSandboxConfig |
| 62 | + rSvc cri.RuntimeService |
| 63 | +} |
| 64 | + |
| 65 | +// createContainer creates a container in that pod. |
| 66 | +func (pCtx *podTCtx) createContainer(name, imageRef string, wantedState criruntime.ContainerState, opts ...ContainerOpts) string { |
| 67 | + t := pCtx.t |
| 68 | + |
| 69 | + t.Logf("Create a container %s (wantedState: %s) in pod %s", name, wantedState, pCtx.name) |
| 70 | + cfg := ContainerConfig(name, imageRef, opts...) |
| 71 | + cnID, err := pCtx.rSvc.CreateContainer(pCtx.id, cfg, pCtx.cfg) |
| 72 | + require.NoError(t, err) |
| 73 | + |
| 74 | + switch wantedState { |
| 75 | + case criruntime.ContainerState_CONTAINER_CREATED: |
| 76 | + // no-op |
| 77 | + case criruntime.ContainerState_CONTAINER_RUNNING: |
| 78 | + require.NoError(t, pCtx.rSvc.StartContainer(cnID)) |
| 79 | + case criruntime.ContainerState_CONTAINER_EXITED: |
| 80 | + require.NoError(t, pCtx.rSvc.StartContainer(cnID)) |
| 81 | + require.NoError(t, pCtx.rSvc.StopContainer(cnID, 0)) |
| 82 | + default: |
| 83 | + t.Fatalf("unsupport state %s", wantedState) |
| 84 | + } |
| 85 | + return cnID |
| 86 | +} |
| 87 | + |
| 88 | +// pullImagesByCRI pulls images by CRI. |
| 89 | +func pullImagesByCRI(t *testing.T, svc cri.ImageManagerService, images ...string) []string { |
| 90 | + expectedRefs := make([]string, 0, len(images)) |
| 91 | + |
| 92 | + for _, image := range images { |
| 93 | + t.Logf("Pulling image %q", image) |
| 94 | + imgRef, err := svc.PullImage(&criruntime.ImageSpec{Image: image}, nil, nil) |
| 95 | + require.NoError(t, err) |
| 96 | + expectedRefs = append(expectedRefs, imgRef) |
| 97 | + } |
| 98 | + return expectedRefs |
| 99 | +} |
| 100 | + |
| 101 | +// cleanupPods deletes all the pods based on the cri.RuntimeService connection. |
| 102 | +func cleanupPods(t *testing.T, criRuntimeService cri.RuntimeService) { |
| 103 | + pods, err := criRuntimeService.ListPodSandbox(nil) |
| 104 | + require.NoError(t, err) |
| 105 | + |
| 106 | + for _, pod := range pods { |
| 107 | + assert.NoError(t, criRuntimeService.StopPodSandbox(pod.Id)) |
| 108 | + assert.NoError(t, criRuntimeService.RemovePodSandbox(pod.Id)) |
| 109 | + } |
| 110 | +} |
| 111 | + |
| 112 | +// criRuntimeService returns cri.RuntimeService based on the grpc address. |
| 113 | +func (p *ctrdProc) criRuntimeService(t *testing.T) cri.RuntimeService { |
| 114 | + service, err := remote.NewRuntimeService(p.grpcAddress(), 1*time.Minute) |
| 115 | + require.NoError(t, err) |
| 116 | + return service |
| 117 | +} |
| 118 | + |
| 119 | +// criImageService returns cri.ImageManagerService based on the grpc address. |
| 120 | +func (p *ctrdProc) criImageService(t *testing.T) cri.ImageManagerService { |
| 121 | + service, err := remote.NewImageService(p.grpcAddress(), 1*time.Minute) |
| 122 | + require.NoError(t, err) |
| 123 | + return service |
| 124 | +} |
| 125 | + |
| 126 | +// newCtrdProc is to start containerd process. |
| 127 | +func newCtrdProc(t *testing.T, ctrdBin string, ctrdWorkDir string) *ctrdProc { |
| 128 | + p := &ctrdProc{workDir: ctrdWorkDir} |
| 129 | + |
| 130 | + var args []string |
| 131 | + args = append(args, "--root", p.rootPath()) |
| 132 | + args = append(args, "--state", p.statePath()) |
| 133 | + args = append(args, "--address", p.grpcAddress()) |
| 134 | + args = append(args, "--config", p.configPath()) |
| 135 | + args = append(args, "--log-level", "debug") |
| 136 | + |
| 137 | + f, err := os.OpenFile(p.logPath(), os.O_RDWR|os.O_CREATE|os.O_APPEND, 0600) |
| 138 | + require.NoError(t, err, "open log file %s", p.logPath()) |
| 139 | + t.Cleanup(func() { f.Close() }) |
| 140 | + |
| 141 | + cmd := exec.Command(ctrdBin, args...) |
| 142 | + cmd.Stdout = f |
| 143 | + cmd.Stderr = f |
| 144 | + cmd.SysProcAttr = &syscall.SysProcAttr{Pdeathsig: syscall.SIGKILL} |
| 145 | + |
| 146 | + p.cmd = cmd |
| 147 | + p.waitBlock = make(chan struct{}) |
| 148 | + go func() { |
| 149 | + // The PDeathSIG is based on the thread which forks the child |
| 150 | + // process instead of the leader of thread group. Lock the |
| 151 | + // thread just in case that the thread exits and causes unexpected |
| 152 | + // SIGKILL to containerd. |
| 153 | + runtime.LockOSThread() |
| 154 | + defer runtime.UnlockOSThread() |
| 155 | + |
| 156 | + defer close(p.waitBlock) |
| 157 | + |
| 158 | + require.NoError(t, p.cmd.Start(), "start containerd(%s)", ctrdBin) |
| 159 | + assert.NoError(t, p.cmd.Wait()) |
| 160 | + }() |
| 161 | + return p |
| 162 | +} |
| 163 | + |
| 164 | +// ctrdProc is used to control the containerd process's lifecycle. |
| 165 | +type ctrdProc struct { |
| 166 | + // workDir has the following layout: |
| 167 | + // |
| 168 | + // - root (dir) |
| 169 | + // - state (dir) |
| 170 | + // - containerd.sock (sock file) |
| 171 | + // - config.toml (toml file, required) |
| 172 | + // - containerd.log (log file, always open with O_APPEND) |
| 173 | + workDir string |
| 174 | + cmd *exec.Cmd |
| 175 | + waitBlock chan struct{} |
| 176 | +} |
| 177 | + |
| 178 | +// kill is to send the signal to containerd process. |
| 179 | +func (p *ctrdProc) kill(sig syscall.Signal) error { |
| 180 | + return p.cmd.Process.Signal(sig) |
| 181 | +} |
| 182 | + |
| 183 | +// wait is to wait for exit event of containerd process. |
| 184 | +func (p *ctrdProc) wait(to time.Duration) error { |
| 185 | + var ctx = context.Background() |
| 186 | + var cancel context.CancelFunc |
| 187 | + |
| 188 | + if to > 0 { |
| 189 | + ctx, cancel = context.WithTimeout(ctx, to) |
| 190 | + defer cancel() |
| 191 | + } |
| 192 | + |
| 193 | + select { |
| 194 | + case <-ctx.Done(): |
| 195 | + return ctx.Err() |
| 196 | + case <-p.waitBlock: |
| 197 | + return nil |
| 198 | + } |
| 199 | +} |
| 200 | + |
| 201 | +// grpcAddress is to return containerd's address. |
| 202 | +func (p *ctrdProc) grpcAddress() string { |
| 203 | + return filepath.Join(p.workDir, "containerd.sock") |
| 204 | +} |
| 205 | + |
| 206 | +// configPath is to return containerd's config file. |
| 207 | +func (p *ctrdProc) configPath() string { |
| 208 | + return filepath.Join(p.workDir, "config.toml") |
| 209 | +} |
| 210 | + |
| 211 | +// rootPath is to return containerd's root path. |
| 212 | +func (p *ctrdProc) rootPath() string { |
| 213 | + return filepath.Join(p.workDir, "root") |
| 214 | +} |
| 215 | + |
| 216 | +// statePath is to return containerd's state path. |
| 217 | +func (p *ctrdProc) statePath() string { |
| 218 | + return filepath.Join(p.workDir, "state") |
| 219 | +} |
| 220 | + |
| 221 | +// logPath is to return containerd's log path. |
| 222 | +func (p *ctrdProc) logPath() string { |
| 223 | + return filepath.Join(p.workDir, "containerd.log") |
| 224 | +} |
| 225 | + |
| 226 | +// isReady checks the containerd is ready or not. |
| 227 | +func (p *ctrdProc) isReady() error { |
| 228 | + var ( |
| 229 | + service cri.RuntimeService |
| 230 | + err error |
| 231 | + ticker = time.NewTicker(1 * time.Second) |
| 232 | + ctx, cancel = context.WithTimeout(context.Background(), 2*time.Minute) |
| 233 | + ) |
| 234 | + defer func() { |
| 235 | + cancel() |
| 236 | + ticker.Stop() |
| 237 | + }() |
| 238 | + |
| 239 | + for { |
| 240 | + select { |
| 241 | + case <-ticker.C: |
| 242 | + service, err = remote.NewRuntimeService(p.grpcAddress(), 5*time.Second) |
| 243 | + if err != nil { |
| 244 | + continue |
| 245 | + } |
| 246 | + |
| 247 | + if _, err = service.Status(); err != nil { |
| 248 | + continue |
| 249 | + } |
| 250 | + return nil |
| 251 | + case <-ctx.Done(): |
| 252 | + return fmt.Errorf("context deadline exceeded: %w", err) |
| 253 | + } |
| 254 | + } |
| 255 | +} |
0 commit comments