Skip to content

Commit 51aee6b

Browse files
anmaxvlanmaxvl
authored
Add new gcs hooks, add expected mounts to security policy (#1258)
Introduce a new `wait-paths` binary, which polls file system until requested paths are available or a timeout is reached. Security policy has been updated to have `ExpectedMounts` entries, which will be used in conjunction with "wait-paths" hook for synchronization purposes. Refactor oci-hook logic into its own internal package and update existing code to use that package. Copy runc HookName and constants definitions to break dependency on runc Introduce `ExpectedMounts` as part of security policy language and the logic to enforce the policy, which resolves the expected mounts in the UVM and adds a wait-paths hook to the spec. Add positive and negative CRI tests. Signed-off-by: Maksim An <[email protected]>
1 parent a2ed14c commit 51aee6b

44 files changed

Lines changed: 944 additions & 227 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎Makefile‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,14 +32,15 @@ clean:
3232
test:
3333
cd $(SRCROOT) && go test -v ./internal/guest/...
3434

35-
out/delta.tar.gz: bin/init bin/vsockexec bin/cmd/gcs bin/cmd/gcstools Makefile
35+
out/delta.tar.gz: bin/init bin/vsockexec bin/cmd/gcs bin/cmd/gcstools bin/cmd/hooks/wait-paths Makefile
3636
@mkdir -p out
3737
rm -rf rootfs
3838
mkdir -p rootfs/bin/
3939
cp bin/init rootfs/
4040
cp bin/vsockexec rootfs/bin/
4141
cp bin/cmd/gcs rootfs/bin/
4242
cp bin/cmd/gcstools rootfs/bin/
43+
cp bin/cmd/hooks/wait-paths rootfs/bin/
4344
for tool in $(GCS_TOOLS); do ln -s gcstools rootfs/bin/$$tool; done
4445
git -C $(SRCROOT) rev-parse HEAD > rootfs/gcs.commit && \
4546
git -C $(SRCROOT) rev-parse --abbrev-ref HEAD > rootfs/gcs.branch
@@ -60,6 +61,7 @@ out/initrd.img: $(BASE) out/delta.tar.gz $(SRCROOT)/hack/catcpio.sh
6061

6162
-include deps/cmd/gcs.gomake
6263
-include deps/cmd/gcstools.gomake
64+
-include deps/cmd/hooks/wait-paths.gomake
6365

6466
# Implicit rule for includes that define Go targets.
6567
%.gomake: $(SRCROOT)/Makefile

‎cmd/gcs/main.go‎

Lines changed: 9 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -14,20 +14,22 @@ import (
1414
"syscall"
1515
"time"
1616

17+
"github.com/containerd/cgroups"
18+
cgroupstats "github.com/containerd/cgroups/stats/v1"
19+
oci "github.com/opencontainers/runtime-spec/specs-go"
20+
"github.com/pkg/errors"
21+
"github.com/sirupsen/logrus"
22+
"go.opencensus.io/trace"
23+
1724
"github.com/Microsoft/hcsshim/internal/guest/bridge"
1825
"github.com/Microsoft/hcsshim/internal/guest/kmsg"
1926
"github.com/Microsoft/hcsshim/internal/guest/runtime/hcsv2"
2027
"github.com/Microsoft/hcsshim/internal/guest/runtime/runc"
2128
"github.com/Microsoft/hcsshim/internal/guest/transport"
29+
"github.com/Microsoft/hcsshim/internal/guestpath"
2230
"github.com/Microsoft/hcsshim/internal/log"
2331
"github.com/Microsoft/hcsshim/internal/oc"
2432
"github.com/cenkalti/backoff/v4"
25-
"github.com/containerd/cgroups"
26-
cgroupstats "github.com/containerd/cgroups/stats/v1"
27-
oci "github.com/opencontainers/runtime-spec/specs-go"
28-
"github.com/pkg/errors"
29-
"github.com/sirupsen/logrus"
30-
"go.opencensus.io/trace"
3133
)
3234

3335
func memoryLogFormat(metrics *cgroupstats.Metrics) logrus.Fields {
@@ -229,7 +231,7 @@ func main() {
229231

230232
log.SetScrubbing(*scrubLogs)
231233

232-
baseLogPath := "/run/gcs/c"
234+
baseLogPath := guestpath.LCOWRootPrefixInUVM
233235

234236
logrus.Info("GCS started")
235237

‎cmd/hooks/wait-paths/main.go‎

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
// +build linux
2+
3+
package main
4+
5+
import (
6+
"context"
7+
"fmt"
8+
"os"
9+
"strings"
10+
"time"
11+
12+
"github.com/sirupsen/logrus"
13+
"github.com/urfave/cli"
14+
)
15+
16+
const (
17+
pathsFlag = "paths"
18+
timeoutFlag = "timeout"
19+
)
20+
21+
// This is a hook that waits for a specific path to appear.
22+
// The hook has required list of comma-separated paths and a default timeout in seconds.
23+
24+
func main() {
25+
app := cli.NewApp()
26+
app.Name = "wait-paths"
27+
app.Usage = "Provide a list paths and an optional timeout"
28+
app.Flags = []cli.Flag{
29+
cli.StringFlag{
30+
Name: pathsFlag + ",p",
31+
Usage: "Comma-separated list of paths that should become available",
32+
Required: true,
33+
},
34+
cli.IntFlag{
35+
Name: timeoutFlag + ",t",
36+
Usage: "Timeout in seconds",
37+
Value: 30,
38+
},
39+
}
40+
app.Action = run
41+
if err := app.Run(os.Args); err != nil {
42+
logrus.Fatalf("%s\n", err)
43+
}
44+
os.Exit(0)
45+
}
46+
47+
func run(cCtx *cli.Context) error {
48+
timeout := cCtx.GlobalInt(timeoutFlag)
49+
paths := strings.Split(cCtx.GlobalString(pathsFlag), ",")
50+
51+
waitCtx, cancel := context.WithTimeout(context.Background(), time.Duration(timeout)*time.Second)
52+
defer cancel()
53+
54+
for _, path := range paths {
55+
for {
56+
if _, err := os.Stat(path); err != nil {
57+
if !os.IsNotExist(err) {
58+
return err
59+
}
60+
select {
61+
case <-waitCtx.Done():
62+
return fmt.Errorf("timeout while waiting for path %q to appear", path)
63+
default:
64+
time.Sleep(time.Millisecond * 10)
65+
continue
66+
}
67+
}
68+
break
69+
}
70+
}
71+
return nil
72+
}

‎internal/devices/drivers.go‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import (
88
"fmt"
99

1010
"github.com/Microsoft/hcsshim/internal/cmd"
11+
"github.com/Microsoft/hcsshim/internal/guestpath"
1112
"github.com/Microsoft/hcsshim/internal/log"
1213
"github.com/Microsoft/hcsshim/internal/resources"
1314
"github.com/Microsoft/hcsshim/internal/uvm"
@@ -45,7 +46,7 @@ func InstallKernelDriver(ctx context.Context, vm *uvm.UtilityVM, driver string)
4546
}
4647
return closer, execPnPInstallDriver(ctx, vm, uvmPath)
4748
}
48-
uvmPathForShare := fmt.Sprintf(uvm.LCOWGlobalMountPrefix, vm.UVMMountCounter())
49+
uvmPathForShare := fmt.Sprintf(guestpath.LCOWGlobalMountPrefixFmt, vm.UVMMountCounter())
4950
scsiCloser, err := vm.AddSCSI(ctx, driver, uvmPathForShare, true, false, []string{}, uvm.VMAccessTypeIndividual)
5051
if err != nil {
5152
return closer, fmt.Errorf("failed to add SCSI disk to utility VM for path %+v: %s", driver, err)

‎internal/guest/runtime/hcsv2/nvidia_utils.go‎

Lines changed: 21 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -10,17 +10,16 @@ import (
1010
"os/exec"
1111
"strings"
1212

13+
oci "github.com/opencontainers/runtime-spec/specs-go"
14+
"github.com/pkg/errors"
15+
1316
"github.com/Microsoft/hcsshim/cmd/gcstools/generichook"
1417
"github.com/Microsoft/hcsshim/internal/guest/storage/pci"
18+
"github.com/Microsoft/hcsshim/internal/guestpath"
19+
"github.com/Microsoft/hcsshim/internal/hooks"
1520
"github.com/Microsoft/hcsshim/pkg/annotations"
16-
oci "github.com/opencontainers/runtime-spec/specs-go"
17-
"github.com/pkg/errors"
1821
)
1922

20-
// path that the shim mounts the nvidia gpu vhd to in the uvm
21-
// this MUST match the path mapped to in the shim
22-
const lcowNvidiaMountPath = "/run/nvidia"
23-
2423
const nvidiaDebugFilePath = "/nvidia-container.log"
2524

2625
const nvidiaToolBinary = "nvidia-container-cli"
@@ -70,35 +69,33 @@ func addNvidiaDevicePreHook(ctx context.Context, spec *oci.Spec) error {
7069
// add template for pid argument to be injected later by the generic hook binary
7170
args = append(args, "--no-cgroups", "--pid={{pid}}", spec.Root.Path)
7271

73-
if spec.Hooks == nil {
74-
spec.Hooks = &oci.Hooks{}
75-
}
76-
7772
hookLogDebugFileEnvOpt := fmt.Sprintf("%s=%s", generichook.LogDebugFileEnvKey, nvidiaDebugFilePath)
7873
hookEnv := append(updateEnvWithNvidiaVariables(), hookLogDebugFileEnvOpt)
79-
nvidiaHook := oci.Hook{
80-
Path: genericHookPath,
81-
Args: args,
82-
Env: hookEnv,
83-
}
84-
85-
spec.Hooks.Prestart = append(spec.Hooks.Prestart, nvidiaHook)
86-
return nil
74+
nvidiaHook := hooks.NewOCIHook(genericHookPath, args, hookEnv)
75+
return hooks.AddOCIHook(spec, hooks.Prestart, nvidiaHook)
8776
}
8877

8978
// updateEnvWithNvidiaVariables creates an env with the nvidia gpu vhd in PATH and insecure mode set
9079
func updateEnvWithNvidiaVariables() []string {
80+
nvidiaBin := fmt.Sprintf("%s/bin", guestpath.LCOWNvidiaMountPath)
81+
env := updatePathEnv(nvidiaBin)
82+
// NVC_INSECURE_MODE allows us to run nvidia-container-cli without seccomp
83+
// we don't currently use seccomp in the uvm, so avoid using it here for now as well
84+
env = append(env, "NVC_INSECURE_MODE=1")
85+
return env
86+
}
87+
88+
// updatePathEnv adds specified `dirs` to PATH variable and returns the result environment variables.
89+
func updatePathEnv(dirs ...string) []string {
9190
pathPrefix := "PATH="
92-
nvidiaBin := fmt.Sprintf("%s/bin", lcowNvidiaMountPath)
91+
additionalDirs := strings.Join(dirs, ":")
9392
env := os.Environ()
9493
for i, v := range env {
9594
if strings.HasPrefix(v, pathPrefix) {
96-
newPath := fmt.Sprintf("%s:%s", v, nvidiaBin)
95+
newPath := fmt.Sprintf("%s:%s", v, additionalDirs)
9796
env[i] = newPath
97+
return env
9898
}
9999
}
100-
// NVC_INSECURE_MODE allows us to run nvidia-container-cli without seccomp
101-
// we don't currently use seccomp in the uvm, so avoid using it here for now as well
102-
env = append(env, "NVC_INSECURE_MODE=1")
103-
return env
100+
return append(env, fmt.Sprintf("PATH=%s", additionalDirs))
104101
}

‎internal/guest/runtime/hcsv2/sandbox_container.go‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -10,16 +10,18 @@ import (
1010
"path/filepath"
1111
"strings"
1212

13-
"github.com/Microsoft/hcsshim/internal/guest/network"
14-
"github.com/Microsoft/hcsshim/internal/oc"
15-
"github.com/Microsoft/hcsshim/pkg/annotations"
1613
oci "github.com/opencontainers/runtime-spec/specs-go"
1714
"github.com/pkg/errors"
1815
"go.opencensus.io/trace"
16+
17+
"github.com/Microsoft/hcsshim/internal/guest/network"
18+
"github.com/Microsoft/hcsshim/internal/guestpath"
19+
"github.com/Microsoft/hcsshim/internal/oc"
20+
"github.com/Microsoft/hcsshim/pkg/annotations"
1921
)
2022

2123
func getSandboxRootDir(id string) string {
22-
return filepath.Join("/run/gcs/c", id)
24+
return filepath.Join(guestpath.LCOWRootPrefixInUVM, id)
2325
}
2426

2527
func getSandboxHugePageMountsDir(id string) string {

‎internal/guest/runtime/hcsv2/spec.go‎

Lines changed: 7 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -10,12 +10,14 @@ import (
1010
"strconv"
1111
"strings"
1212

13-
"github.com/Microsoft/hcsshim/internal/log"
14-
"github.com/Microsoft/hcsshim/pkg/annotations"
1513
"github.com/opencontainers/runc/libcontainer/devices"
1614
"github.com/opencontainers/runc/libcontainer/user"
1715
oci "github.com/opencontainers/runtime-spec/specs-go"
1816
"github.com/pkg/errors"
17+
18+
"github.com/Microsoft/hcsshim/internal/hooks"
19+
"github.com/Microsoft/hcsshim/internal/log"
20+
"github.com/Microsoft/hcsshim/pkg/annotations"
1921
)
2022

2123
// getNetworkNamespaceID returns the `ToLower` of
@@ -257,17 +259,7 @@ func applyAnnotationsToSpec(ctx context.Context, spec *oci.Spec) error {
257259
}
258260

259261
// Helper function to create an oci prestart hook to run ldconfig
260-
func addLDConfigHook(ctx context.Context, spec *oci.Spec, args, env []string) error {
261-
if spec.Hooks == nil {
262-
spec.Hooks = &oci.Hooks{}
263-
}
264-
265-
ldConfigHook := oci.Hook{
266-
Path: "/sbin/ldconfig",
267-
Args: args,
268-
Env: env,
269-
}
270-
271-
spec.Hooks.Prestart = append(spec.Hooks.Prestart, ldConfigHook)
272-
return nil
262+
func addLDConfigHook(_ context.Context, spec *oci.Spec, args, env []string) error {
263+
ldConfigHook := hooks.NewOCIHook("/sbin/ldconfig", args, env)
264+
return hooks.AddOCIHook(spec, hooks.Prestart, ldConfigHook)
273265
}

‎internal/guest/runtime/hcsv2/standalone_container.go‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10,15 +10,17 @@ import (
1010
"path/filepath"
1111
"strings"
1212

13-
"github.com/Microsoft/hcsshim/internal/guest/network"
14-
"github.com/Microsoft/hcsshim/internal/oc"
1513
oci "github.com/opencontainers/runtime-spec/specs-go"
1614
"github.com/pkg/errors"
1715
"go.opencensus.io/trace"
16+
17+
"github.com/Microsoft/hcsshim/internal/guest/network"
18+
"github.com/Microsoft/hcsshim/internal/guestpath"
19+
"github.com/Microsoft/hcsshim/internal/oc"
1820
)
1921

2022
func getStandaloneRootDir(id string) string {
21-
return filepath.Join("/run/gcs/c", id)
23+
return filepath.Join(guestpath.LCOWRootPrefixInUVM, id)
2224
}
2325

2426
func getStandaloneHostnamePath(id string) string {

‎internal/guest/runtime/hcsv2/uvm.go‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -225,12 +225,16 @@ func (h *Host) CreateContainer(ctx context.Context, id string, settings *prot.VM
225225
// We append the variable after the security policy enforcing logic completes so as to bypass it; the
226226
// security policy variable cannot be included in the security policy as its value is not available
227227
// security policy construction time.
228-
229228
if policyEnforcer, ok := (h.securityPolicyEnforcer).(*securitypolicy.StandardSecurityPolicyEnforcer); ok {
230229
secPolicyEnv := fmt.Sprintf("SECURITY_POLICY=%s", policyEnforcer.EncodedSecurityPolicy)
231230
settings.OCISpecification.Process.Env = append(settings.OCISpecification.Process.Env, secPolicyEnv)
232231
}
233232

233+
// Sandbox mount paths need to be resolved in the spec before expected mounts policy can be enforced.
234+
if err = h.securityPolicyEnforcer.EnforceExpectedMountsPolicy(id, settings.OCISpecification); err != nil {
235+
return nil, errors.Wrapf(err, "container creation denied due to policy")
236+
}
237+
234238
// Create the BundlePath
235239
if err := os.MkdirAll(settings.OCIBundlePath, 0700); err != nil {
236240
return nil, errors.Wrapf(err, "failed to create OCIBundlePath: '%s'", settings.OCIBundlePath)

‎internal/guest/runtime/hcsv2/workload_container.go‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -9,16 +9,18 @@ import (
99
"path/filepath"
1010
"strings"
1111

12-
"github.com/Microsoft/hcsshim/internal/oc"
13-
"github.com/Microsoft/hcsshim/pkg/annotations"
1412
oci "github.com/opencontainers/runtime-spec/specs-go"
1513
"github.com/pkg/errors"
1614
"go.opencensus.io/trace"
1715
"golang.org/x/sys/unix"
16+
17+
"github.com/Microsoft/hcsshim/internal/guestpath"
18+
"github.com/Microsoft/hcsshim/internal/oc"
19+
"github.com/Microsoft/hcsshim/pkg/annotations"
1820
)
1921

2022
func getWorkloadRootDir(id string) string {
21-
return filepath.Join("/run/gcs/c", id)
23+
return filepath.Join(guestpath.LCOWRootPrefixInUVM, id)
2224
}
2325

2426
// os.MkdirAll combines the given permissions with the running process's
@@ -32,11 +34,10 @@ func mkdirAllModePerm(target string) error {
3234
}
3335

3436
func updateSandboxMounts(sbid string, spec *oci.Spec) error {
35-
sandboxMountPrefix := "sandbox://"
3637
for i, m := range spec.Mounts {
37-
if strings.HasPrefix(m.Source, sandboxMountPrefix) {
38+
if strings.HasPrefix(m.Source, guestpath.SandboxMountPrefix) {
3839
mountsDir := getSandboxMountsDir(sbid)
39-
subPath := strings.TrimPrefix(m.Source, sandboxMountPrefix)
40+
subPath := strings.TrimPrefix(m.Source, guestpath.SandboxMountPrefix)
4041
sandboxSource := filepath.Join(mountsDir, subPath)
4142

4243
// filepath.Join cleans the resulting path before returning so it would resolve the relative path if one was given.
@@ -59,11 +60,10 @@ func updateSandboxMounts(sbid string, spec *oci.Spec) error {
5960
}
6061

6162
func updateHugePageMounts(sbid string, spec *oci.Spec) error {
62-
mountPrefix := "hugepages://"
6363
for i, m := range spec.Mounts {
64-
if strings.HasPrefix(m.Source, mountPrefix) {
64+
if strings.HasPrefix(m.Source, guestpath.HugePagesMountPrefix) {
6565
mountsDir := getSandboxHugePageMountsDir(sbid)
66-
subPath := strings.TrimPrefix(m.Source, mountPrefix)
66+
subPath := strings.TrimPrefix(m.Source, guestpath.HugePagesMountPrefix)
6767
pageSize := strings.Split(subPath, string(os.PathSeparator))[0]
6868
hugePageMountSource := filepath.Join(mountsDir, subPath)
6969

0 commit comments

Comments
 (0)