Merge pull request #2034 from katiewasnothere/kabaldau/kmod_load_modules_init

Kathryn Baldauf · web-flow · commit df34d1dc7c6b · 2024-03-20T13:46:13.000-07:00
Use kmod library to load modules in init script
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -686,6 +686,11 @@ jobs:
             go.sum
             test/go.sum
 
+      - name: Install kmod
+        shell: bash
+        run: | 
+          sudo apt-get install -y libkmod-dev
+
       - name: Set version info
         shell: pwsh
         run: |
diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml
@@ -92,6 +92,11 @@ jobs:
         with:
           languages: ${{matrix.language}}
 
+      - name: Install kmod
+        shell: bash
+        run: | 
+          sudo apt-get install -y libkmod-dev
+
       # build binaries
       - name: Build go binaries
         shell: pwsh
diff --git a/Makefile b/Makefile
@@ -7,7 +7,8 @@ CGO_ENABLED:=0
 GOMODVENDOR:=
 
 CFLAGS:=-O2 -Wall
-LDFLAGS:=-static -s # strip C binaries
+LDFLAGS:= -s # strip C binaries
+LDLIBS:= -lkmod
 
 GO_FLAGS_EXTRA:=
 ifeq "$(GOMODVENDOR)" "1"
@@ -181,7 +182,7 @@ bin/vsockexec: vsockexec/vsockexec.o vsockexec/vsock.o
 
 bin/init: init/init.o vsockexec/vsock.o
 	@mkdir -p bin
-	$(CC) $(LDFLAGS) -o $@ $^
+	$(CC) $(LDFLAGS) -o $@ $^ $(LDLIBS)
 
 %.o: %.c
 	@mkdir -p $(dir $@)
diff --git a/README.md b/README.md
@@ -10,6 +10,14 @@ It is primarily used in the [Moby](https://github.com/moby/moby) and [Containerd
 
 While this repository can be used as a library of sorts to call the HCS apis, there are a couple binaries built out of the repository as well. The main ones being the Linux guest agent, and an implementation of the [runtime v2 containerd shim api](https://github.com/containerd/containerd/blob/master/runtime/v2/README.md).
 
+### Install dependencies
+To build the init binary used to launch Utility VMs, we rely on "libkmod". To install libkmod, run the following in a linux environment:
+
+```sh
+> sudo apt-get update
+> sudo apt-get install -y libkmod2 libkmod-dev
+```
+
 ### Linux Hyper-V Container Guest Agent
 
 To build the Linux guest agent itself all that's needed is to set your GOOS to "Linux" and build out of ./cmd/gcs.
diff --git a/init/init.c b/init/init.c
@@ -1,7 +1,9 @@
 #define _GNU_SOURCE
 #include <errno.h>
 #include <fcntl.h>
+#include <ftw.h>
 #include <getopt.h>
+#include <libkmod.h>
 #include <net/if.h>
 #include <netinet/ip.h>
 #include <signal.h>
@@ -14,6 +16,7 @@
 #include <sys/stat.h>
 #include <sys/sysmacros.h>
 #include <sys/types.h>
+#include <sys/utsname.h>
 #include <sys/wait.h>
 #include <unistd.h>
 #include "../vsockexec/vsock.h"
@@ -57,15 +60,20 @@ static int opentcp(unsigned short port)
 #define RNDADDENTROPY _IOW( 'R', 0x03, int [2] )
 
 #define DEFAULT_PATH_ENV "PATH=/sbin:/usr/sbin:/bin:/usr/bin"
+#define OPEN_FDS 15
 
 const char *const default_envp[] = {
     DEFAULT_PATH_ENV,
     NULL,
 };
 
+// global kmod k_ctx so we can access it in the file tree traversal 
+struct kmod_ctx *k_ctx;
+
 // When nothing is passed, default to the LCOWv1 behavior.
 const char *const default_argv[] = { "/bin/gcs", "-loglevel", "debug", "-logfile=/run/gcs/gcs.log" };
 const char *const default_shell = "/bin/sh";
+const char *const lib_modules = "/lib/modules";
 
 struct Mount {
     const char *source, *target, *type;
@@ -403,6 +411,110 @@ int reap_until(pid_t until_pid) {
     }
 }
 
+// load_module gets the module from the absolute path to the module and then 
+// inserts into the kernel. 
+int load_module(struct kmod_ctx *ctx, const char *module_path) {
+    struct kmod_module *mod = NULL; 
+    int err;
+
+    #ifdef DEBUG
+    printf("loading module: %s\n", module_path);
+    #endif
+
+    err = kmod_module_new_from_path(ctx, module_path, &mod);
+    if (err < 0) {
+        return err; 
+    }
+
+    err = kmod_module_probe_insert_module(mod, 0, NULL, NULL, NULL, NULL); 
+    if (err < 0) {
+        kmod_module_unref(mod); 
+        return err;
+    }
+
+    kmod_module_unref(mod); 
+    return 0;
+}
+
+// parse_tree_entry is called by ftw for each directory and file in the file tree.
+// If this entry is a file and has a .ko file extension, attempt to load into kernel.
+int parse_tree_entry(const char *fpath, const struct stat *sb, int typeflag) {
+    int result;
+    const char *ext; 
+
+    if (typeflag != FTW_F) {
+        // do nothing if this isn't a file 
+        return 0; 
+    }
+        
+    ext = strrchr(fpath, '.');
+    if (!ext || ext == fpath) {
+        // no file extension found in the filepath
+        return 0; 
+    }
+
+    if ((result = strcmp(ext, ".ko")) != 0) {
+        // file does not have .ko extension so it is not a kernel module
+        return 0; 
+    }
+
+    // print warning if we fail to load the module, but don't fail fn so 
+    // we keep trying to load the rest of the modules. 
+    result = load_module(k_ctx, fpath);
+    if (result != 0) {
+        warn2("failed to load module", fpath);
+    }
+    return 0;
+}
+
+// load_all_modules finds the modules in the image and loads them using kmod, 
+// which accounts for ordering requirements. 
+void load_all_modules() {
+    int max_path = 256;
+    char modules_dir[max_path];
+    struct utsname uname_data;
+    int ret; 
+
+    // get information on the running kernel 
+    ret = uname(&uname_data);
+    if (ret != 0) {
+        die("failed to get kernel information");
+    }
+    
+    // create the absolute path of the modules directory this looks 
+    // like /lib/modules/<uname.release>
+    ret = snprintf(modules_dir, max_path, "%s/%s", lib_modules, uname_data.release); 
+    if (ret < 0) {
+        die("failed to create the modules directory path");
+    } else if (ret > max_path) {
+        die("modules directory buffer larger than expected"); 
+    }
+
+    if (k_ctx == NULL) {
+        k_ctx = kmod_new(NULL, NULL);
+        if (k_ctx == NULL) {
+            die("failed to create kmod context");
+        }
+    }
+
+    kmod_load_resources(k_ctx);
+    ret = ftw(modules_dir, parse_tree_entry, OPEN_FDS); 
+    if (ret < 0) {
+        kmod_unref(k_ctx);
+        die("failed to load kmod resources");
+    } else if (ret != 0) {
+        // Don't fail on error from walking the file tree and loading modules right now. 
+        // ftw may return an error if the modules directory doesn't exist, which
+        // may be the case for some images. Additionally, we don't currently support
+        // using a denylist when loading modules, so we may try to load modules 
+        // that cannot be loaded until later, such as nvidia modules which fail to 
+        // load if no device is present. 
+        warn("error adding modules");
+    }
+
+    kmod_unref(k_ctx);
+}
+
 #ifdef DEBUG
 int debug_main(int argc, char **argv) {
     unsigned int ports[3] = {2056, 2056, 2056};
@@ -533,6 +645,11 @@ int main(int argc, char **argv) {
         init_entropy(entropy_port);
     }
 
+    #ifdef DEBUG
+    printf("loading modules\n");
+    #endif
+    load_all_modules();
+
     pid_t pid = launch(child_argc, child_argv);
     if (debug_shell != NULL) {
         // The debug shell takes over as the primary child.
