Skip to content

Commit f46a031

Browse files
fcrozatgithub-actions[bot]
authored andcommitted
mount /lib/modules from host when using precompiled driver on SUSE distributions
Signed-off-by: Frederic Crozat <[email protected]> (cherry picked from commit a7323f5)
1 parent 30a8bbd commit f46a031

3 files changed

Lines changed: 141 additions & 2 deletions

File tree

‎controllers/object_controls.go‎

Lines changed: 25 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3553,6 +3553,30 @@ func transformDriverContainer(obj *appsv1.DaemonSet, config *gpuv1.ClusterPolicy
35533553
}
35543554
}
35553555

3556+
// Mount /lib/modules for precompiled drivers on SUSE distributions.
3557+
// Those containers need access to host /lib/modules at runtime.
3558+
osID := getOSName(n.gpuNodeOSTag)
3559+
if config.Driver.UsePrecompiledDrivers() && (osID == "sles" || osID == "sl-micro") {
3560+
n.logger.Info("Mounting /lib/modules into the driver container")
3561+
libModulesVolMount := corev1.VolumeMount{
3562+
Name: "lib-modules",
3563+
MountPath: "/run/host/lib/modules",
3564+
ReadOnly: true,
3565+
}
3566+
driverContainer.VolumeMounts = append(driverContainer.VolumeMounts, libModulesVolMount)
3567+
3568+
libModulesVol := corev1.Volume{
3569+
Name: "lib-modules",
3570+
VolumeSource: corev1.VolumeSource{
3571+
HostPath: &corev1.HostPathVolumeSource{
3572+
Path: "/lib/modules",
3573+
Type: ptr.To(corev1.HostPathDirectory),
3574+
},
3575+
},
3576+
}
3577+
podSpec.Volumes = append(podSpec.Volumes, libModulesVol)
3578+
}
3579+
35563580
// no further repo configuration required when using pre-compiled drivers, return here.
35573581
if config.Driver.UsePrecompiledDrivers() {
35583582
return nil
@@ -3594,7 +3618,7 @@ func transformDriverContainer(obj *appsv1.DaemonSet, config *gpuv1.ClusterPolicy
35943618
}
35953619
}
35963620

3597-
osID := getOSName(n.gpuNodeOSTag)
3621+
osID = getOSName(n.gpuNodeOSTag)
35983622
// set up subscription entitlements for RHEL(using K8s with a non-CRIO runtime) and SLES
35993623
if (osID == "rhel" && n.openshift == "" && n.runtime != gpuv1.CRIO) || osID == "sles" || osID == "sl-micro" {
36003624
n.logger.Info("Mounting subscriptions into the driver container", "OS", osID)

‎controllers/object_controls_test.go‎

Lines changed: 93 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -503,7 +503,11 @@ func testDaemonsetCommon(t *testing.T, cp *gpuv1.ClusterPolicy, component string
503503

504504
if component == "Driver" && cp.Spec.Driver.UsePrecompiledDrivers() {
505505
// for pre-compiled drivers, container image is kernel specific
506-
require.Contains(t, mainCtr.Image, "-generic-ubuntu22.04", "unexpected Image")
506+
suffix := "-generic-ubuntu22.04"
507+
if clusterPolicyController.gpuNodeOSTag != "" {
508+
suffix = fmt.Sprintf("-%s", clusterPolicyController.gpuNodeOSTag)
509+
}
510+
require.Contains(t, mainCtr.Image, suffix, "unexpected Image")
507511
} else {
508512
require.Equal(t, mainCtrImage, mainCtr.Image, "unexpected Image")
509513
}
@@ -1883,3 +1887,91 @@ func TestMIGManager(t *testing.T) {
18831887
})
18841888
}
18851889
}
1890+
1891+
// TestDriverPrecompiledLibModulesUbuntu tests that /lib/modules is NOT mounted for precompiled drivers on Ubuntu
1892+
func TestDriverPrecompiledLibModulesUbuntu(t *testing.T) {
1893+
cp := getDriverTestInput("precompiled")
1894+
output := getDriverTestOutput("precompiled")
1895+
1896+
ds, err := testDaemonsetCommon(t, cp, "Driver", output["numDaemonsets"].(int))
1897+
if err != nil {
1898+
t.Fatalf("error in testDaemonsetCommon(): %v", err)
1899+
}
1900+
require.NotNil(t, ds)
1901+
1902+
// Check that /lib/modules volume and mount are NOT present
1903+
for _, vol := range ds.Spec.Template.Spec.Volumes {
1904+
require.NotEqual(t, "lib-modules", vol.Name, "lib-modules volume should not be present for ubuntu")
1905+
}
1906+
1907+
driverContainer := findContainerByName(ds.Spec.Template.Spec.Containers, "nvidia-driver-ctr")
1908+
require.NotNil(t, driverContainer)
1909+
1910+
for _, mount := range driverContainer.VolumeMounts {
1911+
require.NotEqual(t, "lib-modules", mount.Name, "lib-modules volume mount should not be present for ubuntu")
1912+
}
1913+
1914+
// Cleanup
1915+
err = removeState(&clusterPolicyController, clusterPolicyController.idx-1)
1916+
if err != nil {
1917+
t.Fatalf("error removing state %v:", err)
1918+
}
1919+
clusterPolicyController.idx--
1920+
}
1921+
1922+
// TestDriverPrecompiledLibModulesSuse tests that /lib/modules is mounted for precompiled drivers on SLES and SL-Micro
1923+
func TestDriverPrecompiledLibModulesSuse(t *testing.T) {
1924+
osTags := []string{"sles16.0", "sl-micro6.1"}
1925+
1926+
for _, osTag := range osTags {
1927+
t.Run(osTag, func(t *testing.T) {
1928+
// Save original OS tag and restore after test
1929+
originalOSTag := clusterPolicyController.gpuNodeOSTag
1930+
defer func() {
1931+
clusterPolicyController.gpuNodeOSTag = originalOSTag
1932+
}()
1933+
1934+
clusterPolicyController.gpuNodeOSTag = osTag
1935+
1936+
cp := getDriverTestInput("precompiled")
1937+
output := getDriverTestOutput("precompiled")
1938+
1939+
ds, err := testDaemonsetCommon(t, cp, "Driver", output["numDaemonsets"].(int))
1940+
if err != nil {
1941+
t.Fatalf("error in testDaemonsetCommon(): %v", err)
1942+
}
1943+
require.NotNil(t, ds)
1944+
1945+
// Check for /lib/modules volume and mount
1946+
foundVolume := false
1947+
for _, vol := range ds.Spec.Template.Spec.Volumes {
1948+
if vol.Name == "lib-modules" {
1949+
foundVolume = true
1950+
require.NotNil(t, vol.HostPath)
1951+
require.Equal(t, "/lib/modules", vol.HostPath.Path)
1952+
}
1953+
}
1954+
require.Truef(t, foundVolume, "lib-modules volume not found for precompiled drivers on %s", osTag)
1955+
1956+
foundMount := false
1957+
driverContainer := findContainerByName(ds.Spec.Template.Spec.Containers, "nvidia-driver-ctr")
1958+
require.NotNil(t, driverContainer)
1959+
1960+
for _, mount := range driverContainer.VolumeMounts {
1961+
if mount.Name == "lib-modules" {
1962+
foundMount = true
1963+
require.Equal(t, "/run/host/lib/modules", mount.MountPath)
1964+
require.True(t, mount.ReadOnly)
1965+
}
1966+
}
1967+
require.Truef(t, foundMount, "lib-modules volume mount not found for precompiled drivers on %s", osTag)
1968+
1969+
// Cleanup
1970+
err = removeState(&clusterPolicyController, clusterPolicyController.idx-1)
1971+
if err != nil {
1972+
t.Fatalf("error removing state %v:", err)
1973+
}
1974+
clusterPolicyController.idx--
1975+
})
1976+
}
1977+
}

‎internal/state/driver_volumes.go‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -209,6 +209,29 @@ func (s *stateDriver) getDriverAdditionalConfigs(ctx context.Context, cr *v1alph
209209
additionalCfgs.Volumes = append(additionalCfgs.Volumes, subscriptionVol)
210210
}
211211
}
212+
213+
// Mount /lib/modules for precompiled drivers on SUSE distributions.
214+
// Those containers need access to host /lib/modules at runtime.
215+
if cr.Spec.UsePrecompiledDrivers() && (pool.osRelease == "sles" || pool.osRelease == "sl-micro") {
216+
logger.Info("Mounting /lib/modules into the driver container")
217+
libModulesVolMount := corev1.VolumeMount{
218+
Name: "lib-modules",
219+
MountPath: "/run/host/lib/modules",
220+
ReadOnly: true,
221+
}
222+
additionalCfgs.VolumeMounts = append(additionalCfgs.VolumeMounts, libModulesVolMount)
223+
224+
libModulesVol := corev1.Volume{
225+
Name: "lib-modules",
226+
VolumeSource: corev1.VolumeSource{
227+
HostPath: &corev1.HostPathVolumeSource{
228+
Path: "/lib/modules",
229+
Type: ptr.To(corev1.HostPathDirectory),
230+
},
231+
},
232+
}
233+
additionalCfgs.Volumes = append(additionalCfgs.Volumes, libModulesVol)
234+
}
212235
}
213236

214237
// mount any custom kernel module configuration parameters at /drivers

0 commit comments

Comments
 (0)