Skip to content

Commit 9652102

Browse files
rahulaitnvidia-backport-bot
authored andcommitted
store os release name and use it than recalculating from tag which is errorprone
Signed-off-by: Rahul Sharma <[email protected]> (cherry picked from commit 6afb205)
1 parent 5281a7e commit 9652102

7 files changed

Lines changed: 332 additions & 41 deletions

File tree

‎controllers/object_controls.go‎

Lines changed: 33 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -3298,16 +3298,9 @@ func resolveDriverTag(n ClusterPolicyController, driverSpec interface{}) (string
32983298
return image, nil
32993299
}
33003300

3301-
func getOSName(osTag string) string {
3302-
// Extract base OS ID by stripping version suffix from osTag
3303-
// Examples: "rhel10" -> "rhel", "ubuntu22.04" -> "ubuntu", "rocky9" -> "rocky"
3304-
osID := strings.TrimRight(osTag, "0123456789.")
3305-
return osID
3306-
}
3307-
33083301
// getRepoConfigPath returns the standard OS specific path for repository configuration files.
33093302
func (n ClusterPolicyController) getRepoConfigPath() (string, error) {
3310-
osID := getOSName(n.gpuNodeOSTag)
3303+
osID := n.gpuNodeOSRelease
33113304
if path, ok := RepoConfigPathMap[osID]; ok {
33123305
return path, nil
33133306
}
@@ -3316,7 +3309,7 @@ func (n ClusterPolicyController) getRepoConfigPath() (string, error) {
33163309

33173310
// getCertConfigPath returns the standard OS specific path for ssl keys/certificates.
33183311
func (n ClusterPolicyController) getCertConfigPath() (string, error) {
3319-
osID := getOSName(n.gpuNodeOSTag)
3312+
osID := n.gpuNodeOSRelease
33203313
if path, ok := CertConfigPathMap[osID]; ok {
33213314
return path, nil
33223315
}
@@ -3326,7 +3319,7 @@ func (n ClusterPolicyController) getCertConfigPath() (string, error) {
33263319
// getSubscriptionPathsToVolumeSources returns the MountPathToVolumeSource map containing all
33273320
// OS-specific subscription/entitlement paths that need to be mounted in the container.
33283321
func (n ClusterPolicyController) getSubscriptionPathsToVolumeSources() (MountPathToVolumeSource, error) {
3329-
osID := getOSName(n.gpuNodeOSTag)
3322+
osID := n.gpuNodeOSRelease
33303323
if pathToVolumeSource, ok := SubscriptionPathMap[osID]; ok {
33313324
return pathToVolumeSource, nil
33323325
}
@@ -3553,6 +3546,33 @@ func transformDriverContainer(obj *appsv1.DaemonSet, config *gpuv1.ClusterPolicy
35533546
}
35543547
}
35553548

3549+
<<<<<<< HEAD
3550+
=======
3551+
// Mount /lib/modules for precompiled drivers on SUSE distributions.
3552+
// Those containers need access to host /lib/modules at runtime.
3553+
osID := n.gpuNodeOSRelease
3554+
if config.Driver.UsePrecompiledDrivers() && (osID == "sles" || osID == "sl-micro") {
3555+
n.logger.Info("Mounting /lib/modules into the driver container")
3556+
libModulesVolMount := corev1.VolumeMount{
3557+
Name: "lib-modules",
3558+
MountPath: "/run/host/lib/modules",
3559+
ReadOnly: true,
3560+
}
3561+
driverContainer.VolumeMounts = append(driverContainer.VolumeMounts, libModulesVolMount)
3562+
3563+
libModulesVol := corev1.Volume{
3564+
Name: "lib-modules",
3565+
VolumeSource: corev1.VolumeSource{
3566+
HostPath: &corev1.HostPathVolumeSource{
3567+
Path: "/lib/modules",
3568+
Type: ptr.To(corev1.HostPathDirectory),
3569+
},
3570+
},
3571+
}
3572+
podSpec.Volumes = append(podSpec.Volumes, libModulesVol)
3573+
}
3574+
3575+
>>>>>>> 6afb20500 (store os release name and use it than recalculating from tag which is errorprone)
35563576
// no further repo configuration required when using pre-compiled drivers, return here.
35573577
if config.Driver.UsePrecompiledDrivers() {
35583578
return nil
@@ -3594,7 +3614,10 @@ func transformDriverContainer(obj *appsv1.DaemonSet, config *gpuv1.ClusterPolicy
35943614
}
35953615
}
35963616

3617+
<<<<<<< HEAD
35973618
osID := getOSName(n.gpuNodeOSTag)
3619+
=======
3620+
>>>>>>> 6afb20500 (store os release name and use it than recalculating from tag which is errorprone)
35983621
// set up subscription entitlements for RHEL(using K8s with a non-CRIO runtime) and SLES
35993622
if (osID == "rhel" && n.openshift == "" && n.runtime != gpuv1.CRIO) || osID == "sles" || osID == "sl-micro" {
36003623
n.logger.Info("Mounting subscriptions into the driver container", "OS", osID)

‎controllers/object_controls_test.go‎

Lines changed: 103 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -225,13 +225,14 @@ func setup() error {
225225
if gpuNodeCount == 0 {
226226
return fmt.Errorf("no gpu nodes in mock cluster")
227227
}
228-
gpuNodeOSTag, err := clusterPolicyController.getGPUNodeOSTag()
228+
gpuNodeOSRelease, gpuNodeOSTag, err := clusterPolicyController.getGPUNodeOSInfo()
229229
if err != nil {
230230
return fmt.Errorf("unable to get GPU node tag: %w", err)
231231
}
232232

233233
clusterPolicyController.hasGPUNodes = gpuNodeCount != 0
234234
clusterPolicyController.hasNFDLabels = hasNFDLabels
235+
clusterPolicyController.gpuNodeOSRelease = gpuNodeOSRelease
235236
clusterPolicyController.gpuNodeOSTag = gpuNodeOSTag
236237

237238
// setup kernelVersionMap for pre-compiled driver tests
@@ -1883,3 +1884,104 @@ func TestMIGManager(t *testing.T) {
18831884
})
18841885
}
18851886
}
1887+
<<<<<<< HEAD
1888+
=======
1889+
1890+
// TestDriverPrecompiledLibModulesUbuntu tests that /lib/modules is NOT mounted for precompiled drivers on Ubuntu
1891+
func TestDriverPrecompiledLibModulesUbuntu(t *testing.T) {
1892+
cp := getDriverTestInput("precompiled")
1893+
output := getDriverTestOutput("precompiled")
1894+
1895+
ds, err := testDaemonsetCommon(t, cp, "Driver", output["numDaemonsets"].(int))
1896+
if err != nil {
1897+
t.Fatalf("error in testDaemonsetCommon(): %v", err)
1898+
}
1899+
require.NotNil(t, ds)
1900+
1901+
// Check that /lib/modules volume and mount are NOT present
1902+
for _, vol := range ds.Spec.Template.Spec.Volumes {
1903+
require.NotEqual(t, "lib-modules", vol.Name, "lib-modules volume should not be present for ubuntu")
1904+
}
1905+
1906+
driverContainer := findContainerByName(ds.Spec.Template.Spec.Containers, "nvidia-driver-ctr")
1907+
require.NotNil(t, driverContainer)
1908+
1909+
for _, mount := range driverContainer.VolumeMounts {
1910+
require.NotEqual(t, "lib-modules", mount.Name, "lib-modules volume mount should not be present for ubuntu")
1911+
}
1912+
1913+
// Cleanup
1914+
err = removeState(&clusterPolicyController, clusterPolicyController.idx-1)
1915+
if err != nil {
1916+
t.Fatalf("error removing state %v:", err)
1917+
}
1918+
clusterPolicyController.idx--
1919+
}
1920+
1921+
// TestDriverPrecompiledLibModulesSuse tests that /lib/modules is mounted for precompiled drivers on SLES and SL-Micro
1922+
func TestDriverPrecompiledLibModulesSuse(t *testing.T) {
1923+
osTags := []string{"sles16.0", "sl-micro6.1"}
1924+
1925+
for _, osTag := range osTags {
1926+
t.Run(osTag, func(t *testing.T) {
1927+
// Save original OS info and restore after test
1928+
originalOSRelease := clusterPolicyController.gpuNodeOSRelease
1929+
originalOSTag := clusterPolicyController.gpuNodeOSTag
1930+
defer func() {
1931+
clusterPolicyController.gpuNodeOSRelease = originalOSRelease
1932+
clusterPolicyController.gpuNodeOSTag = originalOSTag
1933+
}()
1934+
1935+
switch {
1936+
case strings.HasPrefix(osTag, "sles"):
1937+
clusterPolicyController.gpuNodeOSRelease = "sles"
1938+
case strings.HasPrefix(osTag, "sl-micro"):
1939+
clusterPolicyController.gpuNodeOSRelease = "sl-micro"
1940+
default:
1941+
t.Fatalf("unsupported SUSE osTag in test: %s", osTag)
1942+
}
1943+
clusterPolicyController.gpuNodeOSTag = osTag
1944+
1945+
cp := getDriverTestInput("precompiled")
1946+
output := getDriverTestOutput("precompiled")
1947+
1948+
ds, err := testDaemonsetCommon(t, cp, "Driver", output["numDaemonsets"].(int))
1949+
if err != nil {
1950+
t.Fatalf("error in testDaemonsetCommon(): %v", err)
1951+
}
1952+
require.NotNil(t, ds)
1953+
1954+
// Check for /lib/modules volume and mount
1955+
foundVolume := false
1956+
for _, vol := range ds.Spec.Template.Spec.Volumes {
1957+
if vol.Name == "lib-modules" {
1958+
foundVolume = true
1959+
require.NotNil(t, vol.HostPath)
1960+
require.Equal(t, "/lib/modules", vol.HostPath.Path)
1961+
}
1962+
}
1963+
require.Truef(t, foundVolume, "lib-modules volume not found for precompiled drivers on %s", osTag)
1964+
1965+
foundMount := false
1966+
driverContainer := findContainerByName(ds.Spec.Template.Spec.Containers, "nvidia-driver-ctr")
1967+
require.NotNil(t, driverContainer)
1968+
1969+
for _, mount := range driverContainer.VolumeMounts {
1970+
if mount.Name == "lib-modules" {
1971+
foundMount = true
1972+
require.Equal(t, "/run/host/lib/modules", mount.MountPath)
1973+
require.True(t, mount.ReadOnly)
1974+
}
1975+
}
1976+
require.Truef(t, foundMount, "lib-modules volume mount not found for precompiled drivers on %s", osTag)
1977+
1978+
// Cleanup
1979+
err = removeState(&clusterPolicyController, clusterPolicyController.idx-1)
1980+
if err != nil {
1981+
t.Fatalf("error removing state %v:", err)
1982+
}
1983+
clusterPolicyController.idx--
1984+
})
1985+
}
1986+
}
1987+
>>>>>>> 6afb20500 (store os release name and use it than recalculating from tag which is errorprone)

‎controllers/state_manager.go‎

Lines changed: 44 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -164,11 +164,12 @@ type ClusterPolicyController struct {
164164
openshift string
165165
ocpDriverToolkit OpenShiftDriverToolkit
166166

167-
runtime gpuv1.Runtime
168-
gpuNodeOSTag string
169-
hasGPUNodes bool
170-
hasNFDLabels bool
171-
sandboxEnabled bool
167+
runtime gpuv1.Runtime
168+
gpuNodeOSTag string
169+
gpuNodeOSRelease string
170+
hasGPUNodes bool
171+
hasNFDLabels bool
172+
sandboxEnabled bool
172173
}
173174

174175
func addState(n *ClusterPolicyController, path string) {
@@ -637,7 +638,7 @@ func getRuntimeString(node corev1.Node) (gpuv1.Runtime, error) {
637638
return runtime, nil
638639
}
639640

640-
func (n *ClusterPolicyController) getGPUNodeOSTag() (string, error) {
641+
func (n *ClusterPolicyController) getGPUNodeOSInfo() (string, string, error) {
641642
ctx := n.ctx
642643
opts := []client.ListOption{
643644
client.MatchingLabels(map[string]string{commonGPULabelKey: commonGPULabelValue}),
@@ -646,34 +647,55 @@ func (n *ClusterPolicyController) getGPUNodeOSTag() (string, error) {
646647
nodeList := &corev1.NodeList{}
647648
err := n.client.List(ctx, nodeList, opts...)
648649
if err != nil {
649-
return "", fmt.Errorf("unable to list nodes with GPU present: %w", err)
650+
return "", "", fmt.Errorf("unable to list nodes with GPU present: %w", err)
650651
}
651652
if len(nodeList.Items) == 0 {
652-
return "", fmt.Errorf("no nodes found with GPU present")
653+
return "", "", fmt.Errorf("no nodes found with GPU present")
653654
}
654655

655656
labels := nodeList.Items[0].Labels
656657
osName, ok := labels[nfdOSReleaseIDLabelKey]
657658
if !ok {
658-
return "", fmt.Errorf("unable to retrieve OS name from label %s", nfdOSReleaseIDLabelKey)
659+
return "", "", fmt.Errorf("unable to retrieve OS name from label %s", nfdOSReleaseIDLabelKey)
659660
}
660661
osVersion, ok := labels[nfdOSVersionIDLabelKey]
661662
if !ok {
662-
return "", fmt.Errorf("unable to retrieve OS version from label %s", nfdOSVersionIDLabelKey)
663+
return "", "", fmt.Errorf("unable to retrieve OS version from label %s", nfdOSVersionIDLabelKey)
663664
}
664665
osMajorVersion := strings.Split(osVersion, ".")[0]
665-
osMajorNumber, err := strconv.Atoi(osMajorVersion)
666-
if err != nil {
667-
return "", fmt.Errorf("error processing OS major version %s: %w", osMajorVersion, err)
668-
}
669666

670667
// If the OS is RockyLinux or RHEL 10 & above, we will omit the minor version when constructing the os image tag
671-
if osName == "rocky" || (osName == "rhel" && osMajorNumber >= 10) {
668+
switch osName {
669+
case "rocky":
672670
osVersion = osMajorVersion
671+
case "rhel":
672+
osMajorNumber, err := parseOSMajorVersion(osVersion)
673+
if err != nil {
674+
return "", "", err
675+
}
676+
if osMajorNumber >= 10 {
677+
osVersion = osMajorVersion
678+
}
673679
}
674680
osTag := fmt.Sprintf("%s%s", osName, osVersion)
675681

676-
return osTag, nil
682+
return osName, osTag, nil
683+
}
684+
685+
func parseOSMajorVersion(osVersion string) (int, error) {
686+
osMajorVersion := strings.Split(osVersion, ".")[0]
687+
osMajorVersion = strings.TrimSpace(osMajorVersion)
688+
osMajorVersion = strings.TrimPrefix(strings.TrimPrefix(osMajorVersion, "v"), "V")
689+
if osMajorVersion == "" {
690+
return 0, fmt.Errorf("empty OS major version")
691+
}
692+
693+
osMajorNumber, err := strconv.Atoi(osMajorVersion)
694+
if err != nil {
695+
return 0, fmt.Errorf("error processing OS major version %s: %w", osMajorVersion, err)
696+
}
697+
698+
return osMajorNumber, nil
677699
}
678700

679701
func (n *ClusterPolicyController) setPodSecurityLabelsForNamespace() error {
@@ -939,10 +961,14 @@ func (n *ClusterPolicyController) init(ctx context.Context, reconciler *ClusterP
939961
n.hasNFDLabels = hasNFDLabels
940962

941963
if n.hasGPUNodes {
942-
gpuNodeOSTag, err := n.getGPUNodeOSTag()
964+
gpuNodeOSRelease, gpuNodeOSTag, err := n.getGPUNodeOSInfo()
943965
if err != nil {
944-
return fmt.Errorf("failed to retrieve GPU node OS tag: %w", err)
966+
return fmt.Errorf("failed to retrieve GPU node OS info: %w", err)
967+
}
968+
if gpuNodeOSRelease == "" || gpuNodeOSTag == "" {
969+
return fmt.Errorf("GPU node OS info is empty: release=%q, tag=%q", gpuNodeOSRelease, gpuNodeOSTag)
945970
}
971+
n.gpuNodeOSRelease = gpuNodeOSRelease
946972
n.gpuNodeOSTag = gpuNodeOSTag
947973
}
948974
// fetch all nodes and annotate gpu nodes

0 commit comments

Comments
 (0)