source ~/.gvm/scripts/gvm && gvm use go1.26.2
export AICR_BIN="/Users/haarchri/Documents/aicr/dist/aicr_darwin_arm64_v8.0/aicr"
${AICR_BIN} recipe --service eks --accelerator h100 --os ubuntu \
  --intent training -o /tmp/flux-bundle/recipe.yaml
${AICR_BIN} bundle -r /tmp/flux-bundle/recipe.yaml -o /tmp/flux-bundle \
  --deployer flux --repo https://github.com/haarchri/aicr-flux-bundle

kubectl get helmreleases -A
[...]
flux-system   nodewright-customizations       1m    True    Helm upgrade succeeded for release skyhook/skyhook-nodewright-customizations.v2 with chart [email protected]
[...]
flux-system   gpu-operator-post               1m    True    Helm install succeeded for release gpu-operator/gpu-operator-gpu-operator-post.v1 with chart [email protected]

kubectl get cm -n gpu-operator         dcgm-exporter     -o yaml
apiVersion: v1
data:
  dcgm-metrics.csv: |-
    # Clocks,,
    DCGM_FI_DEV_SM_CLOCK,     gauge, SM clock frequency (in MHz).
    DCGM_FI_DEV_MEM_CLOCK, gauge, Memory clock frequency (in MHz).

    # Temperature,,
    DCGM_FI_DEV_MEMORY_TEMP, gauge, Memory temperature (in C).
    DCGM_FI_DEV_GPU_TEMP,    gauge, GPU temperature (in C).

    # Power,,
    DCGM_FI_DEV_POWER_USAGE,  gauge, Power draw (in W).
    DCGM_FI_DEV_TOTAL_ENERGY_CONSUMPTION, counter, Total energy consumption since boot (in mJ).

    # PCIe,,
    DCGM_FI_PROF_PCIE_TX_BYTES,  counter, Total number of bytes transmitted through PCIe TX (in KB) via NVML.
    DCGM_FI_PROF_PCIE_RX_BYTES,  counter, Total number of bytes received through PCIe RX (in KB) via NVML.
    DCGM_FI_DEV_PCIE_REPLAY_COUNTER, counter, Total number of PCIe retries.

    # Utilization (the sample period varies depending on the product),,
    DCGM_FI_DEV_GPU_UTIL,      gauge, GPU utilization (in %).
    DCGM_FI_DEV_MEM_COPY_UTIL, gauge, Memory utilization (in %).
    DCGM_FI_DEV_ENC_UTIL,      gauge, Encoder utilization (in %).
    DCGM_FI_DEV_DEC_UTIL,      gauge, Decoder utilization (in %).

    # Errors and violations,,
    DCGM_FI_DEV_XID_ERRORS,            gauge, Value of the last XID error encountered.
    DCGM_FI_DEV_POWER_VIOLATION,       counter, Throttling duration due to power constraints (in us).
    DCGM_FI_DEV_THERMAL_VIOLATION,     counter, Throttling duration due to thermal constraints (in us).
    DCGM_FI_DEV_SYNC_BOOST_VIOLATION,  counter, Throttling duration due to sync-boost constraints (in us).
    DCGM_FI_DEV_BOARD_LIMIT_VIOLATION, counter, Throttling duration due to board limit constraints (in us).
    DCGM_FI_DEV_LOW_UTIL_VIOLATION,    counter, Throttling duration due to low utilization (in us).
    DCGM_FI_DEV_RELIABILITY_VIOLATION, counter, Throttling duration due to reliability constraints (in us).

    # Memory usage,,
    DCGM_FI_DEV_FB_FREE, gauge, Framebuffer memory free (in MiB).
    DCGM_FI_DEV_FB_USED, gauge, Framebuffer memory used (in MiB).

    # Retired pages,,
    DCGM_FI_DEV_RETIRED_SBE,     counter, Total number of retired pages due to single-bit errors.
    DCGM_FI_DEV_RETIRED_DBE,     counter, Total number of retired pages due to double-bit errors.
    DCGM_FI_DEV_RETIRED_PENDING, counter, Total number of pages pending retirement.

    # NVLink,,
    DCGM_FI_DEV_NVLINK_BANDWIDTH_TOTAL, counter, Total number of NVLink bandwidth counters for all lanes
    DCGM_FI_PROF_NVLINK_TX_BYTES,       counter, The rate of data transmitted over NVLink not including protocol headers in bytes per second.
    DCGM_FI_PROF_NVLINK_RX_BYTES,       counter, The rate of data received over NVLink not including protocol headers in bytes per second.

    # Add DCP metrics,,
    DCGM_FI_PROF_GR_ENGINE_ACTIVE,   gauge, Ratio of time the graphics engine is active (in %).
    DCGM_FI_PROF_SM_ACTIVE,          gauge, The ratio of cycles an SM has at least 1 warp assigned (in %).
    DCGM_FI_PROF_SM_OCCUPANCY,       gauge, The ratio of number of warps resident on an SM (in %).
    DCGM_FI_PROF_PIPE_TENSOR_ACTIVE, gauge, Ratio of cycles the tensor (HMMA) pipe is active (in %).
    DCGM_FI_PROF_DRAM_ACTIVE,        gauge, Ratio of cycles the device memory interface is active sending or receiving data (in %).
    DCGM_FI_PROF_PCIE_TX_BYTES,      counter, The number of bytes of active pcie tx data including both header and payload.
    DCGM_FI_PROF_PCIE_RX_BYTES,      counter, The number of bytes of active pcie rx data including both header and payload.

    # BCP Additional metrics
    DCGM_FI_DEV_GPU_NVLINK_ERRORS,      gauge, Identifies a GPU NVLink error type returned by DCGM_FI_DEV_GPU_NVLINK_ERRORS.

    # Added RunAI Additional metrics https://docs.run.ai/latest/developer/metrics/metrics-api/#advanced-metrics
    ## NVLink
    DCGM_FI_DEV_NVLINK_BANDWIDTH_L0, counter, The number of bytes of active NVLink rx or tx data including both header and payload.
    ## VGPU License status
    DCGM_FI_DEV_VGPU_LICENSE_STATUS, gauge, vGPU License status
    ## Remapped rows
    DCGM_FI_DEV_UNCORRECTABLE_REMAPPED_ROWS, counter, Number of remapped rows for uncorrectable errors
    DCGM_FI_DEV_CORRECTABLE_REMAPPED_ROWS, counter, Number of remapped rows for correctable errors
    DCGM_FI_DEV_ROW_REMAP_FAILURE, gauge, Whether remapping of rows has failed
    ## Static configuration information. These appear as labels on the other metrics
    DCGM_FI_DRIVER_VERSION, label, Driver Version
    ## Profiling metrics
    DCGM_FI_PROF_PIPE_FP64_ACTIVE, gauge, Ratio of cycles the fp64 pipes are active (in %).
    DCGM_FI_PROF_PIPE_FP32_ACTIVE, gauge, Ratio of cycles the fp32 pipes are active (in %).
    DCGM_FI_PROF_PIPE_FP16_ACTIVE, gauge, Ratio of cycles the fp16 pipes are active (in %).
kind: ConfigMap
metadata:
  annotations:
    meta.helm.sh/release-name: gpu-operator-gpu-operator-post
    meta.helm.sh/release-namespace: gpu-operator
  creationTimestamp: "2026-05-11T14:20:39Z"
  labels:
    app.kubernetes.io/managed-by: Helm
    helm.sh/chart: gpu-operator-post-0.1.0
    helm.toolkit.fluxcd.io/name: gpu-operator-post
    helm.toolkit.fluxcd.io/namespace: flux-system
  name: dcgm-exporter
  namespace: gpu-operator
  resourceVersion: "55664"
  uid: e78046f1-63b6-4e27-b78c-cf4b726226be

kubectl get skyhooks tuning -o yaml
apiVersion: skyhook.nvidia.com/v1alpha1
kind: Skyhook
metadata:
  annotations:
    helm.sh/hook: post-install,post-upgrade
    helm.sh/hook-delete-policy: before-hook-creation
    helm.sh/hook-weight: "10"
    skyhook.nvidia.com/version: v0.15.0
  creationTimestamp: "2026-05-11T14:32:45Z"
  finalizers:
  - skyhook.nvidia.com/skyhook
  generation: 1
  labels:
    app.kubernetes.io/created-by: aicr
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/part-of: skyhook-operator
    helm.sh/chart: nodewright-customizations-0.1.0
  name: tuning
  resourceVersion: "59868"
  uid: a5f24918-86ce-4112-b501-b300d693fcac
spec:
  additionalTolerations:
  - operator: Exists
  autoTaintNewNodes: true
  interruptionBudget: {}
  nodeSelectors: {}
  packages:
    nvidia-setup-full:
      configMap:
        accelerator: h100
        service: eks
      dependsOn:
        nvidia-tuned: 0.3.0
      env:
      - name: NVIDIA_SETUP_INSTALL_KERNEL
        value: "false"
      image: ghcr.io/nvidia/nodewright-packages/nvidia-setup
      interrupt:
        type: reboot
      name: nvidia-setup-full
      resources:
        cpuLimit: "4"
        cpuRequest: "2"
        memoryLimit: 8Gi
        memoryRequest: 4Gi
      version: 0.2.2
    nvidia-setup-kernel:
      configMap:
        accelerator: h100
        service: eks
      env:
      - name: NVIDIA_SETUP_INSTALL_KERNEL
        value: "true"
      - name: NVIDIA_SETUP_KERNEL_ALLOW_NEWER
        value: "false"
      image: ghcr.io/nvidia/nodewright-packages/nvidia-setup
      name: nvidia-setup-kernel
      resources:
        cpuLimit: "4"
        cpuRequest: "2"
        memoryLimit: 8Gi
        memoryRequest: 4Gi
      version: 0.2.2
    nvidia-tuned:
      configMap:
        accelerator: h100
        intent: multiNodeTraining
        service: eks
      dependsOn:
        nvidia-setup-kernel: 0.2.2
      env:
      - name: INTERRUPT
        value: "true"
      image: ghcr.io/nvidia/nodewright-packages/nvidia-tuned
      interrupt:
        type: reboot
      name: nvidia-tuned
      version: 0.3.0
  podNonInterruptLabels: {}
  priority: 200
  runtimeRequired: true
  sequencing: node
  serial: false
[...]

aicr bundle \
  -r recipe.yaml \
  --deployer flux \
  --repo https://github.com/haarchri/aicr-flux-bundle \
  --dynamic gpuoperator:driver.version \
  --dynamic gpuoperator:driver.rdma.enabled \
  --accelerated-node-selector nvidia.com/gpu.present=true \
  --workload-selector app.kubernetes.io/name=training-job \
  -o /tmp/flux-bundle