mixin: exclude iowait and steal from CPU Utilisation (#2194)

julianwiedmann · web-flow · commit 3e6f4ce627e5 · 2021-11-04T11:03:27.000+01:00
'iowait' and 'steal' indicate specific idle/wait states, which shouldn't be counted into CPU Utilisation. Also see prometheus-operator/kube-prometheus#796 and kubernetes-monitoring/kubernetes-mixin#667. Per the iostat man page: %idle Show the percentage of time that the CPU or CPUs were idle and the system did not have an outstanding disk I/O request. %iowait Show the percentage of time that the CPU or CPUs were idle during which the system had an outstanding disk I/O request. %steal Show the percentage of time spent in involuntary wait by the virtual CPU or CPUs while the hypervisor was servicing another virtual processor. Signed-off-by: Julian Wiedmann <jwi@linux.ibm.com>
diff --git a/docs/node-mixin/dashboards/node.libsonnet b/docs/node-mixin/dashboards/node.libsonnet
@@ -23,9 +23,9 @@ local gauge = promgrafonnet.gauge;
         .addTarget(prometheus.target(
           |||
             (
-              (1 - rate(node_cpu_seconds_total{%(nodeExporterSelector)s, mode="idle", instance="$instance"}[$__rate_interval]))
+              (1 - sum without (mode) (rate(node_cpu_seconds_total{%(nodeExporterSelector)s, mode=~"idle|iowait|steal", instance="$instance"}[$__rate_interval])))
             / ignoring(cpu) group_left
-              count without (cpu)( node_cpu_seconds_total{%(nodeExporterSelector)s, mode="idle", instance="$instance"})
+              count without (cpu, mode) (node_cpu_seconds_total{%(nodeExporterSelector)s, mode="idle", instance="$instance"})
             )
           ||| % $._config,
           legendFormat='{{cpu}}',
diff --git a/docs/node-mixin/rules/rules.libsonnet b/docs/node-mixin/rules/rules.libsonnet
@@ -14,11 +14,11 @@
             ||| % $._config,
           },
           {
-            // CPU utilisation is % CPU is not idle.
+            // CPU utilisation is % CPU without {idle,iowait,steal}.
             record: 'instance:node_cpu_utilisation:rate%(rateInterval)s' % $._config,
             expr: |||
-              1 - avg without (cpu, mode) (
-                rate(node_cpu_seconds_total{%(nodeExporterSelector)s, mode="idle"}[%(rateInterval)s])
+              1 - avg without (cpu) (
+                sum without (mode) (rate(node_cpu_seconds_total{%(nodeExporterSelector)s, mode=~"idle|iowait|steal"}[%(rateInterval)s]))
               )
             ||| % $._config,
           },

Original file line number	Diff line number	Diff line change
`@@ -23,9 +23,9 @@ local gauge = promgrafonnet.gauge;`
`23`	`23`	`.addTarget(prometheus.target(`
`24`	`24`	`\|\|\|`
`25`	`25`	`(`
`26`		`- (1 - rate(node_cpu_seconds_total{%(nodeExporterSelector)s, mode="idle", instance="$instance"}[$__rate_interval]))`
	`26`	`+ (1 - sum without (mode) (rate(node_cpu_seconds_total{%(nodeExporterSelector)s, mode=~"idle\|iowait\|steal", instance="$instance"}[$__rate_interval])))`
`27`	`27`	`/ ignoring(cpu) group_left`
`28`		`- count without (cpu)( node_cpu_seconds_total{%(nodeExporterSelector)s, mode="idle", instance="$instance"})`
	`28`	`+ count without (cpu, mode) (node_cpu_seconds_total{%(nodeExporterSelector)s, mode="idle", instance="$instance"})`
`29`	`29`	`)`
`30`	`30`	`\|\|\| % $._config,`
`31`	`31`	`legendFormat='{{cpu}}',`