Skip to content

Commit 877d2a7

Browse files
authored
go.d/k8s_state: remove only the deleted object's charts (netdata#24402)
1 parent 8db4d10 commit 877d2a7

2 files changed

Lines changed: 193 additions & 12 deletions

File tree

‎src/go/plugin/go.d/collector/k8s_state/charts.go‎

Lines changed: 17 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -464,8 +464,7 @@ func (c *Collector) addNodeCharts(ns *nodeState) {
464464
}
465465

466466
func (c *Collector) removeNodeCharts(ns *nodeState) {
467-
prefix := fmt.Sprintf("node_%s", replaceDots(ns.id()))
468-
c.removeCharts(prefix)
467+
c.removeCharts("node", ns.id())
469468
}
470469

471470
var (
@@ -656,9 +655,8 @@ func (c *Collector) addPodCharts(ps *podState) {
656655
}
657656

658657
func (c *Collector) updatePodChartsNodeLabel(ps *podState) {
659-
prefix := fmt.Sprintf("pod_%s", replaceDots(ps.id()))
660658
for _, c := range *c.Charts() {
661-
if strings.HasPrefix(c.ID, prefix) {
659+
if isObjectChart(c.ID, "pod", ps.id()) {
662660
updateNodeLabel(c, ps.nodeName)
663661
c.MarkNotCreated()
664662
}
@@ -675,8 +673,7 @@ func updateNodeLabel(c *collectorapi.Chart, nodeName string) {
675673
}
676674

677675
func (c *Collector) removePodCharts(ps *podState) {
678-
prefix := fmt.Sprintf("pod_%s", replaceDots(ps.id()))
679-
c.removeCharts(prefix)
676+
c.removeCharts("pod", ps.id())
680677
}
681678

682679
var (
@@ -848,8 +845,7 @@ func (c *Collector) addDeploymentCharts(rs *deploymentState) {
848845
}
849846

850847
func (c *Collector) removeDeploymentCharts(st *deploymentState) {
851-
prefix := fmt.Sprintf("deployment_%s", replaceDots(st.id()))
852-
c.removeCharts(prefix)
848+
c.removeCharts("deployment", st.id())
853849
}
854850

855851
var (
@@ -980,19 +976,28 @@ func (c *Collector) addCronJobCharts(st *cronJobState) {
980976
}
981977

982978
func (c *Collector) removeCronJobCharts(st *cronJobState) {
983-
prefix := fmt.Sprintf("cronjob_%s", replaceDots(st.id()))
984-
c.removeCharts(prefix)
979+
c.removeCharts("cronjob", st.id())
985980
}
986981

987-
func (c *Collector) removeCharts(prefix string) {
982+
func (c *Collector) removeCharts(kind, id string) {
988983
for _, c := range *c.Charts() {
989-
if strings.HasPrefix(c.ID, prefix) {
984+
if isObjectChart(c.ID, kind, id) {
990985
c.MarkRemove()
991986
c.MarkNotCreated()
992987
}
993988
}
994989
}
995990

991+
// isObjectChart reports whether chartID belongs to the object: "<kind>_<id>.<chart>", or for pods also
992+
// "pod_<id>_container_<name>.<chart>". The end marker keeps "x-1" from matching the charts of "x-10".
993+
func isObjectChart(chartID, kind, id string) bool {
994+
prefix := kind + "_" + replaceDots(id)
995+
if strings.HasPrefix(chartID, prefix+".") {
996+
return true
997+
}
998+
return kind == "pod" && strings.HasPrefix(chartID, prefix+"_container_")
999+
}
1000+
9961001
var discoveryStatusChart = collectorapi.Chart{
9971002
ID: "discovery_discoverers_state",
9981003
Title: "Running discoverers state",

‎src/go/plugin/go.d/collector/k8s_state/collector_test.go‎

Lines changed: 176 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -954,6 +954,173 @@ func TestCollector_Collect(t *testing.T) {
954954
}
955955
}
956956

957+
func TestCollector_Collect_DeleteObjectKeepsLongerNamedSibling(t *testing.T) {
958+
// Deleting "x-1" must obsolete only the charts of "x-1", not those of the live "x-10".
959+
// The expected chart sets come from collectors that only ever saw one of the two objects.
960+
const deleted, live = "x-1", "x-10"
961+
ctx := context.Background()
962+
963+
tests := map[string]struct {
964+
objects func(names ...string) []runtime.Object
965+
delete func(client kubernetes.Interface, name string) error
966+
}{
967+
"Node": {
968+
objects: func(names ...string) []runtime.Object {
969+
var objs []runtime.Object
970+
for _, name := range names {
971+
objs = append(objs, newNode(name))
972+
}
973+
return objs
974+
},
975+
delete: func(client kubernetes.Interface, name string) error {
976+
return client.CoreV1().Nodes().Delete(ctx, name, metav1.DeleteOptions{})
977+
},
978+
},
979+
"Pod": {
980+
objects: func(names ...string) []runtime.Object {
981+
objs := []runtime.Object{newNode("node01")}
982+
for _, name := range names {
983+
objs = append(objs, newPod("node01", name))
984+
}
985+
return objs
986+
},
987+
delete: func(client kubernetes.Interface, name string) error {
988+
return client.CoreV1().Pods(corev1.NamespaceDefault).Delete(ctx, name, metav1.DeleteOptions{})
989+
},
990+
},
991+
"Deployment": {
992+
objects: func(names ...string) []runtime.Object {
993+
var objs []runtime.Object
994+
for _, name := range names {
995+
objs = append(objs, newDeployment(name))
996+
}
997+
return objs
998+
},
999+
delete: func(client kubernetes.Interface, name string) error {
1000+
return client.AppsV1().Deployments(corev1.NamespaceDefault).Delete(ctx, name, metav1.DeleteOptions{})
1001+
},
1002+
},
1003+
"CronJob": {
1004+
objects: func(names ...string) []runtime.Object {
1005+
var objs []runtime.Object
1006+
for _, name := range names {
1007+
objs = append(objs, prepareCronJob(name))
1008+
}
1009+
return objs
1010+
},
1011+
delete: func(client kubernetes.Interface, name string) error {
1012+
return client.BatchV1().CronJobs(corev1.NamespaceDefault).Delete(ctx, name, metav1.DeleteOptions{})
1013+
},
1014+
},
1015+
}
1016+
1017+
for name, test := range tests {
1018+
t.Run(name, func(t *testing.T) {
1019+
t.Parallel()
1020+
1021+
client := fake.NewClientset(test.objects(deleted, live)...)
1022+
collrs := startCollectors(t, client,
1023+
fake.NewClientset(test.objects(deleted)...),
1024+
fake.NewClientset(test.objects(live)...),
1025+
)
1026+
collr, deletedOnly, liveOnly := collrs[0], collrs[1], collrs[2]
1027+
1028+
expected := make(map[string]bool) // chart ID -> obsolete
1029+
for _, c := range *deletedOnly.Charts() {
1030+
expected[c.ID] = true
1031+
}
1032+
// Runs second: charts both collectors have (discovery, node01) must stay active.
1033+
for _, c := range *liveOnly.Charts() {
1034+
expected[c.ID] = false
1035+
}
1036+
var own int
1037+
for _, obsolete := range expected {
1038+
if obsolete {
1039+
own++
1040+
}
1041+
}
1042+
require.NotZero(t, own, "the deleted object must have charts")
1043+
1044+
require.NoError(t, test.delete(client, deleted))
1045+
require.Eventually(t, func() bool {
1046+
_ = collr.Collect(ctx)
1047+
for id, obsolete := range expected {
1048+
if c := collr.Charts().Get(id); obsolete && (c == nil || !c.Obsolete) {
1049+
return false
1050+
}
1051+
}
1052+
return true
1053+
}, 5*time.Second, 50*time.Millisecond, "the deleted object's charts were not removed")
1054+
1055+
actual := make(map[string]bool)
1056+
for _, c := range *collr.Charts() {
1057+
actual[c.ID] = c.Obsolete
1058+
}
1059+
1060+
assert.Equal(t, expected, actual)
1061+
})
1062+
}
1063+
}
1064+
1065+
func TestCollector_Collect_ScheduledPodUpdatesOnlyItsNodeLabel(t *testing.T) {
1066+
// Scheduling the pending "x-1" must not rewrite the node label of the live "x-10".
1067+
ctx := context.Background()
1068+
pending := newPod("node01", "x-1")
1069+
pending.Spec.NodeName = ""
1070+
client := fake.NewClientset(newNode("node01"), newNode("node02"), pending, newPod("node02", "x-10"))
1071+
collr := startCollectors(t, client)[0]
1072+
1073+
_, err := client.CoreV1().Pods(pending.Namespace).Update(ctx, newPod("node01", "x-1"), metav1.UpdateOptions{})
1074+
require.NoError(t, err)
1075+
require.Eventually(t, func() bool {
1076+
_ = collr.Collect(ctx)
1077+
for _, c := range *collr.Charts() {
1078+
if chartLabelValue(c, labelKeyPodName) == "x-1" && chartLabelValue(c, labelKeyNodeName) != "node01" {
1079+
return false
1080+
}
1081+
}
1082+
return true
1083+
}, 5*time.Second, 50*time.Millisecond, "the scheduled pod's charts did not get its node label")
1084+
1085+
podNode := map[string]string{"x-1": "node01", "x-10": "node02"}
1086+
seen := make(map[string]bool)
1087+
expected := make(map[string]string) // chart ID -> node label
1088+
actual := make(map[string]string)
1089+
for _, c := range *collr.Charts() {
1090+
if pod := chartLabelValue(c, labelKeyPodName); pod != "" {
1091+
seen[pod] = true
1092+
expected[c.ID] = podNode[pod]
1093+
actual[c.ID] = chartLabelValue(c, labelKeyNodeName)
1094+
}
1095+
}
1096+
1097+
require.Equal(t, map[string]bool{"x-1": true, "x-10": true}, seen, "both pods must have charts")
1098+
assert.Equal(t, expected, actual)
1099+
}
1100+
1101+
// startCollectors starts one collector per client and waits once for all of them to pass the initial delay.
1102+
func startCollectors(t *testing.T, clients ...kubernetes.Interface) []*Collector {
1103+
t.Helper()
1104+
ctx := context.Background()
1105+
1106+
var collrs []*Collector
1107+
for _, client := range clients {
1108+
collr := New()
1109+
collr.newKubeClient = func() (kubernetes.Interface, error) { return client, nil }
1110+
require.NoError(t, collr.Init(ctx))
1111+
require.NoError(t, collr.Check(ctx))
1112+
t.Cleanup(func() { collr.Cleanup(ctx) })
1113+
_ = collr.Collect(ctx)
1114+
collrs = append(collrs, collr)
1115+
}
1116+
1117+
time.Sleep(collrs[0].initDelay)
1118+
for _, collr := range collrs {
1119+
_ = collr.Collect(ctx)
1120+
}
1121+
return collrs
1122+
}
1123+
9571124
func newNode(name string) *corev1.Node {
9581125
return &corev1.Node{
9591126
ObjectMeta: metav1.ObjectMeta{
@@ -1212,3 +1379,12 @@ func isLabelValueSet(c *collectorapi.Chart, name string) bool {
12121379
}
12131380
return false
12141381
}
1382+
1383+
func chartLabelValue(c *collectorapi.Chart, name string) string {
1384+
for _, l := range c.Labels {
1385+
if l.Key == name {
1386+
return l.Value
1387+
}
1388+
}
1389+
return ""
1390+
}

0 commit comments

Comments
 (0)