Skip to content
This repository was archived by the owner on Jun 24, 2020. It is now read-only.

Commit 86d46bd

Browse files
jihuinjihuin
committed
Added stats reporter to emit metrics when reconciling knative serving
1 parent 1b8e709 commit 86d46bd

File tree

4 files changed

+245
-0
lines changed

4 files changed

+245
-0
lines changed

‎pkg/reconciler/knativeserving/knativeserving_controller.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ package knativeserving
1818

1919
import (
2020
"context"
21+
"time"
2122

2223
mf "github.com/jcrossley3/manifestival"
2324
"go.uber.org/zap"
@@ -60,6 +61,7 @@ var _ controller.Reconciler = (*Reconciler)(nil)
6061
// converge the two. It then updates the Status block of the Knativeserving resource
6162
// with the current status of the resource.
6263
func (r *Reconciler) Reconcile(ctx context.Context, key string) error {
64+
tStart := time.Now()
6365
// Convert the namespace/name string into a distinct namespace and name
6466
namespace, name, err := cache.SplitMetaNamespaceKey(key)
6567
if err != nil {
@@ -104,6 +106,8 @@ func (r *Reconciler) Reconcile(ctx context.Context, key string) error {
104106
r.Recorder.Event(knativeServing, corev1.EventTypeWarning, "InternalError", reconcileErr.Error())
105107
return reconcileErr
106108
}
109+
110+
r.StatsReporter.ReportReconcile(namespace, name, time.Since(tStart))
107111
return nil
108112
}
109113

‎pkg/reconciler/reconciler.go‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,9 @@ type Base struct {
6262
// Kubernetes API.
6363
Recorder record.EventRecorder
6464

65+
// StatsReporter reports reconciler's metrics.
66+
StatsReporter StatsReporter
67+
6568
// Sugared logger is easier to use but is not as performant as the
6669
// raw logger. In performance critical paths, call logger.Desugar()
6770
// and use the returned raw logger instead. In addition to the
@@ -100,13 +103,24 @@ func NewBase(ctx context.Context, controllerAgentName string, cmw configmap.Watc
100103
}()
101104
}
102105

106+
statsReporter := GetStatsReporter(ctx)
107+
if statsReporter == nil {
108+
logger.Debug("Creating stats reporter")
109+
var err error
110+
statsReporter, err = NewStatsReporter(controllerAgentName)
111+
if err != nil {
112+
logger.Fatal(err)
113+
}
114+
}
115+
103116
base := &Base{
104117
KubeClientSet: kubeClient,
105118
SharedClientSet: sharedclient.Get(ctx),
106119
KnativeServingClientSet: servingclient.Get(ctx),
107120
DynamicClientSet: dynamicclient.Get(ctx),
108121
ConfigMapWatcher: cmw,
109122
Recorder: recorder,
123+
StatsReporter: statsReporter,
110124
Logger: logger,
111125
}
112126

‎pkg/reconciler/stats_reporter.go‎

Lines changed: 136 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,136 @@
1+
/*
2+
Copyright 2018 The Knative Authors
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package reconciler
18+
19+
import (
20+
"context"
21+
"fmt"
22+
"time"
23+
24+
"go.opencensus.io/stats"
25+
"go.opencensus.io/stats/view"
26+
"go.opencensus.io/tag"
27+
"knative.dev/pkg/metrics"
28+
)
29+
30+
const (
31+
// reconcileCountN is the number of successful reconcile operations.
32+
reconcileCountN = "knative_operator_reconcile_count"
33+
// reconcileLatencyN is the time it takes for a sucessful reconcile operation.
34+
reconcileLatencyN = "knative_operator_reconcile_latency"
35+
)
36+
37+
var (
38+
reconcileCountStat = stats.Int64(
39+
reconcileCountN,
40+
"Number of reconcile operations",
41+
stats.UnitNone)
42+
reconcileLatencyStat = stats.Int64(
43+
reconcileLatencyN,
44+
"Latency of reconcile operations",
45+
stats.UnitMilliseconds)
46+
47+
// Create the tag keys that will be used to add tags to our measurements.
48+
// Tag keys must conform to the restrictions described in
49+
// go.opencensus.io/tag/validate.go. Currently those restrictions are:
50+
// - length between 1 and 255 inclusive
51+
// - characters are printable US-ASCII
52+
reconcilerTagKey = tag.MustNewKey("reconciler")
53+
keyTagKey = tag.MustNewKey("key")
54+
)
55+
56+
func init() {
57+
// Create views to see our measurements. This can return an error if
58+
// a previously-registered view has the same name with a different value.
59+
// View name defaults to the measure name if unspecified.
60+
if err := view.Register(
61+
&view.View{
62+
Description: "Number of reconcile operations",
63+
Measure: reconcileCountStat,
64+
Aggregation: view.Count(),
65+
TagKeys: []tag.Key{reconcilerTagKey, keyTagKey},
66+
},
67+
&view.View{
68+
Description: "Latency of reconcile operations",
69+
Measure: reconcileLatencyStat,
70+
Aggregation: view.LastValue(),
71+
TagKeys: []tag.Key{reconcilerTagKey, keyTagKey},
72+
},
73+
); err != nil {
74+
panic(err)
75+
}
76+
}
77+
78+
// StatsReporter defines the interface for sending metrics
79+
type StatsReporter interface {
80+
// ReportReconcile reports the count and latency metrics for a reconcile operation
81+
ReportReconcile(resourceNamespace, resourceName string, duration time.Duration) error
82+
}
83+
84+
// Reporter holds cached metric objects to report metrics
85+
type reporter struct {
86+
reconciler string
87+
ctx context.Context
88+
}
89+
90+
// srKey is used to associate StatsReporters with contexts.
91+
type srKey struct{}
92+
93+
// WithStatsReporter attaches the given StatsReporter to the provided context
94+
// in the returned context.
95+
func WithStatsReporter(ctx context.Context, sr StatsReporter) context.Context {
96+
return context.WithValue(ctx, srKey{}, sr)
97+
}
98+
99+
// GetStatsReporter attempts to look up the StatsReporter on a given context.
100+
// It may return null if none is found.
101+
func GetStatsReporter(ctx context.Context) StatsReporter {
102+
untyped := ctx.Value(srKey{})
103+
if untyped == nil {
104+
return nil
105+
}
106+
return untyped.(StatsReporter)
107+
}
108+
109+
// NewStatsReporter creates a reporter that collects and reports metrics
110+
func NewStatsReporter(reconciler string) (StatsReporter, error) {
111+
// Reconciler tag is static. Create a context containing that and cache it.
112+
ctx, err := tag.New(
113+
context.Background(),
114+
tag.Insert(reconcilerTagKey, reconciler))
115+
if err != nil {
116+
return nil, err
117+
}
118+
119+
return &reporter{reconciler: reconciler, ctx: ctx}, nil
120+
}
121+
122+
// ReportReconcile reports the count and latency metrics for a reconcile operation
123+
func (r *reporter) ReportReconcile(resourceNamespace, resourceName string, duration time.Duration) error {
124+
key := fmt.Sprintf("%s/%s", resourceNamespace, resourceName)
125+
ctx, err := tag.New(
126+
context.Background(),
127+
tag.Insert(reconcilerTagKey, r.reconciler),
128+
tag.Insert(keyTagKey, key))
129+
if err != nil {
130+
return err
131+
}
132+
133+
metrics.Record(ctx, reconcileCountStat.M(1))
134+
metrics.Record(ctx, reconcileLatencyStat.M(duration.Milliseconds()))
135+
return nil
136+
}

‎pkg/reconciler/stats_reporter_test.go‎

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
/*
2+
Copyright 2018 The Knative Authors
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package reconciler
18+
19+
import (
20+
"context"
21+
"fmt"
22+
"testing"
23+
"time"
24+
25+
"go.opencensus.io/stats/view"
26+
"go.opencensus.io/tag"
27+
28+
"knative.dev/pkg/metrics/metricstest"
29+
)
30+
31+
const (
32+
reconcilerMockName = "mock_reconciler"
33+
testResourceNamespace = "test_namespace"
34+
testResourceName = "test_resource"
35+
)
36+
37+
func TestNewStatsReporter(t *testing.T) {
38+
r, err := NewStatsReporter(reconcilerMockName)
39+
if err != nil {
40+
t.Errorf("Failed to create reporter: %v", err)
41+
}
42+
43+
m := tag.FromContext(r.(*reporter).ctx)
44+
v, ok := m.Value(reconcilerTagKey)
45+
if !ok {
46+
t.Fatalf("Expected tag %q", reconcilerTagKey)
47+
}
48+
if v != reconcilerMockName {
49+
t.Fatalf("Expected %q for tag %q, got %q", reconcilerMockName, reconcilerTagKey, v)
50+
}
51+
}
52+
53+
func TestReporter_ReportDuration(t *testing.T) {
54+
reporter, err := NewStatsReporter(reconcilerMockName)
55+
if err != nil {
56+
t.Errorf("Failed to create reporter: %v", err)
57+
}
58+
countWas := int64(0)
59+
if m := getMetric(t, reconcileCountN); m != nil {
60+
countWas = m.Data.(*view.CountData).Value
61+
}
62+
63+
if err = reporter.ReportReconcile(testResourceNamespace, testResourceName, time.Second); err != nil {
64+
t.Error(err)
65+
}
66+
expectedTags := map[string]string{
67+
keyTagKey.Name(): fmt.Sprintf("%s/%s", testResourceNamespace, testResourceName),
68+
reconcilerTagKey.Name(): reconcilerMockName,
69+
}
70+
71+
metricstest.CheckCountData(t, reconcileCountN, expectedTags, countWas+1)
72+
metricstest.CheckLastValueData(t, reconcileLatencyN, expectedTags, 1000)
73+
}
74+
75+
func getMetric(t *testing.T, metric string) *view.Row {
76+
t.Helper()
77+
rows, err := view.RetrieveData(metric)
78+
if err != nil {
79+
t.Errorf("Failed retrieving data: %v", err)
80+
}
81+
if len(rows) == 0 {
82+
return nil
83+
}
84+
return rows[0]
85+
}
86+
87+
func TestWithStatsReporter(t *testing.T) {
88+
if WithStatsReporter(context.TODO(), nil) == nil {
89+
t.Errorf("stats reporter reports empty context")
90+
}
91+
}

0 commit comments

Comments
 (0)