Skip to content

Commit 16cee4d

Browse files
kitsunoffkitsunoff
committed
fix(observability-proxy): forward the default instant query time
An instant query without time had its application lookup sized at the proxy's clock, but the time was not forwarded, so vmselect evaluated at its own later clock. With lookups taking most of the request deadline, the evaluation lands well past the end of the lookup and leaves out pods first sampled in between. Forward the instant the lookup was sized for. Assisted-by: LLM Signed-off-by: ZverGuy <[email protected]>
1 parent 4052e05 commit 16cee4d

3 files changed

Lines changed: 26 additions & 2 deletions

File tree

‎internal/observabilityproxy/metricsparams.go‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -134,8 +134,11 @@ func BuildMetricsParams(endpoint string, in url.Values, namespace string, now ti
134134
return req, herr
135135
}
136136
at = t
137-
out.Set("time", formatTime(t))
138137
}
138+
// The default is forwarded too: the application lookup is sized
139+
// for at, and vmselect's own clock reads later, by as long as the
140+
// lookups took.
141+
out.Set("time", formatTime(at))
139142
step := instantDefaultStep
140143
if v := in.Get("step"); v != "" {
141144
var herr *HTTPError

‎internal/observabilityproxy/metricsparams_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -227,7 +227,7 @@ func TestBuildMetricsParamsCanonicalForms(t *testing.T) {
227227
if req.params.Get("end") != "2026-11-04T12:00:00.000Z" || req.params.Get("start") != "2026-11-04T11:55:00.000Z" {
228228
t.Errorf("labels window = %s..%s, want the last 5m", req.params.Get("start"), req.params.Get("end"))
229229
}
230-
if q := buildMetrics(t, "api/v1/query", url.Values{"query": {"up"}}); q.params.Has("time") || !q.start.Equal(testNow.Add(-15*time.Minute-2*latencyOffset)) {
230+
if q := buildMetrics(t, "api/v1/query", url.Values{"query": {"up"}}); q.params.Get("time") != "2026-11-04T12:00:00.000Z" || !q.start.Equal(testNow.Add(-15*time.Minute-2*latencyOffset)) {
231231
t.Errorf("instant query without time: params %v, start %v", q.params, q.start)
232232
}
233233
}

‎internal/observabilityproxy/server_app_test.go‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -363,6 +363,27 @@ func TestServerApplicationLookupCoversNegativeOffset(t *testing.T) {
363363
}
364364
}
365365

366+
// TestServerApplicationForwardsDefaultTime: without time the lookup is
367+
// sized at the proxy's clock, so that instant is forwarded. Otherwise
368+
// vmselect evaluates at its own later clock, past the end of the lookup by
369+
// however long the lookups took, and leaves out pods first sampled since.
370+
func TestServerApplicationForwardsDefaultTime(t *testing.T) {
371+
h := newPolicyHarness(t, Config{}, appMetricsUpstream(nil, nil, http.StatusOK), appPolicy, nil)
372+
q := url.Values{"query": {"container_cpu_usage_seconds_total"}}
373+
if got := h.status(http.MethodGet, appRoot+"db.main/metrics/api/v1/query?"+q.Encode(), "good"); got != http.StatusOK {
374+
t.Fatalf("status = %d", got)
375+
}
376+
reqs := h.requests()
377+
lookup, final := reqs[0], reqs[len(reqs)-1]
378+
want := formatTime(testNow)
379+
if lookup.path != "/api/v1/label/pod/values" || lookup.form.Get("end") != want {
380+
t.Fatalf("lookup %s end = %s, want %s", lookup.path, lookup.form.Get("end"), want)
381+
}
382+
if final.path != "/api/v1/query" || final.form.Get("time") != want {
383+
t.Fatalf("query %s time = %q, want the lookup's instant %s", final.path, final.form.Get("time"), want)
384+
}
385+
}
386+
366387
// TestServerApplicationRefusesUnboundedSampleTime refuses a query whose
367388
// samples may lie at a time the lookup cannot be sized for, rather than
368389
// answer with series silently left out.

0 commit comments

Comments
 (0)