Skip to content

Commit d652862

Browse files
committed
fix(observability-proxy): keep the vmselect timeout below the request's
An equal vmselect timeout still lost the race: the proxy's deadline starts before the request reaches vmselect, so the proxy cut first and answered 504 for a store within its time. The vmselect timeout must now be strictly below a set request timeout, the flag help says so, and tests pin the accepted side too, including no request timeout. Assisted-by: LLM Signed-off-by: ZverGuy <[email protected]>
1 parent 8b1da2e commit d652862

2 files changed

Lines changed: 20 additions & 5 deletions

File tree

‎cmd/observability-proxy/main.go‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -91,7 +91,7 @@ func parseFlags(args []string) (options, error) {
9191
fs.DurationVar(&o.metrics.ShortTermRetention, "metrics-shortterm-retention", 72*time.Hour, "retention of the short-term store; older windows are read from the long-term one")
9292
fs.DurationVar(&o.metrics.Limits.MaxRange, "metrics-max-range", 30*24*time.Hour, "longest [start, end] and step a metrics request may use")
9393
fs.IntVar(&o.metrics.Limits.MaxPoints, "metrics-max-points", 11000, "most points per series a range query may return")
94-
fs.DurationVar(&o.metrics.Limits.MaxTimeout, "metrics-max-timeout", 30*time.Second, "timeout forwarded to vmselect; a longer client timeout is clamped")
94+
fs.DurationVar(&o.metrics.Limits.MaxTimeout, "metrics-max-timeout", 30*time.Second, "timeout forwarded to vmselect; a longer client timeout is clamped; must be below --request-timeout when that is set")
9595
fs.IntVar(&o.limiter.MaxConcurrentPerUser, "max-concurrent-per-user", 10, "concurrent non-tail requests per user; 0 disables")
9696
fs.Float64Var(&o.limiter.RatePerUser, "rate-limit-per-user", 10, "sustained requests per second per user; 0 disables")
9797
fs.IntVar(&o.limiter.RateBurst, "rate-limit-burst", 20, "request burst per user")
@@ -134,10 +134,11 @@ func parseFlags(args []string) (options, error) {
134134
return o, errors.New("--metrics-max-points must be at least 2")
135135
case o.logs.MaxTailResume < 0:
136136
return o, errors.New("--logs-max-tail-resume must not be negative")
137-
// Past the request timeout the proxy cuts a query vmselect is still
138-
// allowed to run, and answers 504 for a store that is within its time.
139-
case o.requestTimeout > 0 && o.metrics.Limits.MaxTimeout > o.requestTimeout:
140-
return o, errors.New("--metrics-max-timeout must not exceed --request-timeout")
137+
// The proxy's deadline starts before the request reaches vmselect, so
138+
// at an equal or longer timeout it cuts a query vmselect is still
139+
// allowed to run and answers 504 for a store that is within its time.
140+
case o.requestTimeout > 0 && o.metrics.Limits.MaxTimeout >= o.requestTimeout:
141+
return o, errors.New("--metrics-max-timeout must be below --request-timeout")
141142
case o.requestTimeout < 0 || o.tailIdleTimeout < 0 || o.authCacheTTL < 0 || o.shutdownTimeout < 0:
142143
return o, errors.New("timeouts and the auth cache TTL must not be negative")
143144
// NaN passes every comparison and +Inf never empties a bucket; either

‎cmd/observability-proxy/main_test.go‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -106,10 +106,24 @@ func TestParseFlagsInvalid(t *testing.T) {
106106
// The proxy would cut the request while vmselect is still within
107107
// the timeout it was given.
108108
{"--request-timeout=10s", "--metrics-max-timeout=30s"},
109+
{"--request-timeout=30s", "--metrics-max-timeout=30s"},
109110
{"unexpected", "--root-tenant=tenant-x"},
110111
} {
111112
if _, err := parseFlags(args); err == nil {
112113
t.Errorf("parseFlags(%v) accepted", args)
113114
}
114115
}
115116
}
117+
118+
// The vmselect timeout must stay strictly below a request timeout, and is
119+
// free when the request has none.
120+
func TestParseFlagsTimeoutPairs(t *testing.T) {
121+
for _, args := range [][]string{
122+
{"--request-timeout=31s", "--metrics-max-timeout=30s"},
123+
{"--request-timeout=0", "--metrics-max-timeout=10m"},
124+
} {
125+
if _, err := parseFlags(args); err != nil {
126+
t.Errorf("parseFlags(%v) refused: %v", args, err)
127+
}
128+
}
129+
}

0 commit comments

Comments
 (0)