Skip to content

Commit a37f412

Browse files
committed
fix(db-autoscaler): address review — RBAC scope, metrics TLS, quorum & backoff
Resolves the CodeRabbit review threads on #3381: - RBAC: the apps.cozystack.io rule granted patch on "*"; scope it to the four database kinds the adapters can target (postgreses, mariadbs, redises, mongodbs) so the controller cannot touch unrelated aggregated-API resources. - Metrics TLS: the VMServiceScrape used insecureSkipVerify. Serve metrics with the cert-manager serving cert (the one already mounted for the webhook) and verify it in the scrape against its CA, with serverName matching a cert SAN (no metrics-service DNS name needed, since the pod is reached by IP but TLS verifies serverName). - Quorum: the postgres values schema does not lower-bound maxSyncReplicas, so a negative value produced a quorum floor below 1 and silently disabled the floor guard. Report it as non-scalable (AbleToScale=False) and clamp QuorumFloor to a minimum of 1 defensively. - Backoff overflow: cap ConvergenceDeadlineSeconds at 24h in the API and clamp the base in backoffDuration so the 8x re-attempt ceiling cannot overflow a time.Duration. - README: qualify the Postgres/Redis validation claims (functionally validated on a live dev cluster, not load-tested at production scale). Regression tests: negative-maxSyncReplicas (non-scalable + clamped floor), backoffDuration no-overflow, and helm-unittest assertions for the least- privilege RBAC and the verified metrics TLS. CRD regenerated for the new maximum. Signed-off-by: Alexey Artamonov <[email protected]>
1 parent 991fc73 commit a37f412

11 files changed

Lines changed: 134 additions & 10 deletions

File tree

‎api/autoscaling/v1alpha1/databasehorizontalautoscaler_types.go‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -148,8 +148,11 @@ type Behavior struct {
148148

149149
// ConvergenceDeadlineSeconds bounds how long a patched scale may take to
150150
// converge (availableReplicas == replicas) before the operator surfaces
151-
// StuckScaling and rolls back to lastConvergedReplicas.
151+
// StuckScaling and rolls back to lastConvergedReplicas. Capped at 24h so the
152+
// exponential re-attempt backoff (up to 8x this value) cannot overflow a
153+
// time.Duration.
152154
// +kubebuilder:validation:Minimum=1
155+
// +kubebuilder:validation:Maximum=86400
153156
// +optional
154157
ConvergenceDeadlineSeconds *int32 `json:"convergenceDeadlineSeconds,omitempty"`
155158
}

‎cmd/db-autoscaler/main.go‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -112,6 +112,13 @@ func main() {
112112
}
113113
if secureMetrics {
114114
metricsServerOptions.FilterProvider = filters.WithAuthenticationAndAuthorization
115+
// Serve metrics with the cert-manager serving cert (the same one mounted
116+
// for the webhook) so scrapers can verify the endpoint against its CA,
117+
// instead of controller-runtime's in-memory self-signed cert that forces
118+
// insecureSkipVerify on the VMServiceScrape.
119+
metricsServerOptions.CertDir = "/tmp/k8s-webhook-server/serving-certs"
120+
metricsServerOptions.CertName = "tls.crt"
121+
metricsServerOptions.KeyName = "tls.key"
115122
}
116123

117124
config := ctrl.GetConfigOrDie()

‎internal/dbautoscaler/adapter_metrics_quota_test.go‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,21 @@ func TestPostgresQuorumFloor(t *testing.T) {
5757
}
5858
}
5959

60+
// TestPostgresInvalidMaxSyncReplicas guards against a negative maxSyncReplicas
61+
// (the postgres values schema does not lower-bound it): it must be reported as
62+
// non-scalable, and the quorum floor must never fall below 1.
63+
func TestPostgresInvalidMaxSyncReplicas(t *testing.T) {
64+
a := PostgresAdapter{}
65+
bad := map[string]any{"quorum": map[string]any{"maxSyncReplicas": float64(-3)}}
66+
ok, reason := a.Scalable(bad)
67+
if ok || reason == "" {
68+
t.Fatalf("negative maxSyncReplicas must be non-scalable with a reason, got ok=%v reason=%q", ok, reason)
69+
}
70+
if got := a.QuorumFloor(bad); got != 1 {
71+
t.Fatalf("QuorumFloor with negative maxSyncReplicas = %d, want clamped to 1", got)
72+
}
73+
}
74+
6075
func TestPostgresDriverQueryIsNamespaceScoped(t *testing.T) {
6176
a := PostgresAdapter{}
6277
app := types.NamespacedName{Namespace: "tenant-acme", Name: "db"}

‎internal/dbautoscaler/adapter_postgres.go‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -44,15 +44,26 @@ func (PostgresAdapter) ReleaseName(appName string) string { return "postgres-" +
4444

4545
// QuorumFloor is maxSyncReplicas + 1: the chart documents maxSyncReplicas as
4646
// "must be less than total replicas", so dropping to/below it makes CNPG
47-
// cap/reject the change and can starve synchronous commits.
47+
// cap/reject the change and can starve synchronous commits. A negative value is
48+
// clamped to 0 (floor 1) defensively; Scalable rejects it outright before this
49+
// is reached, but the floor must never come out below 1.
4850
func (PostgresAdapter) QuorumFloor(appValues map[string]any) int32 {
4951
maxSync := nestedInt(appValues, 0, "quorum", "maxSyncReplicas")
52+
if maxSync < 0 {
53+
maxSync = 0
54+
}
5055
return maxSync + 1
5156
}
5257

5358
// Scalable: a cozystack postgres is always a single primary-replica CNPG cluster
54-
// (no sharded mode), so it is always horizontally scalable.
59+
// (no sharded mode). It is scalable unless the synchronous-replica configuration
60+
// is invalid: the postgres values schema does not lower-bound maxSyncReplicas, so
61+
// a negative value would yield a quorum floor below 1 and silently disable the
62+
// floor guard. Reject it as non-scalable rather than scaling on a bad floor.
5563
func (PostgresAdapter) Scalable(appValues map[string]any) (bool, string) {
64+
if maxSync := nestedInt(appValues, 0, "quorum", "maxSyncReplicas"); maxSync < 0 {
65+
return false, fmt.Sprintf("invalid quorum.maxSyncReplicas %d (must be >= 0)", maxSync)
66+
}
5667
return true, ""
5768
}
5869

‎internal/dbautoscaler/decision_test.go‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,12 +17,32 @@ limitations under the License.
1717
package dbautoscaler
1818

1919
import (
20+
"math"
2021
"testing"
2122
"time"
2223

2324
autoscalingv1alpha1 "github.com/cozystack/cozystack/api/autoscaling/v1alpha1"
2425
)
2526

27+
// TestBackoffDurationNoOverflow guards the 8x re-attempt backoff against
28+
// time.Duration (int64) overflow: even a base beyond the API's
29+
// ConvergenceDeadlineSeconds maximum must yield a positive, sanely-capped wait,
30+
// never a wrapped negative one.
31+
func TestBackoffDurationNoOverflow(t *testing.T) {
32+
// A base near the int64 ceiling would overflow when doubled three times.
33+
if got := backoffDuration(math.MaxInt64, 4); got <= 0 {
34+
t.Fatalf("backoffDuration overflowed to non-positive: %d", int64(got))
35+
}
36+
// The cap is stable across further consecutive failures.
37+
if got := backoffDuration(math.MaxInt64, 99); got <= 0 {
38+
t.Fatalf("backoffDuration overflowed to non-positive: %d", int64(got))
39+
}
40+
// A normal base still doubles as before: 900s, 2 failures => 1800s.
41+
if got := backoffDuration(900*time.Second, 2); got != 1800*time.Second {
42+
t.Fatalf("backoffDuration(900s, 2) = %s, want 30m", got)
43+
}
44+
}
45+
2646
func ptr[T any](v T) *T { return &v }
2747

2848
func TestDesiredForMetric(t *testing.T) {

‎internal/dbautoscaler/status.go‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ limitations under the License.
1717
package dbautoscaler
1818

1919
import (
20+
"math"
2021
"time"
2122

2223
apimeta "k8s.io/apimachinery/pkg/api/meta"
@@ -65,6 +66,12 @@ func backoffDuration(base time.Duration, count int) time.Duration {
6566
if count < 1 {
6667
count = 1
6768
}
69+
// The loop doubles up to three times (8x). Clamp the base so that ceiling
70+
// cannot overflow time.Duration's int64, even if a caller passes a value
71+
// beyond the API's ConvergenceDeadlineSeconds maximum.
72+
if base > math.MaxInt64/8 {
73+
base = math.MaxInt64 / 8
74+
}
6875
d := base
6976
for i := 1; i < count && i < 4; i++ {
7077
d *= 2

‎packages/system/db-autoscaler/README.md‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10,14 +10,14 @@ Horizontal autoscaling applies to primary-replica engines (read replicas are sca
1010

1111
| Kind | Topology | Status |
1212
|---|---|---|
13-
| `Postgres` | CloudNativePG, 1 primary + standbys | **Enabled — full scale up/down validated on a live cluster.** Quorum floor = `quorum.maxSyncReplicas + 1`; PromQL joins CNPG metrics against `kube_pod_labels` on `label_cnpg_io_instance_role`. |
14-
| `Redis` | spotahome RedisFailover (Sentinel) | **Enabled — full scale up/down validated on a live cluster.** 1 master + read-serving slaves; PromQL selects slaves via `label_redisfailovers_role="slave"`; `redis_exporter` sidecar. No sync-replica quorum (floor = 1). |
13+
| `Postgres` | CloudNativePG, 1 primary + standbys | **Enabled — scale up/down validated on a live dev cluster.** Quorum floor = `quorum.maxSyncReplicas + 1`; PromQL joins CNPG metrics against `kube_pod_labels` on `label_cnpg_io_instance_role`. |
14+
| `Redis` | spotahome RedisFailover (Sentinel) | **Enabled — scale up/down validated on a live dev cluster.** 1 master + read-serving slaves; PromQL selects slaves via `label_redisfailovers_role="slave"`; `redis_exporter` sidecar. No sync-replica quorum (floor = 1). |
1515
| `MariaDB` | mariadb-operator async replication | **Not enabled yet** (`Scalable=false`). The adapter is implemented and its PromQL is calibrated, but the cozystack mariadb chart does not set `replication.replica.bootstrapFrom`, so the mariadb-operator rejects on-the-fly scale-out (`MariaDBScaleOutError`). Reported as not-scalable to avoid a patch→stuck→rollback loop; unblocked once the chart supports scale-out (follow-up in the mariadb package). |
1616
| `MongoDB` | Percona replica set (`rs0`) | **Not enabled yet** (`Scalable=false`). Sharded clusters are inherently non-scalable (data rebalancing); replica sets are topologically scalable but cozystack ships the Percona PMM/mongodb_exporter disabled, so there is no metric source and the queries are uncalibrated. Enabled once the exporter ships on and the PromQL is calibrated (follow-up). |
1717

1818
Engines that require data rebalancing — `ClickHouse`, `Kafka`, and sharded `MongoDB` — are intentionally **not** scalable and report `ScalingActive=False` with a reason.
1919

20-
All queries constrain to the tenant namespace — a tenant can never read another tenant's series. The enabled adapters (`postgres`, `redis`) are calibrated against real metrics on a live cluster.
20+
All queries constrain to the tenant namespace — a tenant can never read another tenant's series. The enabled adapters (`postgres`, `redis`) are calibrated against real metrics and functionally validated on a live dev cluster (decision path, ownership, and guardrails); they have not been load-tested at production scale.
2121

2222
## How it works
2323

‎packages/system/db-autoscaler/definitions/autoscaling.cozystack.io_databasehorizontalautoscalers.yaml‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -68,8 +68,11 @@ spec:
6868
description: |-
6969
ConvergenceDeadlineSeconds bounds how long a patched scale may take to
7070
converge (availableReplicas == replicas) before the operator surfaces
71-
StuckScaling and rolls back to lastConvergedReplicas.
71+
StuckScaling and rolls back to lastConvergedReplicas. Capped at 24h so the
72+
exponential re-attempt backoff (up to 8x this value) cannot overflow a
73+
time.Duration.
7274
format: int32
75+
maximum: 86400
7376
minimum: 1
7477
type: integer
7578
scaleDown:

‎packages/system/db-autoscaler/templates/metrics.yaml‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,15 @@ spec:
2727
endpoints:
2828
- port: metrics
2929
scheme: https
30+
# Verify the metrics server's cert-manager serving cert instead of skipping
31+
# verification: the CA is read from the same secret the server presents, and
32+
# serverName matches a SAN on that cert (the pod is reached by IP, but TLS
33+
# verification uses serverName, so no metrics-service DNS name is needed).
3034
tlsConfig:
31-
insecureSkipVerify: true
35+
ca:
36+
secret:
37+
name: db-autoscaler-cert
38+
key: ca.crt
39+
serverName: db-autoscaler.{{ .Release.Namespace }}.svc
3240
bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
3341
{{- end }}

‎packages/system/db-autoscaler/templates/rbac.yaml‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,9 +10,11 @@ rules:
1010
- apiGroups: ["autoscaling.cozystack.io"]
1111
resources: ["databasehorizontalautoscalers/status"]
1212
verbs: ["get", "update", "patch"]
13-
# Applications (aggregated apps API): read and patch the replicas value.
13+
# Applications (aggregated apps API): read and patch the replicas value. Scoped
14+
# to the database kinds the autoscaler can target (the adapters registered in
15+
# AdapterFor), not "*", so it cannot touch other apps.cozystack.io resources.
1416
- apiGroups: ["apps.cozystack.io"]
15-
resources: ["*"]
17+
resources: ["postgreses", "mariadbs", "redises", "mongodbs"]
1618
verbs: ["get", "list", "watch", "patch"]
1719
# WorkloadMonitor: operational / convergence signal.
1820
- apiGroups: ["cozystack.io"]

0 commit comments

Comments
 (0)