Skip to content

Commit 20f52b0

Browse files
author
Eric Curtin
committed
ateapi: ping the store for /readyz over a dedicated one-connection pool
1 parent 8243b5b commit 20f52b0

8 files changed

Lines changed: 274 additions & 21 deletions

File tree

‎cmd/ateapi/internal/store/atepg/atepg.go‎

Lines changed: 46 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -52,16 +52,24 @@ const (
5252
// Migrations need one connection for Goose's session lock and one for
5353
// migration work. Outbox maintenance is serial after startup.
5454
ownerPoolMaxConns = 2
55+
// The readiness probe pings over one connection that nothing else
56+
// borrows. With no minimum it dials on the first probe.
57+
probePoolMaxConns = 1
58+
probePoolMinConns = 0
5559
)
5660

5761
type Persistence struct {
5862
pool *pgxpool.Pool
5963
// watchPool serves WatchWorkers pollers and expired-lease cleanup.
6064
// ownerPool applies migrations and maintains outbox partitions.
65+
// probePool serves only Ping, so request traffic can never hold the
66+
// connection a readiness probe needs.
6167
watchPool *pgxpool.Pool
6268
ownerPool *pgxpool.Pool
69+
probePool *pgxpool.Pool
6370
ownsWatchPool bool
6471
ownsOwnerPool bool
72+
ownsProbePool bool
6573
policyManager *authz.PolicyManager
6674
leaseTTL time.Duration
6775
pollFailureCloseAfter time.Duration
@@ -125,7 +133,8 @@ type ConnectConfig struct {
125133
PoolMaxConns int32
126134
}
127135

128-
// Connect opens read/write and owner pools. It creates the schema and applies migrations.
136+
// Connect opens the read/write, owner, watch, and probe pools. It creates the
137+
// schema and applies migrations.
129138
func Connect(ctx context.Context, config ConnectConfig) (*Persistence, error) {
130139
if config.Schema == "" {
131140
return nil, fmt.Errorf("PostgreSQL schema must not be empty")
@@ -184,28 +193,44 @@ func Connect(ctx context.Context, config ConnectConfig) (*Persistence, error) {
184193
return nil, err
185194
}
186195

187-
watchCfg := readWriteConfig.Copy()
188-
watchCfg.MaxConns = watchPoolMaxConns
189-
watchCfg.MinConns = watchPoolMinConns
190-
watchPool, err := pgxpool.NewWithConfig(ctx, watchCfg)
196+
watchPool, err := pgxpool.NewWithConfig(ctx, resizedPoolConfig(readWriteConfig, watchPoolMaxConns, watchPoolMinConns))
191197
if err != nil {
192198
ownerPool.Close()
193199
pool.Close()
194200
return nil, fmt.Errorf("opening PostgreSQL watch pool: %w", err)
195201
}
202+
probePool, err := pgxpool.NewWithConfig(ctx, resizedPoolConfig(readWriteConfig, probePoolMaxConns, probePoolMinConns))
203+
if err != nil {
204+
watchPool.Close()
205+
ownerPool.Close()
206+
pool.Close()
207+
return nil, fmt.Errorf("opening PostgreSQL probe pool: %w", err)
208+
}
196209

197-
p, err := newPersistence(ctx, pool, watchPool, ownerPool)
210+
p, err := newPersistence(ctx, pool, watchPool, ownerPool, probePool)
198211
if err != nil {
212+
probePool.Close()
199213
watchPool.Close()
200214
ownerPool.Close()
201215
pool.Close()
202216
return nil, err
203217
}
204218
p.ownsWatchPool = true
205219
p.ownsOwnerPool = true
220+
p.ownsProbePool = true
206221
return p, nil
207222
}
208223

224+
// resizedPoolConfig returns a copy of base, the read/write pool config, with
225+
// its own connection limits, so a dedicated pool gets the same role, search
226+
// path, and TLS handling as the main one.
227+
func resizedPoolConfig(base *pgxpool.Config, maxConns, minConns int32) *pgxpool.Config {
228+
cfg := base.Copy()
229+
cfg.MaxConns = maxConns
230+
cfg.MinConns = minConns
231+
return cfg
232+
}
233+
209234
func createSchema(ctx context.Context, pool *pgxpool.Pool, schema string) error {
210235
tx, err := pool.Begin(ctx)
211236
if err != nil {
@@ -268,12 +293,13 @@ func poolConfig(dsn, role string) (*pgxpool.Config, error) {
268293

269294
// NewPersistence wraps an already-open pool, applying pending migrations.
270295
// Callers that already hold a pool (e.g. tests using testcontainers) use
271-
// this directly instead of Connect; outbox watch traffic shares the given pool.
296+
// this directly instead of Connect; outbox watch traffic and Ping share the
297+
// given pool.
272298
func NewPersistence(ctx context.Context, pool *pgxpool.Pool) (*Persistence, error) {
273-
return newPersistence(ctx, pool, pool, pool)
299+
return newPersistence(ctx, pool, pool, pool, pool)
274300
}
275301

276-
func newPersistence(ctx context.Context, pool, watchPool, ownerPool *pgxpool.Pool) (*Persistence, error) {
302+
func newPersistence(ctx context.Context, pool, watchPool, ownerPool, probePool *pgxpool.Pool) (*Persistence, error) {
277303
if err := applyMigrations(ctx, ownerPool); err != nil {
278304
return nil, err
279305
}
@@ -282,6 +308,7 @@ func newPersistence(ctx context.Context, pool, watchPool, ownerPool *pgxpool.Poo
282308
pool: pool,
283309
watchPool: watchPool,
284310
ownerPool: ownerPool,
311+
probePool: probePool,
285312
leaseTTL: defaultLeaseTTL,
286313
pollFailureCloseAfter: outboxPollFailureCloseAfter,
287314
stopMaintenance: stopMaintenance,
@@ -319,6 +346,16 @@ func (p *Persistence) Close() {
319346
if p.ownsOwnerPool {
320347
p.ownerPool.Close()
321348
}
349+
if p.ownsProbePool {
350+
p.probePool.Close()
351+
}
352+
}
353+
354+
// Ping reports whether PostgreSQL answers. It uses a connection reserved for
355+
// it, so a saturated request pool cannot make it fail: it fails only when the
356+
// database does not answer within ctx.
357+
func (p *Persistence) Ping(ctx context.Context) error {
358+
return p.probePool.Ping(ctx)
322359
}
323360

324361
// Pool returns the underlying PostgreSQL connection pool.

‎cmd/ateapi/internal/store/atepg/connect_test.go‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -350,6 +350,13 @@ func TestConnectSeparatesRuntimeAndDDLPrivileges(t *testing.T) {
350350
if got := p.ownerPool.Config().MaxConns; got != ownerPoolMaxConns {
351351
t.Fatalf("owner pool MaxConns = %d, want %d", got, ownerPoolMaxConns)
352352
}
353+
var probeRole string
354+
if err := p.probePool.QueryRow(ctx, `SELECT current_user`).Scan(&probeRole); err != nil || probeRole != runtimeRole {
355+
t.Fatalf("probe pool role = %q, %v; want the read/write role %q", probeRole, err, runtimeRole)
356+
}
357+
if got := p.probePool.Config().MaxConns; got != probePoolMaxConns {
358+
t.Fatalf("probe pool MaxConns = %d, want %d", got, probePoolMaxConns)
359+
}
353360

354361
if _, err := p.CreateAtespace(ctx, newTestAtespace("runtime-write")); err != nil {
355362
t.Fatalf("read/write operation failed: %v", err)
Lines changed: 206 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,206 @@
1+
// Copyright 2026 Google LLC
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
package atepg
16+
17+
import (
18+
"context"
19+
"fmt"
20+
"io"
21+
"net"
22+
"net/url"
23+
"sync"
24+
"testing"
25+
"time"
26+
)
27+
28+
// connectForPing connects to the shared container with a fresh schema that is
29+
// dropped when the test ends. The returned config lets a test adjust the DSN
30+
// or pool size before connecting.
31+
func connectForPing(t *testing.T, schema string, adjust func(*ConnectConfig)) *Persistence {
32+
t.Helper()
33+
admin := requirePool(t)
34+
ctx := t.Context()
35+
drop := fmt.Sprintf(`DROP SCHEMA IF EXISTS %q CASCADE`, schema)
36+
if _, err := admin.Exec(ctx, drop); err != nil {
37+
t.Fatalf("resetting PostgreSQL schema: %v", err)
38+
}
39+
t.Cleanup(func() { _, _ = admin.Exec(context.Background(), drop) })
40+
41+
cfg := testConnectConfig(schema)
42+
adjust(&cfg)
43+
p, err := Connect(ctx, cfg)
44+
if err != nil {
45+
t.Fatalf("Connect failed: %v", err)
46+
}
47+
t.Cleanup(func() {
48+
p.Close()
49+
p.pool.Close()
50+
})
51+
return p
52+
}
53+
54+
// A saturated request pool must not fail the readiness probe: the probe cannot
55+
// tell a busy replica from an unreachable store if it borrows from that pool.
56+
func TestPingIgnoresSaturatedRequestPool(t *testing.T) {
57+
p := connectForPing(t, "ping-saturated", func(cfg *ConnectConfig) { cfg.PoolMaxConns = 1 })
58+
ctx := t.Context()
59+
60+
if p.probePool == p.pool {
61+
t.Fatal("probe pool is the request pool")
62+
}
63+
if got := p.probePool.Config().MaxConns; got != probePoolMaxConns {
64+
t.Errorf("probe pool MaxConns = %d, want %d", got, probePoolMaxConns)
65+
}
66+
if got := p.probePool.Config().MinConns; got != probePoolMinConns {
67+
t.Errorf("probe pool MinConns = %d, want %d", got, probePoolMinConns)
68+
}
69+
70+
held, err := p.pool.Acquire(ctx)
71+
if err != nil {
72+
t.Fatalf("acquiring the only request connection: %v", err)
73+
}
74+
defer held.Release()
75+
76+
budget := func() (context.Context, context.CancelFunc) {
77+
return context.WithTimeout(ctx, 500*time.Millisecond)
78+
}
79+
80+
// The premise: the request pool really is exhausted, and a ping through it
81+
// blocks past the readiness budget.
82+
poolCtx, cancel := budget()
83+
defer cancel()
84+
if err := p.pool.Ping(poolCtx); err == nil {
85+
t.Fatal("request pool ping succeeded while every connection was held; the pool is not saturated")
86+
}
87+
88+
probeCtx, cancel := budget()
89+
defer cancel()
90+
if err := p.Ping(probeCtx); err != nil {
91+
t.Errorf("Ping with a saturated request pool = %v, want nil", err)
92+
}
93+
}
94+
95+
func TestPingFailsAfterClose(t *testing.T) {
96+
p := connectForPing(t, "ping-closed", func(*ConnectConfig) {})
97+
if err := p.Ping(t.Context()); err != nil {
98+
t.Fatalf("Ping before Close = %v, want nil", err)
99+
}
100+
p.Close()
101+
if err := p.Ping(t.Context()); err == nil {
102+
t.Error("Ping after Close succeeded, want an error from the closed probe pool")
103+
}
104+
}
105+
106+
// Ping fails while PostgreSQL does not answer and passes again once it does,
107+
// with no restart of the pool.
108+
func TestPingFollowsStoreReachability(t *testing.T) {
109+
requirePool(t)
110+
dsn, err := url.Parse(containerDSN)
111+
if err != nil {
112+
t.Fatalf("parsing PostgreSQL DSN: %v", err)
113+
}
114+
proxy := newTCPProxy(t, dsn.Host)
115+
dsn.Host = proxy.addr
116+
p := connectForPing(t, "ping-reachability", func(cfg *ConnectConfig) { cfg.ReadWriteDSN = dsn.String() })
117+
118+
ping := func() error {
119+
ctx, cancel := context.WithTimeout(t.Context(), 2*time.Second)
120+
defer cancel()
121+
return p.Ping(ctx)
122+
}
123+
if err := ping(); err != nil {
124+
t.Fatalf("Ping with the store reachable = %v, want nil", err)
125+
}
126+
proxy.stop()
127+
if err := ping(); err == nil {
128+
t.Error("Ping with the store unreachable succeeded, want an error")
129+
}
130+
proxy.start()
131+
if err := ping(); err != nil {
132+
t.Errorf("Ping after the store became reachable = %v, want nil", err)
133+
}
134+
}
135+
136+
// tcpProxy forwards TCP connections to target until stopped, and start brings
137+
// it back on the same address.
138+
type tcpProxy struct {
139+
t *testing.T
140+
target string
141+
addr string
142+
143+
mu sync.Mutex
144+
ln net.Listener
145+
conns []net.Conn
146+
}
147+
148+
func newTCPProxy(t *testing.T, target string) *tcpProxy {
149+
t.Helper()
150+
p := &tcpProxy{t: t, target: target, addr: "127.0.0.1:0"}
151+
p.start()
152+
t.Cleanup(p.stop)
153+
return p
154+
}
155+
156+
func (p *tcpProxy) start() {
157+
p.t.Helper()
158+
ln, err := net.Listen("tcp", p.addr)
159+
if err != nil {
160+
p.t.Fatalf("listening on %s: %v", p.addr, err)
161+
}
162+
p.mu.Lock()
163+
p.ln = ln
164+
p.addr = ln.Addr().String()
165+
p.mu.Unlock()
166+
go func() {
167+
for {
168+
client, err := ln.Accept()
169+
if err != nil {
170+
return
171+
}
172+
go p.forward(client)
173+
}
174+
}()
175+
}
176+
177+
func (p *tcpProxy) forward(client net.Conn) {
178+
upstream, err := net.Dial("tcp", p.target)
179+
if err != nil {
180+
client.Close()
181+
return
182+
}
183+
p.mu.Lock()
184+
p.conns = append(p.conns, client, upstream)
185+
p.mu.Unlock()
186+
go func() {
187+
io.Copy(upstream, client) //nolint:errcheck // Either side closing ends the copy.
188+
upstream.Close()
189+
}()
190+
io.Copy(client, upstream) //nolint:errcheck // Either side closing ends the copy.
191+
client.Close()
192+
}
193+
194+
// stop refuses new connections and drops the open ones.
195+
func (p *tcpProxy) stop() {
196+
p.mu.Lock()
197+
defer p.mu.Unlock()
198+
if p.ln != nil {
199+
p.ln.Close()
200+
p.ln = nil
201+
}
202+
for _, c := range p.conns {
203+
c.Close()
204+
}
205+
p.conns = nil
206+
}

‎cmd/ateapi/main.go‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -80,7 +80,7 @@ var (
8080
postgresReadWriteRole = pflag.String("postgres-read-write-role", "", "Required PostgreSQL role assumed by read/write connections.")
8181
postgresOwnerRole = pflag.String("postgres-owner-role", "", "Required PostgreSQL role assumed by owner connections.")
8282
postgresSchema = pflag.String("postgres-schema", "substrate", "PostgreSQL schema for Substrate tables. This overrides a search_path connection parameter.")
83-
postgresPoolMaxConns = pflag.Int32("postgres-pool-max-conns", 0, "Maximum connections in the shared Substrate and OpenFGA read/write PostgreSQL pool. Does not affect the owner or watch pools. The DSN or pgx default is used when unset.")
83+
postgresPoolMaxConns = pflag.Int32("postgres-pool-max-conns", 0, "Maximum connections in the shared Substrate and OpenFGA read/write PostgreSQL pool. Does not affect the owner, watch, or readiness probe pools. The DSN or pgx default is used when unset.")
8484
experimentalEnableAuthz = pflag.Bool("experimental-enable-authz", false, "Enforce OpenFGA authorization checks on all registered RPCs (experimental). AccessPolicy RPCs are always checked.")
8585
// TODO: Move the authz settings into the hot-reloadable config proto
8686
// (agent-substrate/substrate#2021) once it lands, so bootstrap owner
@@ -369,8 +369,10 @@ func main() {
369369

370370
// Every RPC needs the store, so a replica that cannot reach it leaves the
371371
// Service until it can. Liveness stays green: the pool reconnects on its own.
372+
// The check pings over its own connection, so a busy request pool does not
373+
// fail it.
372374
readiness := &serverboot.Readiness{}
373-
readiness.AddCheck("store", pool.Ping)
375+
readiness.AddCheck("store", persistence.Ping)
374376
go serverboot.StartHealthServer(ctx, *healthListenAddr, readiness)
375377

376378
drainDone := drainOnShutdown(shutdownCtx, mux, readiness)

‎docs/dev/cloud-sql-scaling-guide.md‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -45,16 +45,16 @@ connections ≈ QPS × mean latency in seconds
4545
≈ 10,000 req/s × 0.006 s ≈ 60 active connections
4646
```
4747

48-
Provision ~2× headroom for bursts. This setting does not affect the owner and
49-
watch pools; their fixed limits add up to 5 connections per replica beyond the
50-
read/write pool limit (e.g. 4 replicas with
48+
Provision ~2× headroom for bursts. This setting does not affect the owner,
49+
watch, and readiness probe pools; their fixed limits add up to 6 connections per
50+
replica beyond the read/write pool limit (e.g. 4 replicas with
5151
`ATE_API_POSTGRES_POOL_MAX_CONNS=32` can open up to
52-
4 × (32 + 2 + 3) = 148 connections).
52+
4 × (32 + 2 + 3 + 1) = 152 connections).
5353
An undersized pool causes client-side queuing inside `pgx` rather than database errors.
5454
Ensure total connections across all replicas stay within Cloud SQL's limit:
5555

5656
```
57-
replicas × (pool_max_conns + 5) ≤ max_connections − slack (superuser, maintenance)
57+
replicas × (pool_max_conns + 6) ≤ max_connections − slack (superuser, maintenance)
5858
```
5959

6060
Exceeding

‎docs/postgres.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -85,7 +85,7 @@ Set `ATE_API_POSTGRES_OWNER_ROLE` and `ATE_API_POSTGRES_READ_WRITE_ROLE` to the
8585

8686
User credentials are passed as connection strings through `ATE_API_POSTGRES_OWNER_CONNECTION_STRING` and `ATE_API_POSTGRES_READ_WRITE_CONNECTION_STRING`. When running `ateapi` directly, pass the DSNs as flags or use `@env` flags to read these environment variables; the installer manifest already uses `@env`.
8787

88-
`ATE_API_POSTGRES_POOL_MAX_CONNS` (or `--postgres-pool-max-conns` when running `ateapi` directly) sets the read/write pool limit after the connection string is loaded. When unset, a `pool_max_conns` value in the DSN or the pgxpool default applies. This setting does not affect the owner or watch pools; they are capped at 2 and 3 connections, respectively.
88+
`ATE_API_POSTGRES_POOL_MAX_CONNS` (or `--postgres-pool-max-conns` when running `ateapi` directly) sets the read/write pool limit after the connection string is loaded. When unset, a `pool_max_conns` value in the DSN or the pgxpool default applies. This setting does not affect the owner, watch, or readiness probe pools; they are capped at 2, 3, and 1 connections, respectively.
8989

9090
`ateapi` uses the same `@env` flag pattern for role names and schema.
9191

0 commit comments

Comments
 (0)