Skip to content

Commit 0fc0089

Browse files
committed
atelet: synchronize stdout writers, harden the usage-event pipeline
The runtime logger and the usage-event drain were two uncoordinated writers on stdout, tear-free only while every record fits a pipe's atomic-write size -- an accident of field sizes, not a contract. One actorlog.SyncedWriter now fronts stdout for both, the same pattern the ateoms use for their actor-log forwarders. The asyncWriter's queue still keeps a stalled log consumer from costing the sweep anything; the shared lock only guarantees whole records. The drop report also moves onto the emitter's own fixed-level pipeline: the loss signal is part of the feed's integrity, so it must be exactly as unkillable by --log-level as the feed itself -- routing it through the leveled logger meant a node quieted to error would lose the only evidence that events were dropped. Also from review of the current shape: document the actual bounds on the labels-key metadata probe (2s typical worst off GCE, 5s pathological cap, first-tick emits only), rename the asyncWriter receiver left over from its earlier name, give the writer tests cancelable contexts, and fix comments that had drifted from the implementation -- dialAteomStats served "both telemetry paths" until the lifecycle sampler was descoped, and the queue-depth comment oversized its own claim.
1 parent 32549be commit 0fc0089

4 files changed

Lines changed: 101 additions & 80 deletions

File tree

‎cmd/atelet/main.go‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ import (
3636
"sync"
3737

3838
"github.com/agent-substrate/substrate/cmd/atelet/internal/ategcs"
39+
"github.com/agent-substrate/substrate/internal/actorlog"
3940
"github.com/agent-substrate/substrate/internal/ateapiauth"
4041
"github.com/agent-substrate/substrate/internal/ateattr"
4142
"github.com/agent-substrate/substrate/internal/ateerrors"
@@ -118,7 +119,13 @@ func main() {
118119
return
119120
}
120121
ctx := context.Background()
121-
serverboot.InitLogger()
122+
// One synchronized writer in front of stdout, shared by the runtime
123+
// logger and the usage-event drain (see startStatsPoller): uncoordinated
124+
// writers stay tear-free only while every record fits a pipe's
125+
// atomic-write size -- an accident of field sizes, not a contract. Same
126+
// pattern as the ateoms' actor-log forwarders.
127+
logSink := actorlog.NewSyncedWriter(os.Stdout)
128+
serverboot.InitLoggerWithWriter(logSink)
122129
if err := serverboot.SetLogLevel(*logLevelFlag); err != nil {
123130
serverboot.Fatal(ctx, "Invalid --log-level", err)
124131
}
@@ -257,7 +264,7 @@ func main() {
257264
// crash-looping every actor operation on the node.
258265
slog.ErrorContext(ctx, "Actor stats sampling disabled: failed to create instruments", slog.Any("err", err))
259266
} else {
260-
startStatsPoller(ctx, interval, statsInst, k8sClient)
267+
startStatsPoller(ctx, interval, statsInst, k8sClient, logSink)
261268
}
262269
}
263270

‎cmd/atelet/statsevents.go‎

Lines changed: 46 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -47,10 +47,13 @@ const eventKindPeriodic = "periodic"
4747

4848
// defaultLabelsKey resolves the actor-identity label group's spelling --
4949
// actorlog's, so usage events and lifecycle events promote into Cloud
50-
// Logging labels the same way. Resolved once, off atelet's boot path:
51-
// metadata.OnGCE probes the metadata server (seconds of timeout off GCE), so
52-
// startStatsPoller warms this on a throwaway goroutine and sync.OnceValue
53-
// makes any emit that arrives first simply wait for the in-flight probe.
50+
// Logging labels the same way. metadata.OnGCE probes the metadata server, so
51+
// startStatsPoller warms this off the boot path on a throwaway goroutine;
52+
// sync.OnceValue makes an emit that arrives first wait for the in-flight
53+
// probe. That wait is first-tick-only and bounded: milliseconds on GCE, the
54+
// transport's 2s dial timeout off GCE, a 5s cap on the one pathological
55+
// branch (SMBIOS says GCE, probes disagree) -- and no request path ever
56+
// touches it.
5457
var defaultLabelsKey = sync.OnceValue(func() string {
5558
return actorlog.LabelsKey(metadata.OnGCE())
5659
})
@@ -67,59 +70,65 @@ type statsEventEmitter struct {
6770
// leveled diagnostics, and quieting the node with --log-level=warn must not
6871
// silently sever them -- the subsystem's one off-switch is
6972
// --actor-stats-poll-interval=0. The records still carry level INFO on the
70-
// wire, so nothing downstream changes; sharing atelet's stdout is safe
71-
// because each record is written whole, in a single Write. In production w
72-
// is a asyncWriter over stdout (see startStatsPoller), so a stalled
73-
// log consumer costs events, never the sweep.
73+
// wire, so nothing downstream changes. In production w is an asyncWriter
74+
// over the process's synchronized stdout writer (see startStatsPoller).
7475
func newStatsEventEmitter(w io.Writer, labelsKey func() string) *statsEventEmitter {
7576
return &statsEventEmitter{
7677
log: slog.New(contextlogging.NewHandler(slog.NewJSONHandler(w, nil))),
7778
labelsKey: labelsKey,
7879
}
7980
}
8081

81-
// usageEventQueueDepth is the asyncWriter's buffer, in records: sized
82-
// to absorb a full sweep's burst (statsSweepConcurrency probes emitting into
83-
// one queue, a few hundred actors on a packed node) while the writer
84-
// goroutine drains at pipe speed.
82+
// usageEventQueueDepth is the asyncWriter's buffer, in records: about one
83+
// tick of a packed node. A healthy drain outruns the RPC-paced producers by
84+
// orders of magnitude, so the queue only fills once the pipe is dead --
85+
// where any finite depth drops, and the choice is cosmetic.
8586
const usageEventQueueDepth = 256
8687

87-
// asyncWriter decouples event emission from the log consumer's health.
88-
// A write to a full stdout pipe blocks forever -- there is no timeout on the
89-
// syscall -- and emit runs inside the sweep's errgroup, so one stalled log
90-
// consumer (wedged rotation, disk-full fallout) would otherwise park a probe,
91-
// wedge the sweep, and silently freeze the metrics channel that shares it,
92-
// which is exactly the signal that must survive such an incident. Writes
93-
// land in a bounded queue drained by one goroutine; a full queue drops the
94-
// record and counts it. Dropping is safe here: the samples are point-in-time
95-
// readings the next healthy tick repairs, and when stdout is dead the events
96-
// are lost either way -- the choice is whether the metrics die with them.
88+
// asyncWriter decouples event emission from the log consumer's health. A
89+
// write to a full stdout pipe blocks forever, and emit runs inside the
90+
// sweep's errgroup, so one stalled log consumer (wedged rotation, disk-full
91+
// fallout) would otherwise park a probe, wedge the sweep, and silently
92+
// freeze the metrics channel that shares it. Writes land in a bounded queue
93+
// drained by one goroutine; a full queue drops the record and counts it.
94+
// Dropping is safe: the samples are point-in-time readings the next healthy
95+
// tick repairs, and when stdout is dead the events are lost either way --
96+
// the choice is whether the metrics die with them.
9797
type asyncWriter struct {
9898
w io.Writer
9999
ch chan []byte
100100

101+
// report carries the drop warning over the same fixed-level pipeline as
102+
// the records: the loss signal is part of the feed's integrity, so it
103+
// must be exactly as unkillable by --log-level as the feed itself.
104+
report *slog.Logger
105+
101106
dropped atomic.Int64
102107
}
103108

104109
func newAsyncWriter(ctx context.Context, w io.Writer, depth int) *asyncWriter {
105-
nb := &asyncWriter{w: w, ch: make(chan []byte, depth)}
106-
go nb.run(ctx)
107-
return nb
110+
aw := &asyncWriter{
111+
w: w,
112+
ch: make(chan []byte, depth),
113+
report: slog.New(contextlogging.NewHandler(slog.NewJSONHandler(w, nil))),
114+
}
115+
go aw.run(ctx)
116+
return aw
108117
}
109118

110-
func (nb *asyncWriter) run(ctx context.Context) {
119+
func (aw *asyncWriter) run(ctx context.Context) {
111120
for {
112121
select {
113122
case <-ctx.Done():
114123
return
115-
case b := <-nb.ch:
116-
// A failed or short write is the log stream's problem, not ours:
117-
// the record is best-effort and there is nobody to report to.
118-
_, _ = nb.w.Write(b)
119-
// The stream just proved itself live again, so a warning about
120-
// the stall can actually land.
121-
if n := nb.dropped.Swap(0); n > 0 {
122-
slog.WarnContext(ctx, "Usage events dropped while the log stream stalled", slog.Int64("count", n))
124+
case b := <-aw.ch:
125+
// Best-effort: a failed write has nobody to report to.
126+
_, _ = aw.w.Write(b)
127+
// Report drops only after a successful write, when the stream
128+
// can carry the warning -- written directly, so the drain does
129+
// not queue behind itself.
130+
if n := aw.dropped.Swap(0); n > 0 {
131+
aw.report.WarnContext(ctx, "Usage events dropped while the log stream stalled", slog.Int64("count", n))
123132
}
124133
}
125134
}
@@ -128,14 +137,14 @@ func (nb *asyncWriter) run(ctx context.Context) {
128137
// Write queues one record without ever blocking. It reports full success
129138
// even on a drop: the emitter has no recovery to offer, and the drop is
130139
// already counted for the writer goroutine to report.
131-
func (nb *asyncWriter) Write(p []byte) (int, error) {
140+
func (aw *asyncWriter) Write(p []byte) (int, error) {
132141
// The slog handler reuses its buffer after Write returns, so the queue
133142
// must own a copy.
134143
b := append([]byte(nil), p...)
135144
select {
136-
case nb.ch <- b:
145+
case aw.ch <- b:
137146
default:
138-
nb.dropped.Add(1)
147+
aw.dropped.Add(1)
139148
}
140149
return len(p), nil
141150
}

‎cmd/atelet/statsevents_test.go‎

Lines changed: 14 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -232,8 +232,10 @@ func (w *stallableWriter) Write(p []byte) (int, error) {
232232
// queue held.
233233
func TestAsyncWriterNeverBlocks(t *testing.T) {
234234
under := &stallableWriter{gate: make(chan struct{})}
235+
ctx, cancel := context.WithCancel(context.Background())
236+
t.Cleanup(cancel)
235237
const depth = 4
236-
nb := newAsyncWriter(context.Background(), under, depth)
238+
aw := newAsyncWriter(ctx, under, depth)
237239

238240
// The writer goroutine dequeues one record and wedges on it; the queue
239241
// holds depth more. Everything past that must drop, not block. The
@@ -243,7 +245,7 @@ func TestAsyncWriterNeverBlocks(t *testing.T) {
243245
go func() {
244246
defer close(done)
245247
for i := 0; i < writes; i++ {
246-
if _, err := nb.Write([]byte("x")); err != nil {
248+
if _, err := aw.Write([]byte("x")); err != nil {
247249
t.Errorf("Write returned %v, want nil", err)
248250
}
249251
}
@@ -253,29 +255,33 @@ func TestAsyncWriterNeverBlocks(t *testing.T) {
253255
case <-time.After(5 * time.Second):
254256
t.Fatal("Write blocked on a wedged log stream")
255257
}
256-
dropped := int(nb.dropped.Load())
258+
dropped := int(aw.dropped.Load())
257259
if dropped < writes-depth-1 || dropped > writes-depth {
258260
t.Errorf("dropped = %d, want %d or %d", dropped, writes-depth-1, writes-depth)
259261
}
260262

261263
// Conservation: everything not dropped -- the wedged in-flight record
262-
// plus the queue -- drains once the stream recovers.
264+
// plus the queue -- drains once the stream recovers, and the drop report
265+
// arrives over the same sink, unkillable by any verbosity knob.
263266
close(under.gate)
264267
wantWritten := writes - dropped
265-
waitFor(t, func() bool { return under.wrote.Len() >= wantWritten })
266-
if got := under.wrote.Len(); got != wantWritten {
268+
waitFor(t, func() bool { return bytes.Count(under.wrote.Bytes(), []byte("x")) >= wantWritten })
269+
if got := bytes.Count(under.wrote.Bytes(), []byte("x")); got != wantWritten {
267270
t.Errorf("drained %d records, want %d", got, wantWritten)
268271
}
272+
waitFor(t, func() bool { return bytes.Contains(under.wrote.Bytes(), []byte("Usage events dropped")) })
269273
}
270274

271275
// TestAsyncWriterCopies: the slog handler reuses its buffer after
272276
// Write returns, so the queue must hold copies, not aliases.
273277
func TestAsyncWriterCopies(t *testing.T) {
274278
under := &stallableWriter{gate: make(chan struct{})}
275-
nb := newAsyncWriter(context.Background(), under, 4)
279+
ctx, cancel := context.WithCancel(context.Background())
280+
t.Cleanup(cancel)
281+
aw := newAsyncWriter(ctx, under, 4)
276282

277283
p := []byte("original")
278-
if _, err := nb.Write(p); err != nil {
284+
if _, err := aw.Write(p); err != nil {
279285
t.Fatal(err)
280286
}
281287
copy(p, "clobber!")

‎cmd/atelet/statspoller.go‎

Lines changed: 32 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -111,28 +111,26 @@ type statsPoller struct {
111111
ateomsDir string
112112

113113
// dial returns a stats client for one ateom plus the closer that releases
114-
// its connection; the probe closes it before returning, so a connection
115-
// lives exactly one probe. Deliberately NOT the lifecycle RPCs' cached
116-
// AteomDialer: at one probe per ateom per minute over a local unix socket
117-
// a cache saves nothing, and sweeping the node's stale sockets through a
118-
// shared cache would let telemetry evict connections the lifecycle RPCs
119-
// are using.
114+
// its connection; a connection lives exactly one probe. Deliberately NOT
115+
// the lifecycle RPCs' cached AteomDialer: at one probe per ateom per
116+
// minute over a local unix socket a cache saves nothing, and sweeping the
117+
// node's stale sockets through a shared cache would let telemetry evict
118+
// connections the lifecycle RPCs are using.
120119
dial func(ctx context.Context, podUID string) (activeStatsClient, io.Closer, error)
121120

122121
// workerPools resolves this node's worker pod UIDs to the pool that owns
123122
// them, called once per sweep. Nil (or a nil map, or a missing entry)
124-
// degrades to samples grouped without pool labels rather than dropped: the
125-
// pool is enrichment, the sample is the point. The real resolver lists the
126-
// node's pods by the ate.dev/worker-pool label the pool controller stamps
127-
// on every worker (see workerpool_apply.go); the ateom directory name IS
123+
// degrades to samples grouped without pool labels rather than dropped:
124+
// the pool is enrichment, the sample is the point. The real resolver
125+
// lists the node's pods by workerPoolLabel; the ateom directory name IS
128126
// the worker pod UID, which is the join key.
129127
workerPools func(ctx context.Context) map[string]workerPoolRef
130128

131129
inst *statsInstruments
132130

133-
// eventEmitter receives one periodic usage event per sample per sweep -- the
134-
// per-actor channel the aggregates deliberately erase identity from. Nil
135-
// disables emission (the emitter is also nil-safe).
131+
// eventEmitter receives one usage event per sample per sweep -- the
132+
// per-actor channel the aggregates deliberately erase identity from.
133+
// Nil disables emission; emit is nil-safe.
136134
eventEmitter *statsEventEmitter
137135

138136
// lastCPU is the previous sweep's cpu_usage_usec per actor uid, the
@@ -145,10 +143,10 @@ type statsPoller struct {
145143
lastCPU map[string]uint64
146144
}
147145

148-
// templateAggregate is one tick's sums for one templateKey group: the bounded
149-
// label set #174 permits on a TSDB series. Actor and
150-
// atespace identity deliberately never reach a metric label; per-actor detail
151-
// is the events channel's job, not this one's.
146+
// templateAggregate is one tick's sums for one templateKey group: the
147+
// bounded label set #174 permits on a TSDB series. Actor and atespace
148+
// identity deliberately never reach a metric label; per-actor detail is the
149+
// events channel's job, not this one's.
152150
//
153151
// The memory fields are point-in-time sums the gauges observe. cpuDeltaUsec is
154152
// different: cpu_usage_usec is a cumulative per-epoch counter per actor, so
@@ -203,7 +201,7 @@ func (k templateKey) attrs() metric.MeasurementOption {
203201
return metric.WithAttributes(attrs...)
204202
}
205203

206-
// run polls until ctx is cancelled. The caller has already validated and
204+
// run polls until ctx is canceled. The caller has already validated and
207205
// clamped interval.
208206
func (p *statsPoller) run(ctx context.Context) {
209207
slog.InfoContext(ctx, "Actor stats poller starting", slog.Duration("interval", p.interval))
@@ -409,12 +407,10 @@ func nodeWorkerPools(client kubernetes.Interface, nodeName string) func(ctx cont
409407
for _, pod := range pods.Items {
410408
name := pod.Labels[workerPoolLabel]
411409
if name == "" {
412-
// The list matches on the key's presence, and an existence
413-
// selector matches empty-valued labels too -- anyone can
414-
// stamp ate.dev/worker-pool: "" on a pod (a bare key in
415-
// YAML parses to exactly that). Half a pair names no pool,
416-
// so it never enters the map: downstream, absent and
417-
// unresolvable are the same answer, unlabeled.
410+
// The existence selector also matches empty-valued labels
411+
// (a bare key in YAML parses to ""), and half a pair names
412+
// no pool: skip it, so absent and unresolvable are the same
413+
// unlabeled answer downstream.
418414
continue
419415
}
420416
pools[string(pod.UID)] = workerPoolRef{
@@ -532,9 +528,9 @@ func (i *statsInstruments) addCPU(ctx context.Context, aggs map[templateKey]*tem
532528
return
533529
}
534530
for key, agg := range aggs {
535-
// The wire carries microseconds; the metric is seconds, the base unit
536-
// CPU time is exported in everywhere else (cAdvisor's
537-
// container_cpu_usage_seconds_total, OTel's *.cpu.time), so the
531+
// The wire carries microseconds; the metric is seconds -- the base
532+
// unit CPU time is exported in everywhere else (cAdvisor's
533+
// container_cpu_usage_seconds_total, OTel's *.cpu.time) -- so the
538534
// existing rate() idioms read directly as cores.
539535
i.cpuUsage.Add(ctx, float64(agg.cpuDeltaUsec)/1e6, key.attrs())
540536
}
@@ -547,7 +543,10 @@ func (i *statsInstruments) addCPU(ctx context.Context, aggs map[templateKey]*tem
547543
// The poller dials its own short-lived connection per probe (see
548544
// dialAteomStats) and takes no AteomDialer: the isolation from the lifecycle
549545
// RPCs' connection cache is structural, not just behavioral.
550-
func startStatsPoller(ctx context.Context, interval time.Duration, inst *statsInstruments, k8sClient kubernetes.Interface) {
546+
// logSink is the process's synchronized stdout writer, shared with the
547+
// runtime logger so the event drain and slog can never tear each other's
548+
// records.
549+
func startStatsPoller(ctx context.Context, interval time.Duration, inst *statsInstruments, k8sClient kubernetes.Interface, logSink io.Writer) {
551550
// Warm the labels-key resolution so the first emit does not pay the
552551
// metadata probe either; see defaultLabelsKey.
553552
go defaultLabelsKey()
@@ -563,7 +562,7 @@ func startStatsPoller(ctx context.Context, interval time.Duration, inst *statsIn
563562
return ateompb.NewAteomClient(conn), closer, nil
564563
},
565564
inst: inst,
566-
eventEmitter: newStatsEventEmitter(newAsyncWriter(ctx, os.Stdout, usageEventQueueDepth), defaultLabelsKey),
565+
eventEmitter: newStatsEventEmitter(newAsyncWriter(ctx, logSink, usageEventQueueDepth), defaultLabelsKey),
567566
}
568567
// NODE_NAME comes from the Downward API; without it the samples still
569568
// flow, just grouped without pool labels.
@@ -575,10 +574,10 @@ func startStatsPoller(ctx context.Context, interval time.Duration, inst *statsIn
575574
go poller.run(ctx)
576575
}
577576

578-
// dialAteomStats opens the short-lived connection both telemetry paths use:
579-
// one connection per read, closed by the caller, never the lifecycle RPCs'
580-
// cached AteomDialer. grpc.NewClient is lazy, so this cannot block; the
581-
// caller's deadline bounds the actual connect inside the RPC.
577+
// dialAteomStats opens the poller's short-lived connection: one per probe,
578+
// closed by the caller, never the lifecycle RPCs' cached AteomDialer.
579+
// grpc.NewClient is lazy, so this cannot block; the caller's deadline bounds
580+
// the actual connect inside the RPC.
582581
func dialAteomStats(podUID string) (*grpc.ClientConn, io.Closer, error) {
583582
conn, err := grpc.NewClient(
584583
"unix://"+ateompath.AteomSocketPath(podUID),

0 commit comments

Comments
 (0)