Skip to content

Commit 7d9ced4

Browse files
committed
ate: have each worker report its own capacity
CreateWorker was fabricating a capacity of one actor for every Worker it stored. That contradicts the rule the API just adopted -- a Worker reports every dimension it has, so an absent one is none of it -- because the control plane was answering on the Worker's behalf before the worker had said anything. Nothing was calling SetWorkerCapacity, so add the reporter that makes it real: ateom answers a new GetCapacity, atelet sweeps the node's ateoms and forwards what they say, and a Worker holds no capacity until that lands. ateom learns its compute limits from the downward API, since it has no reader for its own cgroup and inventing one is a larger change. Left as a TODO: the environment is fixed at pod creation, so it goes stale under in-place pod resize.
1 parent e0d388f commit 7d9ced4

19 files changed

Lines changed: 1076 additions & 304 deletions

File tree

‎cmd/ateapi/internal/controlapi/functionaltest/common_test.go‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -526,15 +526,15 @@ func createWorkerPod(t *testing.T, tc *testContext, ns string, name string, node
526526
NodeName: nodeName,
527527
SandboxClass: string(pool.Spec.SandboxClass),
528528
Labels: pool.GetLabels(),
529-
// No capacity: the syncer takes the compute dimensions from the pod's
530-
// limits, and these pods declare none. The actors ceiling is the
531-
// ateom's and arrives by report, which unset reads as one — enough
532-
// for a test that places a single Actor per worker. See
533-
// setWorkerActorCapacity for the tests that need more.
529+
// Capacity is not settable here: a Worker gets it from its own
530+
// ateom's report, which the reportWorkerCapacity below stands in
531+
// for. These pods declare no limits, so only the actor ceiling is
532+
// reported; see setWorkerActorCapacity for tests needing more.
534533
},
535534
}); err != nil {
536535
t.Fatalf("failed to register worker: %v", err)
537536
}
537+
reportWorkerCapacity(t, tc, string(createdPod.UID), 1)
538538

539539
// Wait for the worker to appear in worker cache.
540540
err = wait.PollUntilContextTimeout(context.Background(), 10*time.Millisecond, 5*time.Second, true, func(ctx context.Context) (bool, error) {

‎cmd/ateapi/internal/controlapi/functionaltest/worker_contention_test.go‎

Lines changed: 31 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -135,38 +135,47 @@ func TestResumeActor_ConcurrentOntoOneWorker(t *testing.T) {
135135
// the scheduler's cache to see it, since placement reads that and not the store.
136136
func setWorkerActorCapacity(t *testing.T, tc *testContext, pool string, actors int32) {
137137
t.Helper()
138-
ctx := context.Background()
139-
page, err := tc.persistence.ListWorkers(ctx, store.ListOptions{PageSize: 1000})
138+
page, err := tc.persistence.ListWorkers(context.Background(), store.ListOptions{PageSize: 1000})
140139
if err != nil {
141140
t.Fatalf("listing workers: %v", err)
142141
}
143-
var names []string
142+
var reported int
144143
for _, w := range page.Items {
145144
if w.GetWorkerPool() != pool {
146145
continue
147146
}
148-
name := w.GetMetadata().GetName()
149-
if _, err := tc.persistence.UpdateWorker(ctx, name, store.PreconditionFrom(w), func(toUpdate *ateapipb.Worker) error {
150-
if toUpdate.Status.Capacity == nil {
151-
toUpdate.Status.Capacity = &ateapipb.WorkerCapacity{}
152-
}
153-
toUpdate.Status.Capacity.Actors = actors
154-
return nil
155-
}); err != nil {
156-
t.Fatalf("setting capacity on worker %s: %v", name, err)
157-
}
158-
names = append(names, name)
147+
reportWorkerCapacity(t, tc, w.GetMetadata().GetName(), actors)
148+
reported++
159149
}
160-
if len(names) == 0 {
150+
if reported == 0 {
161151
t.Fatalf("no workers in pool %q to give capacity", pool)
162152
}
163-
for _, name := range names {
164-
if err := wait.PollUntilContextTimeout(ctx, 50*time.Millisecond, 5*time.Second, true,
165-
func(context.Context) (bool, error) {
166-
got, err := tc.workerCache.Worker(name)
167-
return err == nil && got.GetStatus().GetCapacity().GetActors() == actors, nil
168-
}); err != nil {
169-
t.Fatalf("worker %s did not reach capacity.actors=%d in the cache: %v", name, actors, err)
153+
}
154+
155+
// reportWorkerCapacity stands in for the ateom's capacity report, which atelet
156+
// forwards to WorkerService in a real cluster. It waits for the worker cache,
157+
// which placement reads, to catch up.
158+
func reportWorkerCapacity(t *testing.T, tc *testContext, name string, actors int32) {
159+
t.Helper()
160+
ctx := context.Background()
161+
worker, err := tc.persistence.GetWorker(ctx, name)
162+
if err != nil {
163+
t.Fatalf("getting worker %s: %v", name, err)
164+
}
165+
if _, err := tc.persistence.UpdateWorker(ctx, name, store.PreconditionFrom(worker), func(toUpdate *ateapipb.Worker) error {
166+
if toUpdate.Status.Capacity == nil {
167+
toUpdate.Status.Capacity = &ateapipb.WorkerCapacity{}
170168
}
169+
toUpdate.Status.Capacity.Actors = actors
170+
return nil
171+
}); err != nil {
172+
t.Fatalf("setting capacity on worker %s: %v", name, err)
173+
}
174+
if err := wait.PollUntilContextTimeout(ctx, 10*time.Millisecond, 5*time.Second, true,
175+
func(context.Context) (bool, error) {
176+
got, err := tc.workerCache.Worker(name)
177+
return err == nil && got.GetStatus().GetCapacity().GetActors() == actors, nil
178+
}); err != nil {
179+
t.Fatalf("worker %s did not reach capacity.actors=%d in the cache: %v", name, actors, err)
171180
}
172181
}

‎cmd/ateapi/internal/controlapi/functionaltest/worker_test.go‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -54,8 +54,9 @@ func TestListWorkers(t *testing.T) {
5454
want := []*ateapipb.Worker{
5555
{
5656
Metadata: &ateapipb.ResourceMetadata{
57-
Name: podUID,
58-
Version: 1,
57+
Name: podUID,
58+
// Two writes: the registration, then the capacity report.
59+
Version: 2,
5960
},
6061
WorkerNamespace: ns,
6162
WorkerPool: "pool1",

‎cmd/ateapi/internal/controlapi/worker.go‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -145,10 +145,10 @@ func (s *ServiceImpl) CreateWorker(ctx context.Context, inWorker *ateapipb.Worke
145145
outWorker := proto.CloneOf(inWorker)
146146
outWorker.Status = &ateapipb.WorkerStatus{State: ateapipb.WorkerState_WORKER_STATE_ACTIVE}
147147

148-
// Reify the actor ceiling so every stored Worker carries one and no reader
149-
// has to know a default. A Worker that has not reported is worth one Actor,
150-
// which is what a Worker was before it could report.
151-
outWorker.Status.Capacity = &ateapipb.WorkerCapacity{Actors: 1}
148+
// Capacity is left unset: a Worker holds nothing until its own ateom says
149+
// what it has, through WorkerService.SetWorkerCapacity. Nothing is placed
150+
// on it in the meantime, which is the point -- the alternative is guessing
151+
// on the Worker's behalf and placing against the guess.
152152

153153
// Verify that the result is properly valid before storing it.
154154
if errs := validateWorkerUpdate(ctx, field.NewPath("worker"), outWorker, inWorker, true); len(errs) > 0 {

‎cmd/ateapi/internal/controlapi/worker_test.go‎

Lines changed: 7 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -285,10 +285,7 @@ func TestCreateWorker_IgnoresRequestStatus(t *testing.T) {
285285
if err != nil {
286286
t.Fatalf("CreateWorker() failed: %v", err)
287287
}
288-
want := &ateapipb.WorkerStatus{
289-
State: ateapipb.WorkerState_WORKER_STATE_ACTIVE,
290-
Capacity: &ateapipb.WorkerCapacity{Actors: 1},
291-
}
288+
want := &ateapipb.WorkerStatus{State: ateapipb.WorkerState_WORKER_STATE_ACTIVE}
292289
if diff := cmp.Diff(want, got.GetStatus(), protocmp.Transform()); diff != "" {
293290
t.Errorf("created worker status mismatch (-want +got):\n%s", diff)
294291
}
@@ -1053,27 +1050,27 @@ func TestCreateWorker_IgnoresRequestMetadataServerFields(t *testing.T) {
10531050
// Every stored Worker carries an actor ceiling, so no reader has to know a
10541051
// default. A Worker that reports its own keeps it; one that does not is worth
10551052
// one Actor, which is what a Worker was before it could report.
1056-
func TestCreateWorker_ReifiesActorCeiling(t *testing.T) {
1053+
func TestCreateWorker_HoldsNoCapacityUntilReported(t *testing.T) {
10571054
ctx := context.Background()
10581055
svc, _ := newWorkerAPIService(t)
10591056

10601057
got, err := svc.CreateWorker(ctx, &ateapipb.CreateWorkerRequest{Worker: validWorker(apiWorkerName)})
10611058
if err != nil {
10621059
t.Fatalf("CreateWorker() failed: %v", err)
10631060
}
1064-
if got := got.GetStatus().GetCapacity().GetActors(); got != 1 {
1065-
t.Errorf("created worker actor ceiling = %d, want 1", got)
1061+
if capacity := got.GetStatus().GetCapacity(); capacity != nil {
1062+
t.Errorf("created worker capacity = %v, want none until its ateom reports", capacity)
10661063
}
10671064

10681065
// Capacity is status, so a request cannot bring its own: a Worker only
1069-
// gets a real ceiling by reporting one.
1066+
// gets one by reporting it.
10701067
carried := validWorker("11111111-2222-3333-4444-555555555555")
10711068
carried.Status = &ateapipb.WorkerStatus{Capacity: &ateapipb.WorkerCapacity{Actors: 4094}}
10721069
got, err = svc.CreateWorker(ctx, &ateapipb.CreateWorkerRequest{Worker: carried})
10731070
if err != nil {
10741071
t.Fatalf("CreateWorker() carrying a capacity failed: %v", err)
10751072
}
1076-
if got := got.GetStatus().GetCapacity().GetActors(); got != 1 {
1077-
t.Errorf("a request carrying a ceiling set it to %d, want the reified 1", got)
1073+
if capacity := got.GetStatus().GetCapacity(); capacity != nil {
1074+
t.Errorf("a request carrying a ceiling set capacity to %v, want none", capacity)
10781075
}
10791076
}

‎cmd/atecontroller/internal/controllers/workerpool_apply.go‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ import (
2424
corev1ac "k8s.io/client-go/applyconfigurations/core/v1"
2525
metav1ac "k8s.io/client-go/applyconfigurations/meta/v1"
2626

27+
"github.com/agent-substrate/substrate/internal/ateomcapacity"
2728
"github.com/agent-substrate/substrate/internal/ateompath"
2829
"github.com/agent-substrate/substrate/internal/deviceplugin"
2930
atev1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1"
@@ -204,6 +205,10 @@ func buildDeploymentApplyConfig(wp *atev1alpha1.WorkerPool, otel ateomOTelSettin
204205
func ateomContainerEnv(otel ateomOTelSettings) []*corev1ac.EnvVarApplyConfiguration {
205206
envs := []*corev1ac.EnvVarApplyConfiguration{
206207
fieldRefEnv("POD_UID", "metadata.uid"),
208+
// What the ateom reports as its capacity. A container without a limit
209+
// gets the node's allocatable, which is what it can in fact use.
210+
resourceFieldRefEnv(ateomcapacity.CPULimitEnv, "limits.cpu", milliCores),
211+
resourceFieldRefEnv(ateomcapacity.MemoryLimitEnv, "limits.memory", wholeBytes),
207212
}
208213
if otel.Endpoint == "" {
209214
return envs
@@ -238,6 +243,25 @@ func ateomContainerEnv(otel ateomOTelSettings) []*corev1ac.EnvVarApplyConfigurat
238243
return envs
239244
}
240245

246+
// Divisors for resourceFieldRefEnv. The downward API reports
247+
// ceil(limit/divisor), so these are the units the value arrives in; the default
248+
// divisor of one core would round a fractional CPU limit up to a whole one.
249+
const (
250+
milliCores = "1m"
251+
wholeBytes = "1"
252+
)
253+
254+
// resourceFieldRefEnv reports a container resource in units of divisor.
255+
func resourceFieldRefEnv(name, resourceName, divisor string) *corev1ac.EnvVarApplyConfiguration {
256+
return corev1ac.EnvVar().
257+
WithName(name).
258+
WithValueFrom(corev1ac.EnvVarSource().
259+
WithResourceFieldRef(corev1ac.ResourceFieldSelector().
260+
WithContainerName("ateom").
261+
WithResource(resourceName).
262+
WithDivisor(resource.MustParse(divisor))))
263+
}
264+
241265
func fieldRefEnv(name, fieldPath string) *corev1ac.EnvVarApplyConfiguration {
242266
return corev1ac.EnvVar().
243267
WithName(name).

‎cmd/atecontroller/internal/controllers/workerpool_apply_test.go‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ import (
2828
corev1ac "k8s.io/client-go/applyconfigurations/core/v1"
2929
metav1ac "k8s.io/client-go/applyconfigurations/meta/v1"
3030

31+
"github.com/agent-substrate/substrate/internal/ateomcapacity"
3132
"github.com/agent-substrate/substrate/internal/ateompath"
3233
"github.com/agent-substrate/substrate/internal/deviceplugin"
3334
atev1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1"
@@ -756,6 +757,8 @@ func expectedDeploymentApplyConfig(mutatePodSpec func(*corev1ac.PodSpecApplyConf
756757
WithValueFrom(corev1ac.EnvVarSource().
757758
WithFieldRef(corev1ac.ObjectFieldSelector().
758759
WithFieldPath("metadata.uid"))),
760+
resourceFieldRefEnv(ateomcapacity.CPULimitEnv, "limits.cpu", milliCores),
761+
resourceFieldRefEnv(ateomcapacity.MemoryLimitEnv, "limits.memory", wholeBytes),
759762
).
760763
WithVolumeMounts(
761764
corev1ac.VolumeMount().

‎cmd/atelet/main.go‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -346,6 +346,9 @@ func main() {
346346
ateletpb.RegisterCredentialBrokerServer(brokerServer, &credentialBroker{
347347
actorIdentityClient: ateapipb.NewActorIdentityClient(ateapiConn),
348348
})
349+
ateletpb.RegisterWorkerCapacityServer(brokerServer, &workerCapacityService{
350+
workers: ateapipb.NewWorkerServiceClient(ateapiConn),
351+
})
349352
go func() {
350353
if err := brokerServer.Serve(brokerLis); err != nil {
351354
serverboot.Fatal(ctx, "Failed to serve credential broker", err)

‎cmd/atelet/workercapacity.go‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
// Copyright 2026 Google LLC
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
package main
16+
17+
import (
18+
"context"
19+
"log/slog"
20+
21+
"github.com/agent-substrate/substrate/internal/proto/ateletpb"
22+
"github.com/agent-substrate/substrate/internal/resources"
23+
"github.com/agent-substrate/substrate/pkg/proto/ateapipb"
24+
)
25+
26+
// workerCapacityService forwards a worker's own account of what it can supply
27+
// to the control plane. The worker is the only thing that knows: the control
28+
// plane sees a Pod, not what the runtime will actually give an actor.
29+
type workerCapacityService struct {
30+
ateletpb.UnimplementedWorkerCapacityServer
31+
32+
workers ateapipb.WorkerServiceClient
33+
}
34+
35+
// ReportWorkerCapacity records what the calling worker says it has.
36+
//
37+
// It returns the control plane's error unwrapped so the caller retries: a
38+
// worker reports once, so an accepted call is the only thing that puts
39+
// capacity on the Worker, and a Worker record the syncer has not created yet
40+
// is the ordinary reason for a first attempt to fail.
41+
func (s *workerCapacityService) ReportWorkerCapacity(ctx context.Context, req *ateletpb.ReportWorkerCapacityRequest) (*ateletpb.ReportWorkerCapacityResponse, error) {
42+
// Identity comes only from the mTLS certificate, never from the request:
43+
// a worker can report its own capacity and no one else's.
44+
workerIdentity, err := authenticatedWorkerIdentity(ctx)
45+
if err != nil {
46+
return nil, err
47+
}
48+
capacity := &ateapipb.WorkerCapacity{
49+
Actors: req.GetActors(),
50+
// A dimension the worker could not determine is left out, which the
51+
// control plane reads as none of it.
52+
Resources: resources.CPUMemory(req.GetCpuMilli(), req.GetMemoryBytes()),
53+
}
54+
if _, err := s.workers.SetWorkerCapacity(ctx, &ateapipb.SetWorkerCapacityRequest{
55+
// Workers are global-scoped and named by their pod UID.
56+
Worker: &ateapipb.ObjectRef{Name: workerIdentity.PodUID},
57+
Capacity: capacity,
58+
}); err != nil {
59+
return nil, err
60+
}
61+
slog.InfoContext(ctx, "Recorded worker capacity",
62+
slog.String("pod_uid", workerIdentity.PodUID), slog.Any("capacity", capacity))
63+
return &ateletpb.ReportWorkerCapacityResponse{}, nil
64+
}

0 commit comments

Comments
 (0)