Skip to content

Commit 6dfa5c7

Browse files
committed
feat(platform): migrate Postgres accounts to platform-minted credentials
The converted postgres chart reads each account's verifier from an account Secret the API writes, and nothing seeds those Secrets for the applications that exist before the chart switches: the API seeds only on a write of the application, and the old chart keeps serving the old passwords from <release>-credentials until the switch. Without a migration every existing account would stop logging in at the switch, since the operator leaves a role whose Secret is missing as it is but the chart no longer applies the old password either. Migration 60 runs in the pre-upgrade hook, before any component chart changes version, and writes the account Secrets with the verifiers of the chart-generated passwords, origin Migrated, so clients keep logging in at every point, and records the chart version of each release. A Job the platform release creates, not a hook so a long wait does not hold the upgrade and so it can be run again by hand, waits per release until the converted chart is Ready, removes any users.*.password left in the values and deletes <release>-credentials, which no platform account of postgres shares. It exits non-zero while a release is left out. The verifier is computed by the API's own format function, built into a CLI the migrations image carries, so a migrated release and a mint store the same form; the image is built from the repository root for that reason. Assisted-by: LLM Signed-off-by: Timofei Larkin <[email protected]>
1 parent 97a3eaa commit 6dfa5c7

8 files changed

Lines changed: 940 additions & 5 deletions

File tree

‎cmd/credentials-migrate/main.go‎

Lines changed: 519 additions & 0 deletions
Large diffs are not rendered by default.
Lines changed: 221 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,221 @@
1+
// SPDX-License-Identifier: Apache-2.0
2+
3+
package main
4+
5+
import (
6+
"context"
7+
"encoding/json"
8+
"strings"
9+
"testing"
10+
"time"
11+
12+
helmv2 "github.com/fluxcd/helm-controller/api/v2"
13+
corev1 "k8s.io/api/core/v1"
14+
apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
15+
apierrors "k8s.io/apimachinery/pkg/api/errors"
16+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
17+
"k8s.io/apimachinery/pkg/runtime"
18+
"k8s.io/apimachinery/pkg/types"
19+
"sigs.k8s.io/controller-runtime/pkg/client"
20+
"sigs.k8s.io/controller-runtime/pkg/client/fake"
21+
22+
appsv1alpha1 "github.com/cozystack/cozystack/pkg/apis/apps/v1alpha1"
23+
corev1alpha1 "github.com/cozystack/cozystack/pkg/apis/core/v1alpha1"
24+
"github.com/cozystack/cozystack/pkg/credentials/account"
25+
"github.com/cozystack/cozystack/pkg/credentials/format"
26+
)
27+
28+
func newMigrator(t *testing.T, objs ...client.Object) *migrator {
29+
t.Helper()
30+
scheme := runtime.NewScheme()
31+
if err := corev1.AddToScheme(scheme); err != nil {
32+
t.Fatal(err)
33+
}
34+
if err := helmv2.AddToScheme(scheme); err != nil {
35+
t.Fatal(err)
36+
}
37+
c := fake.NewClientBuilder().WithScheme(scheme).WithObjects(objs...).WithStatusSubresource(&helmv2.HelmRelease{}).Build()
38+
o := &options{
39+
migration: 60, kind: "Postgres", formatName: format.ScramSHA256, accountsPath: "users",
40+
labels: labelFlags{"cnpg.io/reload": "true"}, sourceSuffix: "-credentials",
41+
stateNamespace: "cozy-system", actor: "migration", timeout: -time.Second,
42+
}
43+
return &migrator{c: c, o: o, now: func() time.Time { return time.Date(2026, 10, 14, 12, 0, 0, 0, time.UTC) }}
44+
}
45+
46+
func release(values string, chartVersion string, ready bool) *helmv2.HelmRelease {
47+
hr := &helmv2.HelmRelease{
48+
ObjectMeta: metav1.ObjectMeta{Name: "postgres-orders", Namespace: "tenant-foo", UID: "hr-uid", Labels: map[string]string{
49+
appsv1alpha1.ApplicationGroupLabel: appsv1alpha1.GroupName,
50+
appsv1alpha1.ApplicationKindLabel: "Postgres",
51+
appsv1alpha1.ApplicationNameLabel: "orders",
52+
}},
53+
Spec: helmv2.HelmReleaseSpec{Values: &apiextv1.JSON{Raw: []byte(values)}},
54+
}
55+
if chartVersion != "" {
56+
hr.Status.History = helmv2.Snapshots{{Version: 1, ChartVersion: chartVersion, Status: "deployed"}}
57+
}
58+
if ready {
59+
hr.Status.Conditions = []metav1.Condition{{Type: "Ready", Status: metav1.ConditionTrue, Reason: "x", LastTransitionTime: metav1.Now()}}
60+
}
61+
return hr
62+
}
63+
64+
func sourceSecret(data map[string]string) *corev1.Secret {
65+
sec := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "postgres-orders-credentials", Namespace: "tenant-foo"}, Data: map[string][]byte{}}
66+
for k, v := range data {
67+
sec.Data[k] = []byte(v)
68+
}
69+
return sec
70+
}
71+
72+
func (m *migrator) secret(t *testing.T, name string) (*corev1.Secret, bool) {
73+
t.Helper()
74+
sec := &corev1.Secret{}
75+
err := m.c.Get(context.Background(), types.NamespacedName{Namespace: "tenant-foo", Name: name}, sec)
76+
if apierrors.IsNotFound(err) {
77+
return nil, false
78+
}
79+
if err != nil {
80+
t.Fatal(err)
81+
}
82+
return sec, true
83+
}
84+
85+
func TestSeedRecordsAndMigrates(t *testing.T) {
86+
ctx := context.Background()
87+
m := newMigrator(t, release(`{"users":{"web":{},"Api.1":{}}}`, "0.1.0", true), sourceSecret(map[string]string{"web": "hunter2", "Api.1": "secret"}))
88+
if err := m.seed(ctx); err != nil {
89+
t.Fatal(err)
90+
}
91+
st, err := m.loadState(ctx)
92+
if err != nil {
93+
t.Fatal(err)
94+
}
95+
rec := st.Releases["tenant-foo/postgres-orders"]
96+
if rec == nil || rec.ChartVersion != "0.1.0" || rec.Done || len(st.Skipped) != 0 {
97+
t.Fatalf("state = %+v", st)
98+
}
99+
for _, c := range []struct{ secret, user, plain string }{{"postgres-orders.web.account", "web", "hunter2"}, {"postgres-orders.api-1.account", "Api.1", "secret"}} {
100+
sec, ok := m.secret(t, c.secret)
101+
if !ok {
102+
t.Fatalf("%s not written", c.secret)
103+
}
104+
if string(sec.Data[account.UsernameKey]) != c.user || !strings.HasPrefix(string(sec.Data[account.PasswordKey]), "SCRAM-SHA-256$4096:") {
105+
t.Fatalf("%s data = %q", c.secret, sec.Data)
106+
}
107+
if strings.Contains(string(sec.Data[account.PasswordKey]), c.plain) {
108+
t.Fatal("the plaintext must not be stored")
109+
}
110+
if sec.Labels["cnpg.io/reload"] != "true" || sec.Labels[account.AccountLabel] != "true" || len(sec.OwnerReferences) != 1 || sec.OwnerReferences[0].UID != "hr-uid" {
111+
t.Fatalf("%s meta = %+v", c.secret, sec.ObjectMeta)
112+
}
113+
r := account.ReadRecord(sec)
114+
if r.Origin != corev1alpha1.CredentialOriginMigrated || r.By != "migration" || len(r.History) != 1 || r.History[0].Operation != corev1alpha1.CredentialOperationMigrate {
115+
t.Fatalf("%s record = %+v", c.secret, r)
116+
}
117+
}
118+
// A second run keeps what it wrote.
119+
before, _ := m.secret(t, "postgres-orders.web.account")
120+
if err := m.seed(ctx); err != nil {
121+
t.Fatal(err)
122+
}
123+
after, _ := m.secret(t, "postgres-orders.web.account")
124+
if string(after.Data[account.PasswordKey]) != string(before.Data[account.PasswordKey]) {
125+
t.Fatal("seed must be idempotent")
126+
}
127+
}
128+
129+
func TestSeedLeavesOutUnresolvableAndRecordsMissingSource(t *testing.T) {
130+
ctx := context.Background()
131+
colliding := release(`{"users":{"web_app":{},"web-app":{}}}`, "0.1.0", true)
132+
m := newMigrator(t, colliding)
133+
if err := m.seed(ctx); err != nil {
134+
t.Fatal(err)
135+
}
136+
st, _ := m.loadState(ctx)
137+
if why := st.Skipped["tenant-foo/postgres-orders"]; !strings.Contains(why, "both map to the name") {
138+
t.Fatalf("skipped = %v", st.Skipped)
139+
}
140+
evs := &corev1.EventList{}
141+
if err := m.c.List(ctx, evs, client.InNamespace("tenant-foo")); err != nil || len(evs.Items) != 1 || evs.Items[0].Reason != reasonSkipped {
142+
t.Fatalf("events = %+v err=%v", evs.Items, err)
143+
}
144+
145+
// No source Secret: recorded, nothing seeded, not skipped.
146+
m = newMigrator(t, release(`{"users":{"web":{}}}`, "0.1.0", true))
147+
if err := m.seed(ctx); err != nil {
148+
t.Fatal(err)
149+
}
150+
st, _ = m.loadState(ctx)
151+
if st.Releases["tenant-foo/postgres-orders"] == nil || len(st.Skipped) != 0 {
152+
t.Fatalf("state = %+v", st)
153+
}
154+
if _, ok := m.secret(t, "postgres-orders.web.account"); ok {
155+
t.Fatal("nothing to seed from")
156+
}
157+
}
158+
159+
func TestFinishWaitsForTheSwitchThenStripsAndRetires(t *testing.T) {
160+
ctx := context.Background()
161+
hr := release(`{"users":{"web":{"password":"leftover"}},"size":"10Gi"}`, "0.1.0", true)
162+
m := newMigrator(t, hr, sourceSecret(map[string]string{"web": "hunter2"}))
163+
if err := m.seed(ctx); err != nil {
164+
t.Fatal(err)
165+
}
166+
// Still on the recorded chart version: left out, reported, state kept.
167+
err := m.finish(ctx)
168+
if err == nil || !strings.Contains(err.Error(), "1 release(s) left out") {
169+
t.Fatalf("finish on an unswitched release: %v", err)
170+
}
171+
if _, ok := m.secret(t, "postgres-orders-credentials"); !ok {
172+
t.Fatal("the source must survive until the chart has switched")
173+
}
174+
175+
// Switched and Ready.
176+
cur := &helmv2.HelmRelease{}
177+
if err := m.c.Get(ctx, client.ObjectKeyFromObject(hr), cur); err != nil {
178+
t.Fatal(err)
179+
}
180+
cur.Status.History = helmv2.Snapshots{{Version: 2, ChartVersion: "0.2.0", Status: "deployed"}}
181+
if err := m.c.Status().Update(ctx, cur); err != nil {
182+
t.Fatal(err)
183+
}
184+
if err := m.finish(ctx); err != nil {
185+
t.Fatal(err)
186+
}
187+
st, _ := m.loadState(ctx)
188+
if rec := st.Releases["tenant-foo/postgres-orders"]; rec == nil || !rec.Done || len(st.Skipped) != 0 {
189+
t.Fatalf("state = %+v", st)
190+
}
191+
if err := m.c.Get(ctx, client.ObjectKeyFromObject(hr), cur); err != nil {
192+
t.Fatal(err)
193+
}
194+
var values map[string]any
195+
if err := json.Unmarshal(cur.Spec.Values.Raw, &values); err != nil {
196+
t.Fatal(err)
197+
}
198+
if _, has := values["users"].(map[string]any)["web"].(map[string]any)["password"]; has || values["size"] != "10Gi" {
199+
t.Fatalf("values = %v", values)
200+
}
201+
if _, ok := m.secret(t, "postgres-orders-credentials"); ok {
202+
t.Fatal("the source Secret of a kind without platform accounts must be deleted")
203+
}
204+
if _, ok := m.secret(t, "postgres-orders.web.account"); !ok {
205+
t.Fatal("the account Secret must stay")
206+
}
207+
}
208+
209+
func TestRetireSourceKeepsPlatformKeys(t *testing.T) {
210+
ctx := context.Background()
211+
hr := release(`{"users":{"web":{}}}`, "0.1.0", true)
212+
m := newMigrator(t, hr, sourceSecret(map[string]string{"web": "hunter2", "root": "toor"}))
213+
m.o.platformKeys = stringFlags{"root"}
214+
if err := m.retireSource(ctx, hr); err != nil {
215+
t.Fatal(err)
216+
}
217+
sec, ok := m.secret(t, "postgres-orders-credentials")
218+
if !ok || string(sec.Data["root"]) != "toor" || len(sec.Data) != 1 {
219+
t.Fatalf("source = %v", sec)
220+
}
221+
}

‎packages/core/platform/Makefile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ image-migrations:
3636
rm -rf images/migrations/etcd-operator-crds
3737
mkdir -p images/migrations/etcd-operator-crds
3838
cp ../../system/etcd-operator-crds/templates/*.yaml images/migrations/etcd-operator-crds/
39-
docker buildx build images/migrations \
39+
docker buildx build -f images/migrations/Dockerfile ../../.. \
4040
$(call image-tags,platform-migrations,v$(COZYSTACK_VERSION)) \
4141
$(call cache-args,platform-migrations) \
4242
--metadata-file images/migrations.json \

‎packages/core/platform/images/migrations/Dockerfile‎

Lines changed: 21 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,23 @@
55
# target arch and verify it against the release's cli-SHA256SUMS.txt. Bump
66
# ETCD_OPERATOR_VERSION (and the pinned checksums below) together, keeping it in
77
# sync with packages/system/etcd-operator (Chart appVersion).
8+
# credentials-migrate is built from this repository, because it shares the
9+
# password format functions with the API: a migrated release and a mint must
10+
# store the same verifier. The build context is the repository root for that
11+
# reason, and every other path below is relative to it.
12+
FROM --platform=$BUILDPLATFORM golang:1.27-alpine@sha256:8a5910f31396cd4d89662f56c68b3ae31d374308270a1c3bd96672ee5ed43414 AS builder
13+
ARG GOPROXY
14+
ARG TARGETOS
15+
ARG TARGETARCH
16+
WORKDIR /workspace
17+
COPY go.mod go.sum ./
18+
RUN for attempt in 1 2 3; do GOOS=$TARGETOS GOARCH=$TARGETARCH go mod download && break; [ "$attempt" = 3 ] && exit 1; sleep $((attempt * 10)); done
19+
COPY api api/
20+
COPY pkg pkg/
21+
COPY cmd cmd/
22+
COPY internal internal/
23+
RUN GOOS=$TARGETOS GOARCH=$TARGETARCH CGO_ENABLED=0 go build -ldflags="-extldflags=-static" -o /credentials-migrate ./cmd/credentials-migrate
24+
825
FROM alpine:3.24@sha256:294b683cb724975bec92580e1e685676bd4b50bda910ddb8c51d4cabeaec77e6
926

1027
ARG TARGETOS
@@ -29,13 +46,14 @@ RUN set -eux; \
2946
echo "${sha256} /usr/bin/etcd-migrate" | sha256sum -c -; \
3047
chmod +x /usr/bin/etcd-migrate
3148

32-
COPY migrations /migrations
49+
COPY --from=builder /credentials-migrate /usr/bin/credentials-migrate
50+
COPY packages/core/platform/images/migrations/migrations /migrations
3351
# v1alpha2 etcd CRDs, vendored into the build context by `make image-migrations`.
3452
# migration 50 applies these before adoption so it does not deadlock waiting for
3553
# the etcd-operator-crds HelmRelease (which only reconciles after the migration
3654
# pre-upgrade hook completes).
37-
COPY etcd-operator-crds /etcd-operator-crds
38-
COPY run-migrations.sh /usr/bin/run-migrations.sh
55+
COPY packages/core/platform/images/migrations/etcd-operator-crds /etcd-operator-crds
56+
COPY packages/core/platform/images/migrations/run-migrations.sh /usr/bin/run-migrations.sh
3957

4058
WORKDIR /migrations
4159

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
#!/bin/sh
2+
# Migration 60 --> 61
3+
# Convert PostgreSQL accounts to platform-minted credentials: the first
4+
# part of the user secrets API conversion for the Postgres kind.
5+
#
6+
# This release switches the postgres chart to CloudNativePG managed roles
7+
# that read each account's SCRAM verifier from an account Secret the
8+
# Cozystack API writes, and stops rendering <release>-credentials. Until
9+
# the charts switch, the old chart keeps serving the old passwords from
10+
# that Secret, so this hook, which runs before any component chart changes
11+
# version, records the chart version of every Postgres release and writes
12+
# the account Secrets with the verifiers of the passwords in
13+
# <release>-credentials, origin Migrated, so clients keep logging in at
14+
# every point of the upgrade. The API's own format function computes the
15+
# verifier, compiled into credentials-migrate, so a migrated release and a
16+
# mint store the same form.
17+
#
18+
# The second part, the cozystack-credentials-migration-60 Job the platform
19+
# release creates once the charts switch, waits per release for the
20+
# converted chart to be Ready, removes any users.*.password left in the
21+
# values and deletes <release>-credentials, which no platform account of
22+
# postgres shares. It is not a hook, can be run again by hand, and exits
23+
# non-zero while a release is left out; the state is in the
24+
# credentials-migration-60 ConfigMap in cozy-system.
25+
#
26+
# A release whose accounts cannot be resolved, two users mapping to one
27+
# account name, is left out here with an Event on its HelmRelease and keeps
28+
# working as before; the Job reports it until the cause is fixed. A release
29+
# whose <release>-credentials is gone is recorded anyway: its database keeps
30+
# its passwords, and a tenant mints by name once the chart has switched.
31+
#
32+
# Fail-closed: migrations never re-run, so an error here stops the upgrade
33+
# rather than stamping the version past a release that was not recorded.
34+
35+
set -euo pipefail
36+
37+
# shellcheck source=lib/cozystack-version.sh
38+
. "$(dirname "$0")/lib/cozystack-version.sh"
39+
40+
credentials-migrate seed \
41+
--migration 60 \
42+
--kind Postgres \
43+
--format scram-sha-256 \
44+
--accounts-path users \
45+
--label 'cnpg.io/reload=true'
46+
47+
stamp_cozystack_version 61
Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
{{- if .Values.migrations.enabled }}
2+
{{- /*
3+
Second part of the credentials conversion of migration 60 (Postgres): a Job
4+
rather than a hook, because it waits for every converted release to be
5+
Ready on the switched chart, which may take longer than an upgrade should
6+
hold, and because it has to be run again by hand once an operator fixes a
7+
release it left out. It is idempotent: credentials-migrate keeps its state
8+
in the credentials-migration-60 ConfigMap and acts only on releases the
9+
hook recorded that are not done. The spec never changes, so the platform
10+
release re-applies it as a no-op; to run it again, delete the Job and
11+
reconcile the platform HelmRelease. It exits non-zero while a release is
12+
left out, so the failed Job is the one place that shows what remains.
13+
cluster-admin, as the hook: it writes Secrets and HelmRelease values in
14+
every tenant namespace.
15+
*/}}
16+
---
17+
apiVersion: v1
18+
kind: ServiceAccount
19+
metadata:
20+
name: cozystack-credentials-migration
21+
---
22+
apiVersion: rbac.authorization.k8s.io/v1
23+
kind: ClusterRoleBinding
24+
metadata:
25+
name: cozystack-credentials-migration
26+
roleRef:
27+
apiGroup: rbac.authorization.k8s.io
28+
kind: ClusterRole
29+
name: cluster-admin
30+
subjects:
31+
- kind: ServiceAccount
32+
name: cozystack-credentials-migration
33+
namespace: {{ .Release.Namespace | quote }}
34+
---
35+
apiVersion: batch/v1
36+
kind: Job
37+
metadata:
38+
name: cozystack-credentials-migration-60
39+
spec:
40+
backoffLimit: 6
41+
template:
42+
metadata:
43+
labels:
44+
policy.cozystack.io/allow-to-apiserver: "true"
45+
spec:
46+
serviceAccountName: cozystack-credentials-migration
47+
restartPolicy: OnFailure
48+
containers:
49+
- name: migration
50+
image: {{ .Values.migrations.image }}
51+
command:
52+
- credentials-migrate
53+
- finish
54+
- --migration=60
55+
- --kind=Postgres
56+
- --format=scram-sha-256
57+
- --accounts-path=users
58+
- --label=cnpg.io/reload=true
59+
- --timeout=30m
60+
env:
61+
- name: CREDENTIALS_MIGRATE_ACTOR
62+
value: system:serviceaccount:{{ .Release.Namespace }}:cozystack-credentials-migration
63+
{{- end }}

0 commit comments

Comments
 (0)