Skip to content

Commit a935fec

Browse files
saravmajesticclaude
andcommitted
fix(learn): address sync re-review findings
- Fan a refused `/batch` out to single items only on request-validation answers (400, 413, 422); a 401 or any other answer stops the push and leaves the queue intact - `promotionHooks.afterPublish` never fails a published promotion: a failed enqueue is logged and reopens the backfill, so the next push derives the proposals again - Scope the `unsupported_until` backoff: `forScope` clears it, both checks read it for the current scope, and a pull failure is recorded under its scope - Delivery hides a local lesson when a workspace-wide team lesson has its key, matching the `learn show` (team) marker - Count and log a held proposal `learn push --resubmit` drops; a conflicted remove is satisfied only when a pulled cache shows it gone - CLI error after a failed push points at the push summary; correct the backfill comment - Tests: 401 on `/batch`, enqueue failure after publish, account switch after backoff, workspace-wide lesson through delivery and `learn show`, and backend-generated `submission_hash` cases pinned against `submissionHash` and the test server; the test server refuses a request with 400 `VALIDATION_ERROR` as the backend does Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
1 parent 4e6e378 commit a935fec

10 files changed

Lines changed: 265 additions & 20 deletions

File tree

‎packages/opencode/src/altimate/learn/effective.ts‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,8 @@
33
// The effective lesson set when sync is on: local lessons plus the approved team lessons pulled from the
44
// workspace (`remote.json`), keyed by qualified identity (repo_identity, store, lesson_key).
55
//
6-
// - For the same identity the remote lesson wins, and an exact-identity tombstone hides the local copy.
6+
// - For the same identity the remote lesson wins, and an exact-identity tombstone hides the local copy. A
7+
// workspace-wide team lesson (no repo_identity) with the same key also stands in for the local copy.
78
// Local lessons always have this repository's identity; pending or rejected proposals never hide anything.
89
// - Remote lessons become `Lesson` records with an `L-` id: their key, or a stable alias derived from the
910
// identity when that key is already taken (another repository's lesson with the same key, under sharing).
@@ -46,9 +47,11 @@ export function resolve(store: string, localIds: readonly string[], remote: Remo
4647
const lessons = remote.lessons.filter((lesson) => lesson.store === store)
4748
const live = new Set(lessons.map((lesson) => identityKey(remoteIdentity(lesson))))
4849
const tombstones = new Set(remote.tombstones.filter((t) => t.store === store).map(identityKey))
50+
// A workspace-wide team lesson is the team's copy of a local lesson with its key, as `learn show` marks it.
51+
const workspaceWide = new Set(lessons.filter((lesson) => lesson.repo_identity === null).map((lesson) => lesson.lesson_key))
4952
const hidden = new Set(localIds.filter((id) => {
5053
const key = identityKey(localIdentity(store, id, remote))
51-
return live.has(key) || tombstones.has(key)
54+
return live.has(key) || tombstones.has(key) || workspaceWide.has(id)
5255
}))
5356
// This repository's lessons claim their keys first, then workspace-wide ones, then other repositories'.
5457
const rank = (lesson: LessonOut) => lesson.repo_identity === remote.repo_identity ? 0 : lesson.repo_identity === null ? 1 : 2

‎packages/opencode/src/altimate/learn/ledger.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -211,7 +211,7 @@ export const writeSyncState = (root: string, store: string, state: SyncState) =>
211211
/** The ledger as it stands for `scope`; a different scope's ledger and backfill are discarded, never reused. */
212212
export function forScope(state: SyncState, scope: Scope): SyncState {
213213
if (state.scope === scopeKey(scope)) return state
214-
return { ...state, scope: scopeKey(scope), submissions_cursor: undefined, submissions_after_id: undefined, backfill: { complete: false, done: [] }, ledger: {}, unsyncable: {} }
214+
return { ...state, scope: scopeKey(scope), submissions_cursor: undefined, submissions_after_id: undefined, backfill: { complete: false, done: [] }, ledger: {}, unsyncable: {}, unsupported_until: undefined }
215215
}
216216

217217
/** The pulled cache, only when it was pulled for exactly this scope. */

‎packages/opencode/src/altimate/learn/proposals.ts‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,8 +27,12 @@ import {
2727
type BatchItem, type CoexistsRef, type Origin, type TargetRef, type UsageItem,
2828
} from "../workspace/lesson-api"
2929
import { assertLearnLock } from "./lock"
30+
import { redactSecrets } from "./digest"
31+
import { Log } from "@/util/log"
3032
import * as Store from "./store"
3133

34+
const log = Log.create({ service: "learn.proposals" })
35+
3236
/** A change to one lesson, before the ledger decides how to propose it. */
3337
export interface Intent {
3438
kind: "upsert" | "remove"
@@ -404,8 +408,16 @@ export function promotionHooks(root: string, store: string, scope: Scope, origin
404408
const supersededBy = new Map(retired.flatMap((lesson) => (lesson.supersededBy ? [[lesson.id, lesson.supersededBy] as const] : [])))
405409
intents = promoteIntents(store, remote, before, after, origin, supersededBy)
406410
},
411+
// Sync never fails a publish: the approved set is already replaced. A failed enqueue reopens the backfill, so
412+
// the next push derives these proposals again.
407413
afterPublish: async () => {
408-
if (intents.length) await enqueue(root, store, scope, intents)
414+
if (!intents.length) return
415+
await enqueue(root, store, scope, intents).catch(async (error) => {
416+
log.warn("learn sync proposals not queued after promote", { store, error: redactSecrets(error instanceof Error ? error.message : String(error)) })
417+
const state = forScope(await readSyncState(root, store), scope)
418+
state.backfill = { complete: false, done: [] }
419+
await writeSyncState(root, store, state)
420+
}).catch((error) => log.warn("learn sync backfill not reopened", { store, error: redactSecrets(error instanceof Error ? error.message : String(error)) }))
409421
},
410422
}
411423
}

‎packages/opencode/src/altimate/learn/sync.ts‎

Lines changed: 21 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,8 @@ export const BACKFILL_CAP = 200
3636
const LOCK_OPTIONS = { timeoutMs: 5_000 }
3737
/** Errors the server reports when a proposal's base moved: refresh, compare, and hold instead of resending. */
3838
const CONFLICTS = new Set(["version_conflict", "open_proposal_exists", "bad_target", "stale_target"])
39+
/** Statuses answering a request the server refused as invalid: request validation, wire limits, body size. */
40+
const REFUSED_REQUEST = new Set([400, 413, 422])
3941
/** Item errors that end a proposal: the server will never accept it as is. */
4042
const FINAL = new Set(["invalid", "no_change"])
4143

@@ -153,7 +155,8 @@ export function pull(root: string, store: string, ctx: Extract<Context, { status
153155
async function pullOnce(root: string, store: string, ctx: Extract<Context, { status: "ready" }>): Promise<PullOutcome> {
154156
const { scope, actAs } = ctx
155157
const before = await readSyncState(root, store)
156-
if (before.unsupported_until && Date.parse(before.unsupported_until) > Date.now())
158+
const backoff = forScope(before, scope).unsupported_until
159+
if (backoff && Date.parse(backoff) > Date.now())
157160
return { store, outcome: "unsupported", error: "the workspace server does not support lesson sync yet" }
158161
const cached = await remoteFor(root, store, scope)
159162
const started = Date.now()
@@ -168,7 +171,7 @@ async function pullOnce(root: string, store: string, ctx: Extract<Context, { sta
168171
local_keys: local.keys,
169172
})
170173
} catch (error) {
171-
return recordPullFailure(root, store, error, Date.now() - started)
174+
return recordPullFailure(root, store, scope, error, Date.now() - started)
172175
}
173176
// The caller's own proposals refresh the ledger's pending and receipt slots. Optional: a failure keeps the pull.
174177
const submissions = await ownSubmissions(actAs, scope.datamateId, forScope(before, scope)).catch((error) => {
@@ -195,7 +198,7 @@ async function pullOnce(root: string, store: string, ctx: Extract<Context, { sta
195198
await writeRemote(root, store, remote)
196199
const state = forScope(await readSyncState(root, store), scope)
197200
// No earlier cache: sync was off, or access was lost. Lessons changed meanwhile were never proposed, so the
198-
// backfill runs again (it skips every identity the ledger already knows).
201+
// backfill runs again (it skips a lesson only when the ledger already holds its exact content).
199202
if (!previous) state.backfill = { complete: false, done: [] }
200203
state.local_keys_offset = local.next
201204
applyPull(state, remote, !response.unchanged || !previous)
@@ -230,13 +233,14 @@ async function ownSubmissions(actAs: ActAs, datamateId: number, state: Pick<Sync
230233
return rows
231234
}
232235

233-
async function recordPullFailure(root: string, store: string, error: unknown, latency: number): Promise<PullOutcome> {
236+
async function recordPullFailure(root: string, store: string, scope: Scope, error: unknown, latency: number): Promise<PullOutcome> {
234237
const failure = error instanceof LessonApiError ? error : new LessonApiError("transient", errText(error))
235238
const message = redactSecrets(failure.message)
236239
log.warn("learn sync pull failed", { store, kind: failure.kind, latency_ms: latency, error: message })
237240
track({ operation: "pull", outcome: "error", error_kind: failure.kind, latency_ms: latency })
238241
await Store.transaction(root, async () => {
239-
const state = await readSyncState(root, store)
242+
// Recorded under the scope, so the backoff applies to this account and workspace only.
243+
const state = forScope(await readSyncState(root, store), scope)
240244
state.last_pull = { at: now(), outcome: failure.kind === "unsupported" ? "unsupported" : "error", error: message, latency_ms: latency }
241245
if (failure.kind === "unsupported") state.unsupported_until = new Date(Date.now() + UNSUPPORTED_BACKOFF_MS).toISOString()
242246
// Access answers invalidate the cache; transient failures keep it.
@@ -297,7 +301,8 @@ function satisfied(proposal: Proposal, store: string, remote: Remote | undefined
297301
if (entry?.pending?.submission_hash === proposal.submission_hash || entry?.pending?.content_hash === proposal.submission_hash) return true
298302
if (entry?.receipts.some((r) => r.submission_hash === proposal.submission_hash && r.status !== "rejected")) return true
299303
const live = remote?.lessons.find((l) => identityKey(remoteIdentity(l)) === identityKey(identity))
300-
if (proposal.item.change_type === "remove") return !live
304+
// Without a pulled cache nothing proves the lesson is gone.
305+
if (proposal.item.change_type === "remove") return !!remote && !live
301306
return !!live && live.text === proposal.item.text && live.pinned === proposal.item.pinned &&
302307
JSON.stringify(live.tags) === JSON.stringify(proposal.item.tags) && JSON.stringify(live.trigger_paths) === JSON.stringify(proposal.item.trigger_paths)
303308
}
@@ -327,8 +332,8 @@ export async function push(root: string, store: string, ctx: Extract<Context, {
327332
const report = emptyReport(store)
328333
const { scope, actAs } = ctx
329334
const live = () => opts.deadline === undefined || Date.now() < opts.deadline
330-
const sync = await readSyncState(root, store)
331-
if (sync.unsupported_until && Date.parse(sync.unsupported_until) > Date.now()) {
335+
const backoff = forScope(await readSyncState(root, store), scope).unsupported_until
336+
if (backoff && Date.parse(backoff) > Date.now()) {
332337
report.error = "the workspace server does not support lesson sync yet"
333338
return report
334339
}
@@ -356,7 +361,10 @@ export async function push(root: string, store: string, ctx: Extract<Context, {
356361
outbox.proposals = outbox.proposals.flatMap((p) => {
357362
if (p.state !== "held" || p.item.lesson_key !== opts.resubmit || !sameScope(p.scope, scope)) return [p]
358363
const next = rebase(p, store, remote, state)
359-
return next ? [next] : []
364+
if (next) return [next]
365+
report.dropped++
366+
log.info("learn sync held proposal dropped on resubmit: nothing left to propose", { store, key: p.item.lesson_key })
367+
return []
360368
})
361369
await writeOutbox(root, store, outbox)
362370
}
@@ -405,9 +413,10 @@ export async function push(root: string, store: string, ctx: Extract<Context, {
405413
}
406414
} catch (error) {
407415
const classified = error instanceof LessonApiError ? error : new LessonApiError("transient", errText(error))
408-
// A request the server refused (a 4xx answer, such as a wire limit) is never resent as it was: its items are
409-
// sent one by one to find the item it refused, and that item is held.
410-
if (classified.kind !== "rejected" || classified.status === undefined) failure = classified
416+
// A request the server refused as invalid (a wire limit answers 400 `VALIDATION_ERROR`) is never resent as it
417+
// was: its items are sent one by one to find the item it refused, and that item is held. Any other answer, such
418+
// as a 401 for an expired login, stops the push and leaves the queue as it is.
419+
if (classified.kind !== "rejected" || !REFUSED_REQUEST.has(classified.status ?? 0)) failure = classified
411420
else if (chunk.length > 1) pending.unshift(...chunk.map((proposal) => [proposal]))
412421
else refusedAlone.push({ proposal: chunk[0], message: redactSecrets(classified.message) })
413422
}

‎packages/opencode/src/cli/cmd/learn.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1188,7 +1188,7 @@ const SyncCommand = effectCmd({
11881188
"Lessons retired in review stay in that skill until the workspace owner detaches it in the workspace's Skills page.")
11891189
}
11901190
if (report.pulls.some((pull) => pull.outcome === "error" || pull.outcome === "unsupported")) throw new Error("Lesson sync did not complete; see above.")
1191-
if (report.pushes.some((push) => push.error)) throw new Error("Some proposals were not sent; they stay queued.")
1191+
if (report.pushes.some((push) => push.error)) throw new Error("Some proposals were not sent; see the push summary above.")
11921192
})
11931193
}),
11941194
})
@@ -1213,7 +1213,7 @@ const PushCommand = effectCmd({
12131213
if (report.context.status !== "ready")
12141214
throw new Error(`Lesson sync skipped: ${report.context.status === "skipped" ? report.context.reason : "sync is off"}.`)
12151215
out(`Push: ${describePushes(report.pushes)}`)
1216-
if (report.pushes.some((push) => push.error)) throw new Error("Some proposals were not sent; they stay queued.")
1216+
if (report.pushes.some((push) => push.error)) throw new Error("Some proposals were not sent; see the push summary above.")
12171217
})
12181218
}),
12191219
})

‎packages/opencode/test/altimate/learn/fixtures/README.md‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,3 +26,11 @@ const cases = (await Bun.file(file).json()).map(({ text }: { text: string }) =>
2626
await Bun.write(file, JSON.stringify(cases, null, 2) + "\n")
2727
'
2828
```
29+
30+
## `submission-hash-cases.json`
31+
32+
`/batch` items with the `submission_hash` altimate-backend computes for them (`material_hash` in
33+
`app/crud/learned_lessons.py`, over a validated `BatchLessonItem`). Push matches results on exact hashes, so
34+
`sync-effective.test.ts` checks that `submissionHash` here, and the test server's `materialHash`, produce the same
35+
values. The cases cover non-ASCII text and tags, escapes, trigger paths, pinned, coexists refs and replaces. After
36+
changing the hashed fields on either side, regenerate the file from the backend and keep the cases.
Lines changed: 137 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,137 @@
1+
{
2+
"_comment": "Generated from altimate-backend app.crud.learned_lessons.material_hash over BatchLessonItem; regenerate there when the hash changes.",
3+
"cases": [
4+
{
5+
"name": "plain ascii add",
6+
"item": {
7+
"tags": [],
8+
"trigger_paths": [],
9+
"coexists": [],
10+
"pinned": false,
11+
"replaces": [],
12+
"lesson_key": "L-0001",
13+
"change_type": "add",
14+
"text": "Use cents for money columns.",
15+
"origin": "reflection"
16+
},
17+
"submission_hash": "feb8a11b41e096a97aefc8ce5c83bff50a7f0dd38bb2cd3d36d0d919a68b3081"
18+
},
19+
{
20+
"name": "non-ASCII text and tags",
21+
"item": {
22+
"tags": [
23+
"finance",
24+
"données"
25+
],
26+
"trigger_paths": [],
27+
"coexists": [],
28+
"pinned": false,
29+
"replaces": [],
30+
"lesson_key": "L-0002",
31+
"change_type": "add",
32+
"text": "Montant en euros (€): arrondir à 2 décimales — 金额 🙂",
33+
"origin": "reflection"
34+
},
35+
"submission_hash": "67276a6d3b44c7cfcbbdf1e425e5bcdfba74b662347ada9ebca365cc44973230"
36+
},
37+
{
38+
"name": "trigger paths and pinned",
39+
"item": {
40+
"tags": [],
41+
"trigger_paths": [
42+
"models/staging/**",
43+
"dbt_project.yml"
44+
],
45+
"coexists": [],
46+
"pinned": true,
47+
"replaces": [],
48+
"lesson_key": "L-0003",
49+
"change_type": "add",
50+
"text": "Staging models stay views.",
51+
"origin": "person"
52+
},
53+
"submission_hash": "66fc827de93551eb24fe922b7f85229db13de464d27b22165be672ec473ec369"
54+
},
55+
{
56+
"name": "coexists refs",
57+
"item": {
58+
"tags": [],
59+
"trigger_paths": [],
60+
"coexists": [
61+
{
62+
"lesson_key": "L-0001",
63+
"source_text_hash": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
64+
"target_text_hash": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"
65+
},
66+
{
67+
"lesson_key": "L-0002",
68+
"source_text_hash": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",
69+
"target_text_hash": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
70+
}
71+
],
72+
"pinned": false,
73+
"replaces": [],
74+
"lesson_key": "L-0004",
75+
"change_type": "add",
76+
"text": "Prefer incremental for events.",
77+
"origin": "reflection"
78+
},
79+
"submission_hash": "6148d135d590185ed4eba478b9cf47359bd2ccd419a2cdd77402f3e81252b0c0"
80+
},
81+
{
82+
"name": "edit with replaces",
83+
"item": {
84+
"tags": [],
85+
"trigger_paths": [],
86+
"coexists": [],
87+
"pinned": false,
88+
"replaces": [
89+
{
90+
"public_id": "LL-abc123",
91+
"version": 3
92+
}
93+
],
94+
"lesson_key": "L-0001",
95+
"change_type": "edit",
96+
"text": "Use integer cents for every money column.",
97+
"origin": "person"
98+
},
99+
"submission_hash": "a3518de4f520dfb5a89298793ed2f6cbf9fd717196b0dd2c4202e9914bf30653"
100+
},
101+
{
102+
"name": "remove",
103+
"item": {
104+
"tags": [],
105+
"trigger_paths": [],
106+
"coexists": [],
107+
"pinned": false,
108+
"replaces": [
109+
{
110+
"public_id": "LL-def456",
111+
"version": 1
112+
}
113+
],
114+
"lesson_key": "L-0005",
115+
"change_type": "remove",
116+
"text": "Old rule to drop.",
117+
"origin": "person"
118+
},
119+
"submission_hash": "1c4a386930a3a7adac49578eda982fb413c282cce2a8034633145f55233a327f"
120+
},
121+
{
122+
"name": "escapes in text",
123+
"item": {
124+
"tags": [],
125+
"trigger_paths": [],
126+
"coexists": [],
127+
"pinned": false,
128+
"replaces": [],
129+
"lesson_key": "L-0006",
130+
"change_type": "add",
131+
"text": "Quote \"identifiers\" and use \\ in regex; tab\there",
132+
"origin": "reflection"
133+
},
134+
"submission_hash": "b2ed9d7754e1412b83e0c2590ae263a96cfe4bf5b6a5186645c8bfc93221341b"
135+
}
136+
]
137+
}

‎packages/opencode/test/altimate/learn/sync-effective.test.ts‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -267,6 +267,18 @@ describe("first-sync backfill", () => {
267267
})
268268
})
269269

270+
describe("submission hash fixture", () => {
271+
// Generated by the backend's `material_hash`. Results are matched on exact hashes, so a drift here would leave
272+
// every proposal unmatched and re-sent on each push.
273+
const file = path.join(import.meta.dir, "fixtures", "submission-hash-cases.json")
274+
const { cases }: { cases: { name: string; item: Parameters<typeof submissionHash>[0]; submission_hash: string }[] } = JSON.parse(fs.readFileSync(file, "utf8"))
275+
test.each(cases.map((c) => [c.name, c] as const))("%s matches the backend", (_name, c) => {
276+
expect(submissionHash(c.item)).toBe(c.submission_hash)
277+
// The in-memory server must hash the same way, or its tests would pass against a drifted client.
278+
expect(materialHash(c.item)).toBe(c.submission_hash)
279+
})
280+
})
281+
270282
describe("shared lint fixture", () => {
271283
// The backend validates submissions with a port of this lint; both repositories run the same cases.
272284
const file = path.join(import.meta.dir, "fixtures", "lint_cases.json")

‎packages/opencode/test/altimate/learn/sync-fixture.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -65,7 +65,7 @@ export interface Server {
6565
delayMs: number
6666
/** Answer every lesson route with `lessons_not_available` (a tenant whose migration has not run). */
6767
notMigrated: boolean
68-
/** Refuse a whole `/batch` request (422 with an array `detail`) that carries an item with this key. */
68+
/** Refuse a whole `/batch` request that carries an item with this key, as the backend answers a wire limit: 400 `VALIDATION_ERROR`. */
6969
refuseKey?: string
7070
/** Rewrite the next `/batch` results (drop, reorder, duplicate or blank them) to probe the client's matching. */
7171
mangleBatch?: (results: any[]) => any[]
@@ -226,7 +226,7 @@ export function startServer(opts: { datamateId?: number } = {}): Server {
226226
}
227227
// batch
228228
if (server.refuseKey && (body.items as any[]).some((item) => item.lesson_key === server.refuseKey))
229-
return json({ detail: [{ loc: ["body", "items"], msg: "too long", type: "value_error" }] }, 422)
229+
return json({ error_code: "VALIDATION_ERROR", detail: "items.0.text: String should have at most 280 characters" }, 400)
230230
const identity = bound(body.repo_remote)
231231
if (!identity) return coded(422, "repo_not_bound")
232232
const store = body.store ?? "team-playbook"

0 commit comments

Comments
 (0)