Skip to content

Commit 0e1ff3e

Browse files
committed
Expose compact transport and finish bounded embedding API v2
1 parent b35139d commit 0e1ff3e

31 files changed

Lines changed: 2415 additions & 88 deletions

‎CHANGELOG.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,11 @@ format: `Unreleased` then release buckets, grouped by impact.
88

99
## Unreleased
1010

11+
- Expose compact projections through CLI replay and retained service/MCP routes,
12+
with explicit format/profile negotiation and offline model-token measurement
13+
tooling (#201). Introduce Rust embedding API v2 for mandatory analysis/execution input caps, add
14+
cooperative request cancellation, and verify opaque session release (#203).
15+
1116
- Add authoritative task-context, Project API, candidate-diff, and Agent graph
1217
compact profiles with regeneration-bound replay (#201). Extend Rust embedding
1318
negotiation, cancellation, and the external consumer (#203). Connect the
Lines changed: 241 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,241 @@
1+
{
2+
"cases": [
3+
{
4+
"case": "examples/banking_ledger.spx",
5+
"full": {
6+
"bytes": 139770,
7+
"sha256": "sha256:ed56d26ae5cedd7ee8d019646eafe023909d63020bf3d5c4b0077076dd834f14"
8+
},
9+
"projections": [
10+
{
11+
"compact": {
12+
"bytes": 88586,
13+
"sha256": "sha256:d73cb49cc4a44286505b65b74459841bba718861a0aea11d4f71a665084b8ca8"
14+
},
15+
"encoding": "text",
16+
"replay": {
17+
"bytes": 139770,
18+
"sha256": "sha256:ed56d26ae5cedd7ee8d019646eafe023909d63020bf3d5c4b0077076dd834f14"
19+
}
20+
},
21+
{
22+
"compact": {
23+
"bytes": 139129,
24+
"sha256": "sha256:9ece9aa329a4595d4181ab6b81b16f42214d5c06cfb3f8bbd7f15b3397d9480c"
25+
},
26+
"encoding": "binary",
27+
"replay": {
28+
"bytes": 139770,
29+
"sha256": "sha256:ed56d26ae5cedd7ee8d019646eafe023909d63020bf3d5c4b0077076dd834f14"
30+
}
31+
}
32+
],
33+
"tokenizers": {
34+
"cl100k_base": {
35+
"encoding": "cl100k_base",
36+
"token_ratios": {
37+
"compact_text": 1.2360883580827564
38+
},
39+
"tokens": {
40+
"compact_text": 46781,
41+
"full": 37846
42+
},
43+
"version": "0.12.0",
44+
"vocab_fingerprint": "sha256:a86973639e26516c66f95cab5ff8b9f16ccdf898968ea6bcd89d3e54f2042f3f",
45+
"vocab_size": 100277
46+
},
47+
"o200k_base": {
48+
"encoding": "o200k_base",
49+
"token_ratios": {
50+
"compact_text": 1.215193790453596
51+
},
52+
"tokens": {
53+
"compact_text": 47124,
54+
"full": 38779
55+
},
56+
"version": "0.12.0",
57+
"vocab_fingerprint": "sha256:4b1a9844f30e3055f22aa578137518c8529991e55c12c8d7e25a81010617ccab",
58+
"vocab_size": 200019
59+
}
60+
}
61+
},
62+
{
63+
"case": "examples/http_app_routing.spx",
64+
"full": {
65+
"bytes": 610864,
66+
"sha256": "sha256:5bed00626ae3547960aece42fbd2ee579d1903c2f80f7bbe4212ae31684380f5"
67+
},
68+
"projections": [
69+
{
70+
"compact": {
71+
"bytes": 399030,
72+
"sha256": "sha256:e606d2697afff2d3290e6ed7384fcb31a99192ce3eb2fa213516fd2dae6c2e75"
73+
},
74+
"encoding": "text",
75+
"replay": {
76+
"bytes": 610864,
77+
"sha256": "sha256:5bed00626ae3547960aece42fbd2ee579d1903c2f80f7bbe4212ae31684380f5"
78+
}
79+
},
80+
{
81+
"compact": {
82+
"bytes": 566677,
83+
"sha256": "sha256:ae43f83bfe2af6c34d784c17192edd7b3318e60b7702f32accc1aeb0f9cb07d1"
84+
},
85+
"encoding": "binary",
86+
"replay": {
87+
"bytes": 610864,
88+
"sha256": "sha256:5bed00626ae3547960aece42fbd2ee579d1903c2f80f7bbe4212ae31684380f5"
89+
}
90+
}
91+
],
92+
"tokenizers": {
93+
"cl100k_base": {
94+
"encoding": "cl100k_base",
95+
"token_ratios": {
96+
"compact_text": 1.3622243777067977
97+
},
98+
"tokens": {
99+
"compact_text": 220491,
100+
"full": 161861
101+
},
102+
"version": "0.12.0",
103+
"vocab_fingerprint": "sha256:a86973639e26516c66f95cab5ff8b9f16ccdf898968ea6bcd89d3e54f2042f3f",
104+
"vocab_size": 100277
105+
},
106+
"o200k_base": {
107+
"encoding": "o200k_base",
108+
"token_ratios": {
109+
"compact_text": 1.3401460118914408
110+
},
111+
"tokens": {
112+
"compact_text": 221565,
113+
"full": 165329
114+
},
115+
"version": "0.12.0",
116+
"vocab_fingerprint": "sha256:4b1a9844f30e3055f22aa578137518c8529991e55c12c8d7e25a81010617ccab",
117+
"vocab_size": 200019
118+
}
119+
}
120+
},
121+
{
122+
"case": "examples/calculator-project/semaprax.toml",
123+
"full": {
124+
"bytes": 9893,
125+
"sha256": "sha256:42b4e8300a4ffa0f42bca996d20210c3aca0f2899345b3ef948adc248e78069e"
126+
},
127+
"projections": [
128+
{
129+
"compact": {
130+
"bytes": 7474,
131+
"sha256": "sha256:264292caa0f0e303042d61e088f15112e32a22517b3a598fc3d1c72bf3cf7539"
132+
},
133+
"encoding": "text",
134+
"replay": {
135+
"bytes": 9893,
136+
"sha256": "sha256:42b4e8300a4ffa0f42bca996d20210c3aca0f2899345b3ef948adc248e78069e"
137+
}
138+
},
139+
{
140+
"compact": {
141+
"bytes": 11513,
142+
"sha256": "sha256:c81e157b1d70a48996d5d4eca89536de23433ee9902f63145ae7ddf695b1c3eb"
143+
},
144+
"encoding": "binary",
145+
"replay": {
146+
"bytes": 9893,
147+
"sha256": "sha256:42b4e8300a4ffa0f42bca996d20210c3aca0f2899345b3ef948adc248e78069e"
148+
}
149+
}
150+
],
151+
"tokenizers": {
152+
"cl100k_base": {
153+
"encoding": "cl100k_base",
154+
"token_ratios": {
155+
"compact_text": 1.4424
156+
},
157+
"tokens": {
158+
"compact_text": 3606,
159+
"full": 2500
160+
},
161+
"version": "0.12.0",
162+
"vocab_fingerprint": "sha256:a86973639e26516c66f95cab5ff8b9f16ccdf898968ea6bcd89d3e54f2042f3f",
163+
"vocab_size": 100277
164+
},
165+
"o200k_base": {
166+
"encoding": "o200k_base",
167+
"token_ratios": {
168+
"compact_text": 1.4383181277270924
169+
},
170+
"tokens": {
171+
"compact_text": 3626,
172+
"full": 2521
173+
},
174+
"version": "0.12.0",
175+
"vocab_fingerprint": "sha256:4b1a9844f30e3055f22aa578137518c8529991e55c12c8d7e25a81010617ccab",
176+
"vocab_size": 200019
177+
}
178+
}
179+
},
180+
{
181+
"case": "examples/frame-payload-project/semaprax.toml",
182+
"full": {
183+
"bytes": 6615,
184+
"sha256": "sha256:9399cfebe191b9a3182c2b73dc7beb4d6ea2ced87f0ab37000d6b2f6bb9d0926"
185+
},
186+
"projections": [
187+
{
188+
"compact": {
189+
"bytes": 5471,
190+
"sha256": "sha256:a2adb287f00de3636cd9e640c57eb7a7afca49140af6a8a3fa72d4d60ded45ad"
191+
},
192+
"encoding": "text",
193+
"replay": {
194+
"bytes": 6615,
195+
"sha256": "sha256:9399cfebe191b9a3182c2b73dc7beb4d6ea2ced87f0ab37000d6b2f6bb9d0926"
196+
}
197+
},
198+
{
199+
"compact": {
200+
"bytes": 8203,
201+
"sha256": "sha256:4f507b19e0ab6ce8efe4a5196a8c2e700bd6e5957233b8e5a19b448e02aca1fd"
202+
},
203+
"encoding": "binary",
204+
"replay": {
205+
"bytes": 6615,
206+
"sha256": "sha256:9399cfebe191b9a3182c2b73dc7beb4d6ea2ced87f0ab37000d6b2f6bb9d0926"
207+
}
208+
}
209+
],
210+
"tokenizers": {
211+
"cl100k_base": {
212+
"encoding": "cl100k_base",
213+
"token_ratios": {
214+
"compact_text": 1.5169875424688561
215+
},
216+
"tokens": {
217+
"compact_text": 2679,
218+
"full": 1766
219+
},
220+
"version": "0.12.0",
221+
"vocab_fingerprint": "sha256:a86973639e26516c66f95cab5ff8b9f16ccdf898968ea6bcd89d3e54f2042f3f",
222+
"vocab_size": 100277
223+
},
224+
"o200k_base": {
225+
"encoding": "o200k_base",
226+
"token_ratios": {
227+
"compact_text": 1.5100896860986548
228+
},
229+
"tokens": {
230+
"compact_text": 2694,
231+
"full": 1784
232+
},
233+
"version": "0.12.0",
234+
"vocab_fingerprint": "sha256:4b1a9844f30e3055f22aa578137518c8529991e55c12c8d7e25a81010617ccab",
235+
"vocab_size": 200019
236+
}
237+
}
238+
}
239+
],
240+
"schema": "semaprax.compact-projection-benchmark.v1"
241+
}

‎docs/ARCHITECTURE.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -140,6 +140,11 @@ Proposal schema. It does not advertise durable checkpoint policy.
140140
`compact_semantic_projection/selected` wraps existing task, public API, candidate,
141141
and Agent graph producers and independently regenerates selected content on
142142
replay; encoded evidence cannot replace the owning compiler object.
143+
`semantic_service_transport/compact` selects from exact retained service
144+
snapshots and feeds the same codec; MCP forwards to that dispatcher.
145+
`cli/compact` owns explicit host input/replay paths and writes only stdout.
146+
`embedding_api/analysis_request` owns source/label/symbol preflight and bounded
147+
refusal echoes for stateless operations, with shared hard caps for execution.
143148
`embedding_api/project_session` exposes caller-owned source admission and the
144149
existing semantic service through opaque handles and canonical reports. It
145150
retains the service's staged refresh and replay boundaries; caught stateful

‎docs/COMPACT-SEMANTIC-PROJECTION-V1.md‎

Lines changed: 55 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -95,15 +95,30 @@ revision, retained Project revision, candidate revision plus expected candidate
9595
digest, and AgentGraph digest; they do not accept a self-rehashed envelope as
9696
replay evidence.
9797

98-
## Deliberately out of scope
98+
## Negotiation and transport
9999

100-
Issue #201 additionally asks for byte/token benchmarks across multiple *model*
101-
tokenizer versions, version *negotiation* (as opposed to version *refusal*,
102-
which this module does implement), and CLI/MCP exposure. Those are not in this
103-
module. The selected profiles reuse existing authoritative producers and the
104-
unchanged bounded dictionary/text/binary codec. Selected replay adds exact
105-
regeneration equality (`SPX-Z910` on disagreement) to envelope integrity and
106-
profile/root/revision checks. It grants no candidate commit or runtime authority.
100+
`compact_semantic_projection::negotiation` selects only existing v1 text or
101+
binary encodings and the six implemented profile names. Both parties provide
102+
at most 16 explicit offers; an unknown version or profile refuses
103+
the exchange rather than falling back. The encoding is a closed Rust enum. Selection is deterministic across
104+
offer ordering: it chooses the lexicographically earliest common profile,
105+
then binary before text. Negotiation changes no codec or wire format.
106+
107+
The CLI and retained semantic service expose these projections through the
108+
existing compiler producers. `compact graph <project>` also wraps the full
109+
retained Project graph with profile `full-graph`, root `*`, and Project revision;
110+
its reconstructed graph schema distinguishes it from a single-source graph.
111+
`semaprax compact ... --replay <wire>` reconstructs only after comparison with
112+
freshly selected content. MCP `workspace__compact_projection` uses the same
113+
`workspace/compact-projection` dispatcher and requires the exact active
114+
workspace revision. Caller source labels select retained Project sources;
115+
they cannot open host paths. See [Unified CLI v1](UNIFIED-CLI-V1.md) and
116+
[Semantic Service Transport v1](PERSISTENT-SEMANTIC-SERVICE-TRANSPORT-V1.md).
117+
118+
The library exposes explicit format/encoding/profile negotiation, and the
119+
host-side `scripts/benchmark_compact_projection.py` measures actual tokenizer
120+
counts using cached, versioned encodings. Measurement evidence and its
121+
limitations are recorded below; byte compaction alone is not a token claim.
107122

108123
## Honesty bar: what "lossless" means here, exactly
109124

@@ -123,9 +138,8 @@ dropped, summarized, or approximated.
123138
This module claims a materially smaller wire size **only where measured**
124139
below, not as a universal property, and it does not claim:
125140

126-
- a real model tokenizer's token count (only wire bytes are measured here;
127-
`crate::semantic_task_context` owns this repository's honestly-labeled
128-
token-accounting units);
141+
- a universal token reduction or provider billing equivalence; the host-side
142+
measurements below name exact encoding versions and vocabulary fingerprints;
129143
- that every profile or content shape compacts (a selected view with few
130144
repeated strings could compact only slightly, or -- for the binary
131145
form specifically -- not at all, see the measurements below);
@@ -224,3 +238,33 @@ network.
224238
`crate::graph::to_json`/`agent_context_v2_json`. [`decode_text`] and
225239
[`decode_binary`] take only byte/string slices. Nothing in this module
226240
opens a file, spawns a process, or contacts a network.
241+
242+
## Offline model-token measurements (2026-09-13)
243+
244+
The reproducible helper runs the actual CLI encoder and replay for both wire
245+
forms, compares reconstructed bytes with the ordinary graph command (allowing
246+
its one display newline), then measures UTF-8 text with cached `tiktoken 0.12.0`
247+
encodings. Socket connections are disabled while tokenizer assets load. The
248+
[complete local report](../benchmarks/compact-semantic-projection-v1/local-token-measurements.json)
249+
records exact content hashes and tokenizer vocabulary/pattern fingerprints.
250+
Binary bytes are measured as transport bytes, never decoded as model text.
251+
252+
| Input | Full bytes | Text bytes | Binary bytes | cl100k full → text tokens | o200k full → text tokens |
253+
| --- | ---: | ---: | ---: | ---: | ---: |
254+
| `examples/banking_ledger.spx` | 139,770 | 88,586 | 139,129 | 37,846 → 46,781 | 38,779 → 47,124 |
255+
| `examples/http_app_routing.spx` | 610,864 | 399,030 | 566,677 | 161,861 → 220,491 | 165,329 → 221,565 |
256+
| `examples/calculator-project/semaprax.toml` | 9,893 | 7,474 | 11,513 | 2,500 → 3,606 | 2,521 → 3,626 |
257+
| `examples/frame-payload-project/semaprax.toml` | 6,615 | 5,471 | 8,203 | 1,766 → 2,679 | 1,784 → 2,694 |
258+
259+
Text saves bytes on all four examples but **increases model tokens by 21–52%**.
260+
Binary is larger than ordinary JSON on both measured Projects. These results
261+
are a negative control against assuming byte savings imply token savings.
262+
Issue #201 remains open for a measured token-efficient representation; the
263+
current text form is a byte-compaction option, not a recommended model-token
264+
optimization. Neither measurement implies provider billing or model quality.
265+
266+
Reproduce with an already installed `tiktoken` and cached encoding assets:
267+
268+
```sh
269+
python3 scripts/benchmark_compact_projection.py --semaprax target/debug/semaprax --root . --output compact-measurements.json
270+
```

0 commit comments

Comments
 (0)