|
| 1 | +// @effect-diagnostics nodeBuiltinImport:off |
| 2 | +import * as NodeFS from "node:fs"; |
| 3 | + |
| 4 | +import * as NodeServices from "@effect/platform-node/NodeServices"; |
| 5 | +import { assert, describe, it } from "@effect/vitest"; |
| 6 | +import { |
| 7 | + MessageId, |
| 8 | + type ModelSelection, |
| 9 | + NodeId, |
| 10 | + ProjectId, |
| 11 | + ProviderInstanceId, |
| 12 | + ProviderSessionId, |
| 13 | + RunAttemptId, |
| 14 | + RunId, |
| 15 | + ThreadId, |
| 16 | +} from "@t3tools/contracts"; |
| 17 | +import * as HostProcess from "@t3tools/shared/HostProcess"; |
| 18 | +import { resolveSelfInvocation } from "@t3tools/shared/nodeRuntime"; |
| 19 | +import * as SecureRandom from "effect/crypto/SecureRandom"; |
| 20 | +import * as DateTime from "effect/DateTime"; |
| 21 | +import * as Effect from "effect/Effect"; |
| 22 | +import * as FileSystem from "effect/FileSystem"; |
| 23 | +import * as Layer from "effect/Layer"; |
| 24 | +import * as Path from "effect/Path"; |
| 25 | +import * as Schema from "effect/Schema"; |
| 26 | +import * as Stream from "effect/Stream"; |
| 27 | + |
| 28 | +import * as AcpSessionRuntime from "@t3tools/provider-acp/server/AcpSessionRuntime"; |
| 29 | +import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator"; |
| 30 | +import * as McpProviderSessions from "@t3tools/provider-core/server/McpProviderSessions"; |
| 31 | +import * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter"; |
| 32 | +import * as TestProviderHost from "@t3tools/provider-testing/TestProviderHost"; |
| 33 | +import { GrokSettings } from "../settings.ts"; |
| 34 | +import { makeGrokAdapterV2 } from "./adapter.ts"; |
| 35 | + |
| 36 | +const layerTest = Layer.mergeAll( |
| 37 | + NodeServices.layer, |
| 38 | + IdAllocator.layer, |
| 39 | + McpProviderSessions.layer, |
| 40 | + SecureRandom.layerWebCrypto, |
| 41 | + TestProviderHost.layer().pipe(Layer.provide(NodeServices.layer)), |
| 42 | +); |
| 43 | + |
| 44 | +const DEFAULT_GROK_SETTINGS = Schema.decodeSync(GrokSettings)({}); |
| 45 | +const instanceId = ProviderInstanceId.make("grok"); |
| 46 | +const select = (model: string, reasoningEffort?: string): ModelSelection => ({ |
| 47 | + instanceId, |
| 48 | + model, |
| 49 | + ...(reasoningEffort === undefined |
| 50 | + ? {} |
| 51 | + : { options: [{ id: "reasoningEffort", value: reasoningEffort }] }), |
| 52 | +}); |
| 53 | + |
| 54 | +const ModelRequest = Schema.Struct({ |
| 55 | + method: Schema.String, |
| 56 | + params: Schema.Struct({ |
| 57 | + modelId: Schema.optional(Schema.String), |
| 58 | + configId: Schema.optional(Schema.String), |
| 59 | + value: Schema.optional(Schema.String), |
| 60 | + _meta: Schema.optional(Schema.Struct({ reasoningEffort: Schema.optional(Schema.String) })), |
| 61 | + }), |
| 62 | +}); |
| 63 | +const decodeModelRequest = Schema.decodeUnknownSync(Schema.fromJsonString(ModelRequest)); |
| 64 | + |
| 65 | +/** The model and effort writes Grok received, in wire order. */ |
| 66 | +function modelWrites(requestLogPath: string): ReadonlyArray<string> { |
| 67 | + return NodeFS.readFileSync(requestLogPath, "utf8") |
| 68 | + .split("\n") |
| 69 | + .filter( |
| 70 | + (line) => |
| 71 | + line.includes('"session/set_model"') || line.includes('"session/set_config_option"'), |
| 72 | + ) |
| 73 | + .map((line) => { |
| 74 | + const { method, params } = decodeModelRequest(line); |
| 75 | + return method === "session/set_model" |
| 76 | + ? `set_model ${params.modelId}${params._meta?.reasoningEffort ? ` effort=${params._meta.reasoningEffort}` : ""}` |
| 77 | + : `set_config_option ${params.configId}=${params.value}`; |
| 78 | + }); |
| 79 | +} |
| 80 | + |
| 81 | +/** |
| 82 | + * Opens a Grok session on the first selection, then runs one turn per later |
| 83 | + * selection on the same live session. Returns the model and effort writes each |
| 84 | + * step sent. |
| 85 | + */ |
| 86 | +const writesPerSelection = ( |
| 87 | + selections: ReadonlyArray<ModelSelection>, |
| 88 | + agentEnvironment: Readonly<Record<string, string>>, |
| 89 | +) => |
| 90 | + Effect.gen(function* () { |
| 91 | + const fileSystem = yield* FileSystem.FileSystem; |
| 92 | + const path = yield* Path.Path; |
| 93 | + const directory = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3-grok-effort-" }); |
| 94 | + const requestLogPath = path.join(directory, "requests.ndjson"); |
| 95 | + yield* fileSystem.writeFileString(requestLogPath, ""); |
| 96 | + const agentPath = yield* path.fromFileUrl( |
| 97 | + new URL("./testFixtures/grok-model-selection-agent.mjs", import.meta.url), |
| 98 | + ); |
| 99 | + const adapter = yield* makeGrokAdapterV2({ |
| 100 | + instanceId, |
| 101 | + settings: DEFAULT_GROK_SETTINGS, |
| 102 | + environment: {}, |
| 103 | + hostPlatform: yield* HostProcess.Platform, |
| 104 | + selfInvocation: yield* resolveSelfInvocation(), |
| 105 | + makeRuntime: (runtimeInput) => |
| 106 | + Effect.gen(function* () { |
| 107 | + const context = yield* Layer.build( |
| 108 | + AcpSessionRuntime.layer({ |
| 109 | + ...runtimeInput, |
| 110 | + spawn: { |
| 111 | + command: process.execPath, |
| 112 | + args: [agentPath], |
| 113 | + cwd: runtimeInput.cwd, |
| 114 | + env: { ...agentEnvironment, T3_ACP_REQUEST_LOG_PATH: requestLogPath }, |
| 115 | + }, |
| 116 | + authMethodId: "cached_token", |
| 117 | + }), |
| 118 | + ); |
| 119 | + return yield* Effect.service(AcpSessionRuntime.AcpSessionRuntime).pipe( |
| 120 | + Effect.provide(context), |
| 121 | + ); |
| 122 | + }), |
| 123 | + }); |
| 124 | + const threadId = ThreadId.make("grok-effort-thread"); |
| 125 | + const runtimePolicy = ProviderAdapter.ProviderAdapterV2RuntimePolicy.make({ |
| 126 | + runtimeMode: "full-access", |
| 127 | + interactionMode: "default", |
| 128 | + cwd: directory, |
| 129 | + }); |
| 130 | + const [first, ...rest] = selections; |
| 131 | + if (first === undefined) return []; |
| 132 | + const writes: Array<ReadonlyArray<string>> = []; |
| 133 | + const step = <A, E, R>(effect: Effect.Effect<A, E, R>) => |
| 134 | + Effect.gen(function* () { |
| 135 | + const before = modelWrites(requestLogPath).length; |
| 136 | + const result = yield* effect; |
| 137 | + writes.push(modelWrites(requestLogPath).slice(before)); |
| 138 | + return result; |
| 139 | + }); |
| 140 | + const runtime = yield* step( |
| 141 | + adapter.openSession({ |
| 142 | + threadId, |
| 143 | + providerSessionId: ProviderSessionId.make("grok-effort-session"), |
| 144 | + modelSelection: first, |
| 145 | + runtimePolicy, |
| 146 | + }), |
| 147 | + ); |
| 148 | + const providerThread = yield* runtime.ensureThread({ |
| 149 | + threadId, |
| 150 | + modelSelection: first, |
| 151 | + runtimePolicy, |
| 152 | + }); |
| 153 | + const now = yield* DateTime.now; |
| 154 | + for (const [index, modelSelection] of rest.entries()) { |
| 155 | + const ordinal = index + 1; |
| 156 | + const suffix = `${threadId}:${ordinal}`; |
| 157 | + yield* step( |
| 158 | + runtime.startTurn({ |
| 159 | + appThread: { |
| 160 | + createdBy: "user", |
| 161 | + creationSource: "web", |
| 162 | + id: threadId, |
| 163 | + projectId: ProjectId.make("grok-effort-project"), |
| 164 | + title: "Grok effort", |
| 165 | + providerInstanceId: instanceId, |
| 166 | + modelSelection, |
| 167 | + runtimeMode: "full-access", |
| 168 | + interactionMode: "default", |
| 169 | + branch: null, |
| 170 | + worktreePath: null, |
| 171 | + activeProviderThreadId: providerThread.id, |
| 172 | + lineage: { parentThreadId: null, relationshipToParent: null, rootThreadId: threadId }, |
| 173 | + forkedFrom: null, |
| 174 | + createdAt: now, |
| 175 | + updatedAt: now, |
| 176 | + archivedAt: null, |
| 177 | + settledOverride: null, |
| 178 | + settledAt: null, |
| 179 | + lastVisitedAt: null, |
| 180 | + deletedAt: null, |
| 181 | + }, |
| 182 | + threadId, |
| 183 | + runId: RunId.make(`run:${suffix}`), |
| 184 | + runOrdinal: ordinal, |
| 185 | + providerTurnOrdinal: ordinal, |
| 186 | + attemptId: RunAttemptId.make(`attempt:${suffix}`), |
| 187 | + rootNodeId: NodeId.make(`node:${suffix}`), |
| 188 | + providerThread, |
| 189 | + message: { |
| 190 | + createdBy: "user", |
| 191 | + creationSource: "web", |
| 192 | + messageId: MessageId.make(`message:${suffix}`), |
| 193 | + text: "hello", |
| 194 | + attachments: [], |
| 195 | + }, |
| 196 | + modelSelection, |
| 197 | + runtimePolicy, |
| 198 | + }), |
| 199 | + ); |
| 200 | + yield* runtime.events.pipe( |
| 201 | + Stream.filter((event) => event.type === "turn.terminal"), |
| 202 | + Stream.runHead, |
| 203 | + ); |
| 204 | + } |
| 205 | + return writes; |
| 206 | + }).pipe(Effect.scoped, Effect.provide(layerTest)); |
| 207 | + |
| 208 | +describe("Grok reasoning effort", () => { |
| 209 | + // Grok's session starts at xhigh on grok-4.6. grok-4.6-fast offers only |
| 210 | + // high and low, and resets to high when a switch carries no effort. |
| 211 | + const selections = [ |
| 212 | + select("grok-4.6", "low"), |
| 213 | + select("grok-4.6", "high"), |
| 214 | + select("grok-4.6"), |
| 215 | + select("grok-4.6", "high"), |
| 216 | + select("grok-4.6-fast", "xhigh"), |
| 217 | + select("grok-4.6", "low"), |
| 218 | + ]; |
| 219 | + |
| 220 | + it.live("protocol 1 carries the selected effort on session/set_model", () => |
| 221 | + Effect.gen(function* () { |
| 222 | + assert.deepEqual(yield* writesPerSelection(selections, { T3_GROK_MOCK_PROTOCOL: "1" }), [ |
| 223 | + ["set_model grok-4.6 effort=low"], |
| 224 | + ["set_model grok-4.6 effort=high"], |
| 225 | + [], |
| 226 | + ["set_model grok-4.6 effort=high"], |
| 227 | + ["set_model grok-4.6-fast"], |
| 228 | + ["set_model grok-4.6 effort=low"], |
| 229 | + ]); |
| 230 | + }), |
| 231 | + ); |
| 232 | + |
| 233 | + it.live("newer protocols write the session's own reasoning_effort option", () => |
| 234 | + Effect.gen(function* () { |
| 235 | + assert.deepEqual(yield* writesPerSelection(selections, { T3_GROK_MOCK_PROTOCOL: "2" }), [ |
| 236 | + ["set_config_option reasoning_effort=low"], |
| 237 | + ["set_config_option reasoning_effort=high"], |
| 238 | + [], |
| 239 | + [], |
| 240 | + ["set_config_option model=grok-4.6-fast"], |
| 241 | + ["set_config_option model=grok-4.6", "set_config_option reasoning_effort=low"], |
| 242 | + ]); |
| 243 | + }), |
| 244 | + ); |
| 245 | + |
| 246 | + it.live("an effort Grok refuses keeps the model switch and opens the session", () => |
| 247 | + Effect.gen(function* () { |
| 248 | + const refused = [select("grok-4.6-fast", "low")]; |
| 249 | + assert.deepEqual( |
| 250 | + yield* writesPerSelection(refused, { |
| 251 | + T3_GROK_MOCK_PROTOCOL: "1", |
| 252 | + T3_GROK_MOCK_REJECT_EFFORT: "low", |
| 253 | + }), |
| 254 | + [["set_model grok-4.6-fast effort=low", "set_model grok-4.6-fast"]], |
| 255 | + ); |
| 256 | + assert.deepEqual( |
| 257 | + yield* writesPerSelection(refused, { |
| 258 | + T3_GROK_MOCK_PROTOCOL: "2", |
| 259 | + T3_GROK_MOCK_REJECT_EFFORT: "low", |
| 260 | + }), |
| 261 | + [["set_config_option model=grok-4.6-fast", "set_config_option reasoning_effort=low"]], |
| 262 | + ); |
| 263 | + }), |
| 264 | + ); |
| 265 | +}); |
0 commit comments