Skip to content

Commit 5dc36b6

Browse files
fix(evaluations): audit provider routing and refresh API model discovery
Merge current main, preserve Codex routing overrides in isolated judges, and keep API model discovery failures retryable on mobile. Add focused routing and independent-discovery regressions.
2 parents b905c49 + d57f643 commit 5dc36b6

44 files changed

Lines changed: 214 additions & 2068 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎apps/mobile/src/features/settings/ModelEvaluationSettings.tsx‎

Lines changed: 30 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import { useAtomValue } from "@effect/atom-react";
1+
import { useAtomRefresh, useAtomValue } from "@effect/atom-react";
22
import type { MenuAction } from "@react-native-menu/menu";
33
import {
44
isManagedSettingPath,
@@ -16,10 +16,9 @@ import {
1616
modelEvaluationSelectionLabel,
1717
isModelEvaluationJudge,
1818
} from "@t3tools/client-runtime/modelEvaluations";
19-
import { runAtomCommand, squashAtomCommandFailure } from "@t3tools/client-runtime/state/runtime";
20-
import * as Effect from "effect/Effect";
19+
import { squashAtomCommandFailure } from "@t3tools/client-runtime/state/runtime";
2120
import * as Hash from "effect/Hash";
22-
import { AsyncResult, Atom } from "effect/reactivity";
21+
import { AsyncResult } from "effect/reactivity";
2322
import { useState } from "react";
2423
import { View } from "react-native";
2524

@@ -41,6 +40,7 @@ import { serverEnvironment } from "../../state/server";
4140
import { useAtomCommand } from "../../state/use-atom-command";
4241
import { SettingsSection } from "./components/SettingsSection";
4342
import { SettingsSwitchRow } from "./components/SettingsSwitchRow";
43+
import { apiJudgeModelsAtom } from "./modelEvaluationApiModels";
4444
import { useSettingsEnvironmentFilter } from "./settings-environment-filter";
4545

4646
export function ModelEvaluationSettings() {
@@ -77,27 +77,6 @@ export function ModelEvaluationSettings() {
7777
);
7878
}
7979

80-
const apiJudgeModelsAtom = Atom.family((key: string) => {
81-
const apis = JSON.parse(key) as ReadonlyArray<
82-
readonly [EnvironmentId, ProviderInstanceId, number]
83-
>;
84-
return Atom.make((get) =>
85-
Effect.forEach(
86-
apis,
87-
([environmentId, id]) =>
88-
Effect.promise(() =>
89-
runAtomCommand(
90-
get.registry,
91-
serverEnvironment.listTextGeneratorModels,
92-
{ environmentId, input: { id } },
93-
{ reportFailure: false },
94-
),
95-
).pipe(Effect.map((result) => (result._tag === "Success" ? result.value.models : []))),
96-
{ concurrency: "unbounded" },
97-
),
98-
).pipe(Atom.setIdleTTL(5 * 60_000), Atom.withLabel("model-evaluation-api-judges"));
99-
});
100-
10180
function EnvironmentEvaluationSettings(props: {
10281
environmentId: EnvironmentId;
10382
label: string;
@@ -121,15 +100,15 @@ function EnvironmentEvaluationSettings(props: {
121100
>)
122101
: []
123102
).filter(([, generator]) => generator.enabled);
124-
const apiModelLists = useAtomValue(
125-
apiJudgeModelsAtom(
126-
JSON.stringify(
127-
apiJudges.map(
128-
([id, generator]) => [environmentId, id, Hash.string(JSON.stringify(generator))] as const,
129-
),
103+
const apiModelsAtom = apiJudgeModelsAtom(
104+
JSON.stringify(
105+
apiJudges.map(
106+
([id, generator]) => [environmentId, id, Hash.string(JSON.stringify(generator))] as const,
130107
),
131108
),
132109
);
110+
const apiModelLists = useAtomValue(apiModelsAtom);
111+
const refreshApiModels = useAtomRefresh(apiModelsAtom);
133112
if (config === null) return null;
134113
const readOnly = saving || !canWrite || props.projectSelected;
135114
const project =
@@ -141,7 +120,11 @@ function EnvironmentEvaluationSettings(props: {
141120
const capable = new Set(
142121
config.providers.filter(isModelEvaluationJudge).map((provider) => provider.instanceId),
143122
);
144-
const listedApiModels = AsyncResult.getOrElse(apiModelLists, () => []);
123+
const listedApiModels = apiModelLists.map((result) => AsyncResult.getOrElse(result, () => []));
124+
const apiModelsLoading = apiModelLists.some((result) => result.waiting);
125+
const failedApis = apiJudges.flatMap(([, generator], index) =>
126+
apiModelLists[index]?._tag === "Failure" ? [generator.label || generator.baseUrl] : [],
127+
);
145128
const models = [
146129
...buildModelOptions(config, selection, undefined, null).filter(
147130
(model) =>
@@ -230,6 +213,21 @@ function EnvironmentEvaluationSettings(props: {
230213
if (model) void write({ modelEvaluationModelSelection: model.selection });
231214
}}
232215
/>
216+
{apiJudges.length > 0 ? (
217+
<View className="gap-2 px-4 py-2">
218+
{failedApis.length > 0 ? (
219+
<Text className="text-sm text-foreground-muted">
220+
Could not load models from {failedApis.join(", ")}.
221+
</Text>
222+
) : null}
223+
<Button
224+
label={apiModelsLoading ? "Loading API models…" : "Refresh API models"}
225+
tone="text"
226+
disabled={apiModelsLoading}
227+
onPress={refreshApiModels}
228+
/>
229+
</View>
230+
) : null}
233231
{!judgeAvailable ? (
234232
<Text className="px-4 py-2 text-sm text-foreground-muted">
235233
Choose an available judge before enabling.

‎apps/mobile/src/features/settings/SettingsServerControlsRouteScreen.tsx‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ import {
4444
import { PullRequestGlyph } from "../pull-requests/pullRequestGlyphs";
4545
import { ModelEvaluationSettings } from "./ModelEvaluationSettings";
4646
import { SkillsLibrarySettings } from "./SkillsLibrarySettings";
47-
import { AutoSettleSettingsRows, ThreadSplitSettingsSection } from "./ThreadSettingsSections";
47+
import { AutoSettleSettingsRows } from "./ThreadSettingsSections";
4848
import { supportsSharedSettingsSync } from "@t3tools/client-runtime/state/shared-settings";
4949
import { useHomeListOptions } from "../home/home-list-options";
5050
import { useEnvironments } from "../../state/environments";
@@ -565,7 +565,6 @@ function ServerSettingsDetail(props: { readonly page: SettingsPage }) {
565565
<SkillsLibrarySettings environment={target} />
566566
</SettingsSection>
567567
))}
568-
<ThreadSplitSettingsSection />
569568
<ModelEvaluationSettings />
570569
</>
571570
) : null}

‎apps/mobile/src/features/settings/ThreadSettingsSections.tsx‎

Lines changed: 0 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -393,47 +393,3 @@ function LegacySettingsSection() {
393393
</View>
394394
);
395395
}
396-
397-
export function ThreadSplitSettingsSection() {
398-
const { environments } = useEnvironments();
399-
const connected = environments.filter(
400-
(environment) => environment.connection.phase === "connected",
401-
);
402-
const writable = useEnvironmentsWithScope(connected, AuthSettingsWriteScope);
403-
const updateSettings = useAtomCommand(serverEnvironment.updateSettings, {
404-
label: "thread split suggestions setting",
405-
reportFailure: true,
406-
});
407-
return (
408-
<SettingsSection title="Thread suggestions">
409-
{connected.map((environment) => {
410-
const policy = environment.serverConfig?.managedPolicy;
411-
const managed = isManagedSettingPath(policy, "threadSplitSuggestionsEnabled");
412-
return (
413-
<SettingsSwitchRow
414-
key={environment.environmentId}
415-
icon="square.and.pencil"
416-
label={environment.label}
417-
subtitle={
418-
environment.serverConfig?.settings.threadSplitModelEndpoint
419-
? "Suggest a fresh thread for an independent message. Nothing moves until you choose it."
420-
: "Configure the local decision model in Labs on web or desktop first."
421-
}
422-
value={environment.serverConfig?.settings.threadSplitSuggestionsEnabled === true}
423-
disabled={
424-
managed ||
425-
!writable.has(environment.environmentId) ||
426-
!environment.serverConfig?.settings.threadSplitModelEndpoint
427-
}
428-
onValueChange={(value) =>
429-
void updateSettings({
430-
environmentId: environment.environmentId,
431-
input: { patch: { threadSplitSuggestionsEnabled: value } },
432-
})
433-
}
434-
/>
435-
);
436-
})}
437-
</SettingsSection>
438-
);
439-
}
Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
1+
import { it } from "@effect/vitest";
2+
import {
3+
EnvironmentId,
4+
ProviderInstanceId,
5+
type ListTextGeneratorModelsInput,
6+
type ListTextGeneratorModelsResult,
7+
} from "@t3tools/contracts";
8+
import type { AtomCommandResult } from "@t3tools/client-runtime/state/runtime";
9+
import * as Cause from "effect/Cause";
10+
import * as Effect from "effect/Effect";
11+
import { AsyncResult, Atom, AtomRegistry } from "effect/reactivity";
12+
import { expect, vi } from "vite-plus/test";
13+
14+
type ModelListResult = AtomCommandResult<ListTextGeneratorModelsResult, string>;
15+
type Request = { environmentId: EnvironmentId; input: ListTextGeneratorModelsInput };
16+
17+
const state = vi.hoisted(() => ({
18+
listModels: (_request: Request): Promise<ModelListResult> => {
19+
throw new Error("model discovery not set up");
20+
},
21+
}));
22+
23+
vi.mock("../../state/server", () => ({
24+
serverEnvironment: {
25+
listTextGeneratorModels: {
26+
label: "list models",
27+
run: (_registry: unknown, request: Request) => state.listModels(request),
28+
},
29+
},
30+
}));
31+
32+
import { apiJudgeModelsAtom } from "./modelEvaluationApiModels";
33+
34+
function setup(ids: readonly string[], listModels: typeof state.listModels) {
35+
state.listModels = listModels;
36+
const registry = AtomRegistry.make();
37+
const catalog = apiJudgeModelsAtom(
38+
JSON.stringify(
39+
ids.map((id) => [EnvironmentId.make("judge-env"), ProviderInstanceId.make(id), 0]),
40+
),
41+
);
42+
registry.mount(catalog);
43+
const result = (index: number) => Atom.map(catalog, (results) => results[index]!);
44+
return {
45+
registry,
46+
catalog,
47+
read: (index: number) =>
48+
AtomRegistry.getResult(registry, result(index), { suspendOnWaiting: true }),
49+
[Symbol.dispose]: () => registry.dispose(),
50+
};
51+
}
52+
53+
it.effect("keeps discovery failures visible and can reload unchanged settings after recovery", () =>
54+
Effect.gen(function* () {
55+
using h = setup(["offline-api"], async () => AsyncResult.failure(Cause.fail("offline")));
56+
57+
const failed = yield* Effect.exit(h.read(0));
58+
expect(failed._tag).toBe("Failure");
59+
expect(h.registry.get(h.catalog)[0]?._tag).toBe("Failure");
60+
61+
state.listModels = async ({ input }) =>
62+
AsyncResult.success({
63+
models: input.refresh ? [{ id: "recovered-model", name: "Recovered model" }] : [],
64+
});
65+
h.registry.refresh(h.catalog);
66+
67+
expect(yield* h.read(0)).toEqual([{ id: "recovered-model", name: "Recovered model" }]);
68+
}),
69+
);
70+
71+
it.effect("shows each API's models while another discovery request is still pending", () =>
72+
Effect.gen(function* () {
73+
const slow = Promise.withResolvers<ModelListResult>();
74+
using h = setup(["slow-api", "ready-api"], async ({ input }) =>
75+
input.id === "slow-api"
76+
? slow.promise
77+
: AsyncResult.success({ models: [{ id: "ready-model", name: "Ready model" }] }),
78+
);
79+
80+
expect(yield* h.read(1)).toEqual([{ id: "ready-model", name: "Ready model" }]);
81+
expect(h.registry.get(h.catalog)[0]?.waiting).toBe(true);
82+
83+
slow.resolve(AsyncResult.success({ models: [{ id: "slow-model", name: "Slow model" }] }));
84+
expect(yield* h.read(0)).toEqual([{ id: "slow-model", name: "Slow model" }]);
85+
expect(h.registry.get(h.catalog)[1]?._tag).toBe("Success");
86+
}),
87+
);
Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
import type { EnvironmentId, ProviderInstanceId } from "@t3tools/contracts";
2+
import { runAtomCommand } from "@t3tools/client-runtime/state/runtime";
3+
import * as Effect from "effect/Effect";
4+
import * as Option from "effect/Option";
5+
import { Atom } from "effect/reactivity";
6+
7+
import { serverEnvironment } from "../../state/server";
8+
9+
type ApiJudgeKey = readonly [EnvironmentId, ProviderInstanceId, number];
10+
11+
const apiJudgeModelListAtom = Atom.family((key: string) => {
12+
const [environmentId, id] = JSON.parse(key) as ApiJudgeKey;
13+
return Atom.make((get) => {
14+
const refresh = Option.isSome(get.self());
15+
return Effect.promise(() =>
16+
runAtomCommand(
17+
get.registry,
18+
serverEnvironment.listTextGeneratorModels,
19+
{ environmentId, input: { id, ...(refresh ? { refresh: true } : {}) } },
20+
{ reportFailure: false },
21+
),
22+
).pipe(
23+
Effect.flatMap((result) =>
24+
result._tag === "Success"
25+
? Effect.succeed(result.value.models)
26+
: Effect.failCause(result.cause),
27+
),
28+
);
29+
}).pipe(Atom.setIdleTTL(5 * 60_000));
30+
});
31+
32+
export const apiJudgeModelsAtom = Atom.family((key: string) => {
33+
const requests = (JSON.parse(key) as ReadonlyArray<ApiJudgeKey>).map((api) =>
34+
apiJudgeModelListAtom(JSON.stringify(api)),
35+
);
36+
return Atom.readable(
37+
(get) => requests.map((request) => get(request)),
38+
(refresh) => requests.forEach(refresh),
39+
).pipe(Atom.withLabel("model-evaluation-api-judges"));
40+
});

‎apps/mobile/src/features/threads/ThreadComposer.tsx‎

Lines changed: 1 addition & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
import { groupParticipants, isGroupActive } from "@t3tools/client-runtime/state/thread-group";
22
import { orchestrationEnvironment } from "../../state/orchestration";
3-
import { ThreadSplitSuggestion } from "./ThreadSplitSuggestion";
43
import { ProjectKnowledgeEntry } from "../../components/ProjectKnowledgeEntry";
54
import { useSidebarWorkspaceTabs } from "../layout/workspace/useSystemWorkspaceTabBar";
65
import { Button } from "../../components/ui/Button";
@@ -460,11 +459,7 @@ export const ThreadComposer = function ThreadComposer(props: ThreadComposerProps
460459
const [previewVideo, setPreviewVideo] = useState<VideoPreviewSource | null>(null);
461460
const composerOwnerKey = scopedThreadKey(props.environmentId, props.selectedThread.id);
462461
const composerDraftKey = props.draftKey ?? composerOwnerKey;
463-
const {
464-
text: draftMessage,
465-
attachments: draftAttachments,
466-
context: draftContext,
467-
} = useComposerDraft(composerDraftKey);
462+
const { text: draftMessage, attachments: draftAttachments } = useComposerDraft(composerDraftKey);
468463
const queuedEdit = props.queuedEdit ?? null;
469464
const hasContent =
470465
draftMessage.trim().length > 0 ||
@@ -931,28 +926,6 @@ export const ThreadComposer = function ThreadComposer(props: ThreadComposerProps
931926
/>
932927
) : null}
933928

934-
<ThreadSplitSuggestion
935-
key={composerOwnerKey}
936-
environmentId={props.environmentId}
937-
threadId={props.selectedThread.id}
938-
projectId={props.selectedThread.projectId}
939-
draftKey={composerDraftKey}
940-
message={draftMessage}
941-
enabled={
942-
props.serverConfig?.settings.threadSplitSuggestionsEnabled === true &&
943-
Boolean(props.serverConfig.settings.threadSplitModelEndpoint) &&
944-
!props.isDraft &&
945-
!props.hidden &&
946-
!props.activeThreadBusy &&
947-
props.canOperateThread &&
948-
props.connectionState === "connected" &&
949-
props.queueCount === 0 &&
950-
queuedEdit === null &&
951-
draftAttachments.length === 0 &&
952-
!draftContext
953-
}
954-
/>
955-
956929
<ComposerSurface
957930
animateLayout={false}
958931
style={{

0 commit comments

Comments
 (0)