@@ -18,6 +18,7 @@ import * as Fiber from "effect/Fiber";
1818import * as Layer from "effect/Layer" ;
1919import * as Path from "effect/Path" ;
2020import * as SqlClient from "effect/sql/SqlClient" ;
21+ import { ATTACHMENT_ONLY_BOOTSTRAP_PROMPT } from "@t3tools/shared/composerContextReferences" ;
2122
2223import * as SqlitePersistence from "../persistence/Sqlite.ts" ;
2324import * as ProjectionStore from "./ProjectionStore.ts" ;
@@ -141,6 +142,119 @@ const message = (
141142 } ,
142143} ) ;
143144
145+ it . effect . each ( [ true , false ] ) (
146+ "ThreadSearch excludes bootstrap text before the hit and candidate limits (indexed: %s)" ,
147+ ( indexed ) =>
148+ Effect . gen ( function * ( ) {
149+ const projections = yield * ProjectionStore . ProjectionStoreV2 ;
150+ const search = yield * ThreadSearch . ThreadSearch ;
151+ const sql = yield * SqlClient . SqlClient ;
152+ const project = ProjectId . make ( "project:attachments" ) ;
153+ const threadId = ThreadId . make ( "thread:attachments" ) ;
154+ yield * createProject ( project ) ;
155+ yield * Effect . forEach (
156+ [
157+ thread ( threadId , project ) ,
158+ message ( threadId , "ordinary" , "user" , "Review the attached files in context" , {
159+ minute : 1 ,
160+ } ) ,
161+ message ( threadId , "attachment" , "user" , ATTACHMENT_ONLY_BOOTSTRAP_PROMPT , { minute : 2 } ) ,
162+ message (
163+ threadId ,
164+ "ultrathink" ,
165+ "user" ,
166+ `Ultrathink:\n${ ATTACHMENT_ONLY_BOOTSTRAP_PROMPT } ` ,
167+ { minute : 3 } ,
168+ ) ,
169+ ] ,
170+ projections . apply ,
171+ { discard : true } ,
172+ ) ;
173+ if ( ! indexed ) {
174+ yield * sql `DROP TRIGGER orchestration_v2_message_search_insert` ;
175+ yield * sql `DROP TRIGGER orchestration_v2_message_search_update` ;
176+ yield * sql `DROP TRIGGER orchestration_v2_message_search_delete` ;
177+ yield * sql `DROP TABLE orchestration_v2_message_search` ;
178+ }
179+ assert . deepEqual ( ( yield * search . search ( { query : "conversation context" } ) ) . matches , [ ] ) ;
180+ assert . deepEqual ( ( yield * search . search ( { query : "ultrathink" } ) ) . matches , [ ] ) ;
181+ assert . deepEqual (
182+ ( yield * search . search ( { query : "attached files" } ) ) . matches . map ( ( match ) => [
183+ match . messageId ,
184+ match . snippet ,
185+ ] ) ,
186+ [ [ MessageId . make ( "ordinary" ) , "Review the attached files in context" ] ] ,
187+ ) ;
188+ for ( const text of [
189+ ATTACHMENT_ONLY_BOOTSTRAP_PROMPT ,
190+ `Ultrathink:\n${ ATTACHMENT_ONLY_BOOTSTRAP_PROMPT } ` ,
191+ `uLtRaThInK:\r\n\t\u00a0${ ATTACHMENT_ONLY_BOOTSTRAP_PROMPT } ` ,
192+ ] ) {
193+ yield * sql `
194+ WITH RECURSIVE n(i) AS (SELECT 1 UNION ALL SELECT i + 1 FROM n WHERE i < ${ indexed ? 10_000 : 2000 } )
195+ INSERT INTO orchestration_v2_projection_messages
196+ (message_id, thread_id, role, streaming, created_at, updated_at, payload_json)
197+ SELECT 'attachment-limit:' || i, ${ threadId } , 'user', 0,
198+ ${ DateTime . formatIso ( at ( 5 ) ) } , ${ DateTime . formatIso ( at ( 5 ) ) } , json_object('text', ${ text } )
199+ FROM n
200+ ` ;
201+ assert . deepEqual (
202+ ( yield * search . search ( { query : "context" } ) ) . matches . map ( ( match ) => match . messageId ) ,
203+ [ MessageId . make ( "ordinary" ) ] ,
204+ ) ;
205+ yield * sql `
206+ DELETE FROM orchestration_v2_projection_messages WHERE message_id LIKE 'attachment-limit:%'
207+ ` ;
208+ }
209+ } ) . pipe ( Effect . provide ( layerTest ) ) ,
210+ ) ;
211+
212+ it . effect ( "ThreadSearch excludes legacy placeholders still in the index" , ( ) =>
213+ Effect . gen ( function * ( ) {
214+ const projections = yield * ProjectionStore . ProjectionStoreV2 ;
215+ const search = yield * ThreadSearch . ThreadSearch ;
216+ const sql = yield * SqlClient . SqlClient ;
217+ const project = ProjectId . make ( "project:legacy-attachments" ) ;
218+ const threadId = ThreadId . make ( "thread:legacy-attachments" ) ;
219+ yield * createProject ( project ) ;
220+ // Simulate the pre-upgrade trigger, which indexed attachment placeholders.
221+ yield * sql `DROP TRIGGER orchestration_v2_message_search_insert` ;
222+ yield * sql `
223+ CREATE TRIGGER orchestration_v2_message_search_insert
224+ AFTER INSERT ON orchestration_v2_projection_messages
225+ BEGIN
226+ INSERT INTO orchestration_v2_message_search (rowid, text)
227+ VALUES (new.rowid, json_extract(new.payload_json, '$.text'));
228+ END
229+ ` ;
230+ yield * Effect . forEach (
231+ [
232+ thread ( threadId , project ) ,
233+ message ( threadId , "ordinary" , "user" , "Review the attached files in context" ) ,
234+ message ( threadId , "attachment" , "user" , ATTACHMENT_ONLY_BOOTSTRAP_PROMPT ) ,
235+ message (
236+ threadId ,
237+ "ultrathink" ,
238+ "user" ,
239+ `uLtRaThInK:\r\n\t\u00a0${ ATTACHMENT_ONLY_BOOTSTRAP_PROMPT } ` ,
240+ ) ,
241+ ] ,
242+ projections . apply ,
243+ { discard : true } ,
244+ ) ;
245+ assert . lengthOf (
246+ yield * sql `SELECT rowid FROM orchestration_v2_message_search WHERE orchestration_v2_message_search MATCH '"context"'` ,
247+ 3 ,
248+ ) ;
249+ assert . deepEqual ( ( yield * search . search ( { query : "conversation context" } ) ) . matches , [ ] ) ;
250+ assert . deepEqual ( ( yield * search . search ( { query : "ultrathink" } ) ) . matches , [ ] ) ;
251+ assert . deepEqual (
252+ ( yield * search . search ( { query : "context" } ) ) . matches . map ( ( match ) => match . messageId ) ,
253+ [ MessageId . make ( "ordinary" ) ] ,
254+ ) ;
255+ } ) . pipe ( Effect . provide ( layerTest ) ) ,
256+ ) ;
257+
144258it . layer ( layerTest ) ( "ThreadSearch" , ( it ) => {
145259 it . effect ( "reads the newest hits without sorting all of them" , ( ) =>
146260 Effect . gen ( function * ( ) {
@@ -153,6 +267,15 @@ it.layer(layerTest)("ThreadSearch", (it) => {
153267 plan . some ( ( row ) => row . detail . includes ( "TEMP B-TREE" ) ) ,
154268 plan . map ( ( row ) => row . detail ) . join ( "\n" ) ,
155269 ) ;
270+ if ( match !== null ) {
271+ assert . lengthOf (
272+ plan . filter ( ( row ) =>
273+ row . detail . startsWith ( "SEARCH messages USING INTEGER PRIMARY KEY" ) ,
274+ ) ,
275+ 1 ,
276+ plan . map ( ( row ) => row . detail ) . join ( "\n" ) ,
277+ ) ;
278+ }
156279 }
157280 } ) ,
158281 ) ;
0 commit comments