Skip to content

Commit 70420f5

Browse files
author
i2cjak
committed
Combine workspace controls, terminal, hierarchy, cat and cooperative browser fixes for local acceptance
1 parent be3c046 commit 70420f5

61 files changed

Lines changed: 4709 additions & 2202 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎desktop/browser-actions.cjs‎

Lines changed: 47 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -4,11 +4,16 @@ const {targetKey}=require('./browser-wire.cjs');
44
const refused=message=>Object.assign(new Error(message),{dispatched:false});
55
const pause=ms=>new Promise(resolve=>setTimeout(resolve,ms));
66
const BUTTONS={left:1,right:2,middle:4};
7-
function normalizeURL(value){const raw=String(value||'').trim();if(!raw)throw refused('Browser URL required');return /^[a-z][a-z0-9+.-]*:/i.test(raw)?raw:'https://'+raw;}
7+
function normalizeURL(value){
8+
const raw=String(value||'').trim();if(!raw)throw refused('Browser URL required');
9+
if(/^[a-z][a-z0-9+.-]*:/i.test(raw)&&!/^[^/]+:\d+(\/|$)/.test(raw))return raw;
10+
const host=raw.startsWith('[')?raw.slice(0,raw.indexOf(']')+1):raw.split(/[/:?#]/)[0].toLowerCase();
11+
return (['localhost','127.0.0.1','[::1]','0.0.0.0'].includes(host)||host.endsWith('.localhost')?'http://':'https://')+raw;
12+
}
813
function createBrowserActions(){
914
const references=new Map(),pressed=new Map();
1015
return async function execute(c){
11-
const {op,args:a,target}=c.payload,key=targetKey(target),timeout=Math.max(1,Math.min(14000,Number(a.timeoutMs)||14000));
16+
const {op,args:a,target}=c.payload,key=targetKey(target),timeout=Math.max(1,Math.min(14000,Number(a.timeoutMs)||14000)),deadline=Date.now()+timeout;
1217
const status=()=>{const m=c.metadata();return{page:target.owner,url:m.url,title:m.title,loading:m.loading,tabs:c.list().map((t,index)=>({id:t.tabId,index,url:t.url,title:t.title,active:t.active})),engine:'electron'};};
1318
const command=async(method,params)=>{const answer=await c.command(method,params);return answer.result;};
1419
const evaluate=async(expression,returnByValue=true)=>{
@@ -21,11 +26,34 @@ function createBrowserActions(){
2126
const backendNodeId=references.get(key)?.get(String(a.ref));if(!backendNodeId)throw refused('Unknown or stale Browser reference; take a new snapshot');
2227
return (await command('DOM.resolveNode',{backendNodeId})).object?.objectId;
2328
}
24-
let selector=a.selector,locator=a.locator;
25-
if(locator){if(String(locator).startsWith('css='))selector=String(locator).slice(4);else if(!/^[a-z-]+=/.test(String(locator)))selector=String(locator);else throw refused('This Browser locator engine is not yet supported; use a snapshot reference or CSS selector');}
26-
if(!selector)return null;
27-
const start=Date.now();
28-
do{const value=await evaluate('document.querySelector('+JSON.stringify(String(selector))+')',false);if(value?.objectId)return value.objectId;if(Date.now()-start>=timeout)break;await pause(75);}while(true);
29+
let selector=a.selector,locator=a.locator,role=null,text=null;
30+
if(locator){
31+
const source=String(locator);
32+
if(source.startsWith('css='))selector=source.slice(4);
33+
else if(source.startsWith('text=')){const raw=source.slice(5);text={value:raw,exact:false};if(/^(["']).*\1$/.test(raw))text={value:raw.slice(1,-1),exact:true};}
34+
else if(source.startsWith('role=')){
35+
const match=/^role=([\w-]+)(?:\[name=(?:"([^"\\]*)"|'([^'\\]*)')\])?$/.exec(source);
36+
if(!match)throw refused('Use role=button[name="Name"], text=Text, CSS, or a snapshot reference');
37+
role={name:match[1],label:match[2]??match[3]};
38+
}else if(!/^[a-z-]+=/.test(source))selector=source;
39+
else throw refused('Unsupported Browser locator engine; use a snapshot reference or CSS selector');
40+
}
41+
if(!selector&&!role&&!text)return null;
42+
do{
43+
if(role){
44+
const ax=await command('Accessibility.getFullAXTree',{});
45+
const node=(ax.nodes||[]).find(n=>!n.ignored&&n.role?.value===role.name&&(role.label===undefined||String(n.name?.value||'')===role.label)&&n.backendDOMNodeId);
46+
if(node)return (await command('DOM.resolveNode',{backendNodeId:node.backendDOMNodeId})).object?.objectId;
47+
}else{
48+
const expression=text?'('+function(t){
49+
const normalize=s=>String(s||'').replace(/\s+/g,' ').trim();
50+
const matches=e=>{const s=normalize(e.innerText||e.value);return t.exact?s===normalize(t.value):s.includes(normalize(t.value));};
51+
return [...document.querySelectorAll('body *')].find(e=>!['SCRIPT','STYLE','NOSCRIPT'].includes(e.tagName)&&e.getClientRects().length&&matches(e)&&![...e.children].some(matches))||null;
52+
}.toString()+')('+JSON.stringify(text)+')':'document.querySelector('+JSON.stringify(String(selector))+')';
53+
const value=await evaluate(expression,false);if(value?.objectId)return value.objectId;
54+
}
55+
if(Date.now()>=deadline)break;await pause(75);
56+
}while(true);
2957
throw refused('Browser target was not found');
3058
}
3159
async function useObject(fn){const id=await object();if(!id)throw refused('Browser target required');try{return await fn(id);}finally{void command('Runtime.releaseObject',{objectId:id}).catch(()=>{});}}
@@ -49,7 +77,7 @@ function createBrowserActions(){
4977
if(!a.url&&op==='open')return status();
5078
const url=normalizeURL(a.url),before=c.navigation;
5179
if(c.grant.operations.includes('navigate'))await c.invoke('navigate',{url});else await command('Page.navigate',{url});
52-
await c.waitNavigation({before,readiness:a.readiness==='none'?'commit':'load',timeoutMs:timeout});return status();
80+
await c.waitNavigation({before,readiness:a.readiness==='none'?'commit':a.readiness==='domContentLoaded'?'domContentLoaded':'load',timeoutMs:timeout});return status();
5381
}
5482
case 'back':case 'forward':case 'reload':{
5583
if(c.grant.operations.includes('navigate'))return c.invoke(op==='reload'?'reload':'history',op==='reload'?{}:{direction:op});
@@ -64,13 +92,19 @@ function createBrowserActions(){
6492
const tabs=c.list();return{page:target.owner,active:tabs.findIndex(t=>t.active),tabs:tabs.map((t,index)=>({id:t.tabId,index,url:t.url,title:t.title,active:t.active}))};
6593
}
6694
case 'snapshot':{
67-
if(a.screenshot)throw refused('Direct-guest snapshot images require the explicit capture adapter');
6895
const page=await evaluate(SNAPSHOT_SCRIPT),ax=await command('Accessibility.getFullAXTree',{}),refs=new Map(),lines=[];
6996
for(const node of ax.nodes||[]){if(node.ignored)continue;const role=String(node.role?.value||''),name=String(node.name?.value||'').slice(0,300);if(!role||role==='generic')continue;
7097
const id=node.backendDOMNodeId;if(Number.isSafeInteger(id)&&id>0)refs.set('e'+id,id);
7198
lines.push('- '+role+(name?' '+JSON.stringify(name):'')+(id?' [ref=e'+id+']':''));if(lines.length>=600)break;
7299
}
73-
references.set(key,refs);const meta=c.metadata();return{url:meta.url,title:meta.title,...page,outline:lines.join('\n'),console:meta.console.slice(-50)};
100+
references.set(key,refs);const meta=c.metadata();
101+
let screenshot;
102+
if(a.screenshot){
103+
const capture=await command('Page.captureScreenshot',{format:'png',captureBeyondViewport:false}),metrics=await command('Page.getLayoutMetrics',{}),v=metrics.cssVisualViewport||metrics.visualViewport;
104+
if(typeof capture.data!=='string'||capture.data.length>700000)throw refused('Browser screenshot exceeds the private image limit; request a text snapshot');
105+
screenshot={mimeType:'image/png',data:capture.data,width:Math.round(v.clientWidth),height:Math.round(v.clientHeight)};
106+
}
107+
return{url:meta.url,title:meta.title,...page,outline:lines.join('\n'),console:meta.console.slice(-50),...(screenshot?{screenshot}:{})};
74108
}
75109
case 'text':{
76110
const full=a.selector||a.locator||a.ref?await useObject(id=>onObject(id,'function(){return this.innerText||"";}')):await evaluate(TEXT_SCRIPT);
@@ -100,7 +134,9 @@ function createBrowserActions(){
100134
const modifiers=mods.reduce((v,m)=>v|({Alt:1,Control:2,Meta:4,Shift:8}[m]),0),virtual={Enter:13,Tab:9,Escape:27,Backspace:8,Delete:46,ArrowLeft:37,ArrowUp:38,ArrowRight:39,ArrowDown:40,Home:36,End:35,PageUp:33,PageDown:34};
101135
if(keyName.length===1&&!modifiers){await command('Input.insertText',{text:keyName});return{};}
102136
const windowsVirtualKeyCode=virtual[keyName]??(keyName.length===1?keyName.toUpperCase().charCodeAt(0):0);if(!windowsVirtualKeyCode)throw refused('Unsupported Browser key');
103-
const params={key:keyName,modifiers,windowsVirtualKeyCode};await command('Input.dispatchKeyEvent',{type:'rawKeyDown',...params});await command('Input.dispatchKeyEvent',{type:'keyUp',...params});return{};
137+
const params={key:keyName,modifiers,windowsVirtualKeyCode};
138+
await command('Input.dispatchKeyEvent',{type:'keyDown',...params,...(keyName==='Enter'?{text:'\r',unmodifiedText:'\r'}:{})});
139+
await command('Input.dispatchKeyEvent',{type:'keyUp',...params});return{};
104140
}
105141
case 'scroll':{
106142
const dx=Number(a.dx??a.deltaX??0),dy=Number(a.dy??a.deltaY??0);if(!Number.isFinite(dx)||!Number.isFinite(dy))throw refused('Invalid Browser wheel delta');

‎desktop/browser-controller.cjs‎

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -111,7 +111,7 @@ function createDesktopBrowserController({electron,window,appSession,origin,versi
111111
const prep=preparing.get(target.owner);
112112
if(!prep||prep.createId!==event.createId||prep.tab)throw new Error('Uncorrelated local Browser creation');
113113
lifecycle('created',event);leases.consumeCreate(target.owner,event.createId);
114-
const tab={target,ackTarget:target,createId:event.createId,creationCompleted:false,prep,offered:false,revision:null,lastGrantedRevision:null,tail:Promise.resolve(),queued:0,url:'about:blank',title:'',active:true,navigation:0,loaded:-1,console:[],activeCall:null};
114+
const tab={target,ackTarget:target,createId:event.createId,creationCompleted:false,prep,offered:false,revision:null,lastGrantedRevision:null,tail:Promise.resolve(),queued:0,url:'about:blank',title:'',active:true,navigation:0,domReady:-1,loaded:-1,console:[],activeCall:null};
115115
tabs.set(key,tab);prep.tab=tab;
116116
// Creation establishes registry identity only. prepareGuest enables
117117
// capture after the own recorder proof and initialization grant exist.
@@ -136,7 +136,7 @@ function createDesktopBrowserController({electron,window,appSession,origin,versi
136136
const observedAction=tab.activeCall,observedPrep=tab.prep;
137137
lifecycle(event.type,event);tab.target=target;
138138
if(event.type==='generation'){
139-
tab.loaded=-1;
139+
tab.loaded=-1;tab.domReady=-1;
140140
if(tab.offered)void append(tab,async()=>{
141141
if(observedPrep&&!phaseCurrent(observedPrep))return;
142142
if(!same(tab.ackTarget,priorTarget))throw new Error('Browser generation did not follow its acknowledged predecessor');
@@ -181,6 +181,10 @@ function createDesktopBrowserController({electron,window,appSession,origin,versi
181181
}
182182
if(event.type==='navigated'){tab.url=String(event.url||'');tab.navigation++;if(tab.offered)write('guest.event',{event});wake();return;}
183183
if(event.type==='load-state'){
184+
if(event.nativeEvent==='dom-ready'&&event.mainDocumentReady===true&&event.document&&same(tab.target,target)){
185+
const own=documents.mainProof(target);
186+
if(event.document.documentNonce===own.documentNonce&&event.document.webContentsId===own.webContentsId)tab.domReady=target.generation;
187+
}
184188
if(event.nativeEvent==='did-finish-load'&&event.finishObserved===true&&event.mainDocumentReady===true&&event.document&&same(tab.target,target)){
185189
const own=documents.mainProof(target);
186190
if(event.document?.documentNonce===own.documentNonce&&event.document?.webContentsId===own.webContentsId)tab.loaded=target.generation;
@@ -308,7 +312,10 @@ function createDesktopBrowserController({electron,window,appSession,origin,versi
308312
invoke:(operation,params={})=>dispatch.invoke({id:nonce(),target:p.target,revision:p.grantRevision,operation,params}),
309313
record:on=>{currentGrant(p.target,p.grantRevision);if(unknownState.has(p.owner))throw new Error('Browser owner outcome unresolved');return manager.setRecording(p.target,on);},
310314
waitNavigation:async({before,readiness='load',timeoutMs=15000})=>{
311-
await wait(()=>tab.navigation>before&&(readiness!=='load'||tab.loaded===tab.target.generation),timeoutMs);await tab.tail;
315+
await wait(()=>{
316+
if(tab.navigation<=before||(readiness==='load'&&tab.loaded!==tab.target.generation)||(readiness==='domContentLoaded'&&tab.domReady!==tab.target.generation))return false;
317+
try{return documents.mainProof(tab.target);}catch{return false;}
318+
},timeoutMs);await tab.tail;
312319
documents.mainProof(tab.target);return observe();
313320
},navigation:tab.navigation,grant:leases.targetGrantOf(p.target)});
314321
await drain(p.owner);
@@ -374,10 +381,10 @@ function createDesktopBrowserController({electron,window,appSession,origin,versi
374381
}
375382
async function present({owner,bounds}){
376383
desired={owner,bounds};
377-
if(closed||!identity)return{shown:false,message:'Browser connection unavailable'};
384+
if(closed||!identity)return{shown:false,phase:closed?'error':'connecting',message:'Browser connection unavailable'};
378385
if(!leases.authorize({owner,tabId:null,operation:'list'}))await rpc('owner.request',{owner},{expected:'owner.grant'});
379386
if(desired?.owner!==owner)return{shown:false,message:'Browser selection changed'};
380-
try{return surface.present(desired);}catch{return{shown:false,message:'Open a page with the address bar.'};}
387+
try{const result=surface.present(desired);return result.shown?result:{...result,message:'Open a page with the address bar.'};}catch{return{shown:false,message:'Open a page with the address bar.'};}
381388
}
382389
function hide(){desired=null;surface?.hide();}
383390
function edit(operation){

‎desktop/browser-guest.cjs‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -98,6 +98,7 @@ const CDP_METHODS = new Set([
9898
'Runtime.addBinding', 'Runtime.removeBinding',
9999
'Page.enable', 'Page.navigate', 'Page.reload', 'Page.stopLoading',
100100
'Page.getNavigationHistory', 'Page.navigateToHistoryEntry', 'Page.getLayoutMetrics',
101+
'Page.captureScreenshot',
101102
'Page.addScriptToEvaluateOnNewDocument', 'Page.removeScriptToEvaluateOnNewDocument',
102103
'Page.handleJavaScriptDialog',
103104
'Input.dispatchMouseEvent', 'Input.dispatchKeyEvent', 'Input.insertText',
@@ -107,7 +108,11 @@ const SESSION_OWNERS = new WeakSet();
107108
const RECORDER_SOURCE_SHA256 = '2e4b03765cc4ae6595e32048555ffe27e750994a85c18b89027064f3c0c15350';
108109
const STEP_KEYS = new Set(['click', 'type', 'select', 'press', 'submit']);
109110
const TARGET_KEYS = ['role', 'name', 'label', 'placeholder', 'text', 'testid', 'css', 'nth', 'near', 'tag', 'type'];
110-
const NAVIGATION_METHODS = new Set(['Page.navigate', 'Page.reload', 'Page.navigateToHistoryEntry']);
111+
// Native input may commit a link/form navigation before its CDP reply. The
112+
// original fence still authorizes dispatch; returning its receipt does not
113+
// authorize any command in the new document.
114+
const NAVIGATION_METHODS = new Set(['Page.navigate', 'Page.reload', 'Page.navigateToHistoryEntry',
115+
'Input.dispatchMouseEvent', 'Input.dispatchKeyEvent', 'Input.insertText', 'Input.imeSetComposition']);
111116
const PREPARATION_METHODS = new Set(['Runtime.enable', 'Runtime.addBinding', 'Runtime.removeBinding',
112117
'Page.enable', 'Page.addScriptToEvaluateOnNewDocument', 'Page.removeScriptToEvaluateOnNewDocument']);
113118

@@ -663,6 +668,7 @@ function createBrowserGuestManager({
663668
wc.on('preload-error', () => { if (recorderConfig) recorderFailure(tab, 'PRELOAD_ERROR'); });
664669
wc.on('dom-ready', () => {
665670
if (recorderConfig && !documentReady(tab)) recorderFailure(tab, 'DOCUMENT_NOT_READY');
671+
observeLoad(tab, 'dom-ready');
666672
});
667673
wc.on('destroyed', () => retire(tab, false));
668674
wc.debugger.on('detach', () => {

‎desktop/browser-shield.html‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,8 @@
22
<meta charset="utf-8">
33
<meta http-equiv="Content-Security-Policy" content="default-src 'none'; style-src 'unsafe-inline'; base-uri 'none'; form-action 'none'">
44
<style>
5-
html,body{margin:0;height:100%;overflow:hidden;background:#111410e8;color:#e5eadb;font:16px system-ui,sans-serif;user-select:none}
6-
body{display:grid;place-content:center;text-align:center;cursor:pointer;padding:24px;box-sizing:border-box}
7-
p{max-width:38em;line-height:1.5;margin:6px}strong{font-size:20px}
5+
html,body{margin:0;height:100%;overflow:hidden;background:transparent;color:#e5eadb;font:14px system-ui,sans-serif;user-select:none}
6+
body{display:flex;align-items:flex-end;justify-content:center;cursor:pointer;padding:12px;box-sizing:border-box}
7+
p{max-width:38em;line-height:1.5;margin:0;padding:8px 12px;background:#111410ed;border:1px solid #6d8774}
88
</style>
9-
<strong>The agent is using this page</strong>
10-
<p>Click to request control. Backplane will keep the page protected until control is granted.</p>
9+
<p>The agent is using this page · click to take control</p>

‎desktop/browser-surface.cjs‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ function createBrowserSurface({electron, window, manager, authorize, onTakeover,
1010
shieldSession.setPermissionRequestHandler((_wc,_permission,callback)=>callback(false));
1111
shieldSession.setPermissionCheckHandler(()=>false);
1212
const shield=new electron.WebContentsView({webPreferences:{session:shieldSession,preload:'',nodeIntegration:false,contextIsolation:true,sandbox:true,webSecurity:true,webviewTag:false,navigateOnDragDrop:false}});
13+
shield.setBackgroundColor('rgba(0,0,0,0)');
1314
shield.setVisible(false);
1415
const shieldURL=require('node:url').pathToFileURL(path.join(__dirname,'browser-shield.html')).href;
1516
shield.webContents.setWindowOpenHandler(()=>({action:'deny'}));
@@ -56,7 +57,14 @@ function createBrowserSurface({electron, window, manager, authorize, onTakeover,
5657
const tab=manager.list(target.owner).find(tab=>tab.active&&tab.tabId===target.tabId&&tab.generation===target.generation);
5758
if(!tab)throw new Error('Stale Browser surface');
5859
manager.reconcileInput();
59-
if(inputAllowed(target))removeShield();else cover();
60+
if(inputAllowed(target)){
61+
const took=covered;
62+
removeShield();
63+
// The takeover click is consumed by the shield. Focus only after the
64+
// service grants human input, once, so later lease renewals cannot
65+
// steal focus from the address bar or composer.
66+
if(took)manager.focus(target);
67+
}else cover();
6068
}catch(error){
6169
// An expired or replaced lease must not leave an old interactive view.
6270
hide();onError(error);

0 commit comments

Comments
 (0)