Skip to content

Commit fde9788

Browse files
author
i2cjak
authored
Merge pull request #551 from BackplaneWorks/backplane/viewer-asset-memo
Viewer: answer a warm asset request in the hub
2 parents 62339b6 + 642b540 commit fde9788

4 files changed

Lines changed: 72 additions & 2 deletions

File tree

‎src/server/effects/render_asset.js‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -107,6 +107,49 @@ function ra_prune(dir,keep) {
107107
count++;bytes+=f.size;
108108
}
109109
}
110+
// A published asset is remembered per (project, kind, exact revision): the
111+
// revision is the source's dev:ino:size:mtime:ctime stamp, so it names one
112+
// file. The hub answers a later request for that same revision from this
113+
// private index without starting a child (about 0.24 s of every request);
114+
// any doubt falls back to the child.
115+
function ra_memo_name(crypto,cwd,kind,revision) {
116+
return crypto.createHash('sha256').update(JSON.stringify([cwd,String(kind),revision])).digest('hex')+'.json';
117+
}
118+
function ra_memo_dir(create) {
119+
const {fs,path}=ra_modules(),dir=path.join(ra_cache(create),'memo');
120+
if(create&&!fs.existsSync(dir))fs.mkdirSync(dir,{mode:0o700});
121+
const st=fs.lstatSync(dir);
122+
if(!st.isDirectory()||st.isSymbolicLink()||(typeof process.getuid==='function'&&st.uid!==process.getuid())||(st.mode&0o077)!==0)throw Error('Asset memo directory is unavailable.');
123+
return dir;
124+
}
125+
function ra_memo_write(cwd,kind,revision,asset) {
126+
const {fs,path,crypto}=ra_modules();let temp=null;
127+
try {
128+
const dir=ra_memo_dir(true),name=ra_memo_name(crypto,fs.realpathSync(cwd),kind,revision);
129+
temp=path.join(dir,`.memo-${process.pid}-${crypto.randomBytes(12).toString('hex')}.tmp`);
130+
fs.writeFileSync(temp,JSON.stringify({kind:String(kind),revision,asset}),{mode:0o600,flag:'wx'});fs.renameSync(temp,path.join(dir,name));temp=null;
131+
const names=fs.readdirSync(dir).filter(n=>/^[0-9a-f]{64}\.json$/.test(n)).map(n=>({n,t:fs.statSync(path.join(dir,n)).mtimeMs})).sort((a,b)=>b.t-a.t);
132+
for(const f of names.slice(64))try{fs.unlinkSync(path.join(dir,f.n));}catch{}
133+
}catch{}finally{if(temp)try{fs.unlinkSync(temp);}catch{}}
134+
}
135+
function render_asset_memo(cwd,kind,expected) {
136+
const {fs,path,crypto}=ra_modules();let file=null;
137+
try {
138+
if(!expected)return '';
139+
const dir=ra_memo_dir(false),p=path.join(dir,ra_memo_name(crypto,fs.realpathSync(cwd),kind,expected));
140+
if(!fs.existsSync(p))return '';
141+
const st=fs.lstatSync(p);
142+
if(!st.isFile()||st.isSymbolicLink()||st.nlink!==1||(typeof process.getuid==='function'&&st.uid!==process.getuid())||(st.mode&0o077)!==0||st.size>16000)return '';
143+
const memo=JSON.parse(fs.readFileSync(p,'utf8')),asset=memo?.asset;
144+
if(memo?.revision!==expected||memo?.kind!==String(kind)||!asset||asset.revision!==expected||!/^[0-9a-f]{64}$/.test(asset.sha256||''))return '';
145+
// the source is still exactly the revision asked for
146+
if(ra_stamp(fs.statSync(asset.source,{bigint:true}))!==expected)return '';
147+
// and its published copy is still the private, immutable file it was
148+
const a=path.join(ra_cache(false),asset.sha256+'.glb');ra_regular(fs,a,{privateFile:true});file=ra_open(fs,a);
149+
if((Number(file.st.mode)&0o222)!==0||Number(file.st.size)!==asset.bytes)return '';
150+
return ra_json({asset});
151+
}catch{return '';}finally{if(file)fs.closeSync(file.fd);}
152+
}
110153
function render_asset_authorize(cwd,requested,canonical,kind,expected) {
111154
try {return ra_json(ra_authorize(cwd,requested,canonical,kind,expected));}catch(e){return ra_error(e);}
112155
}
@@ -132,6 +175,7 @@ function render_asset_publish(cwd,source,canonical,kind,expected,glb,missing) {
132175
const extras=json.asset?.extras||{},orientation=extras.unit==='mm'&&String(extras.up).toUpperCase()==='Z'?'step-z-up-mm':authorized.kind==='step'?'step-z-up-mm':'gltf-y-up';
133176
const asset={url:'/viewer-asset?id='+sha256,source:authorized.source,revision:authorized.revision,bytes:size,sha256,orientation};
134177
if(missing)asset.missingModels=missing.slice(0,500);
178+
ra_memo_write(cwd,kind,expected,asset);
135179
return ra_json({asset});
136180
}catch(e){return ra_error(e);}finally{if(input)fs.closeSync(input.fd);if(outFd!==null)fs.closeSync(outFd);if(temp)try{fs.unlinkSync(temp);}catch{}}
137181
}
@@ -147,3 +191,4 @@ function render_asset_lookup(id) {
147191
io_eff(CID(RenderAssetFx.authorize),render_asset_authorize);
148192
io_eff(CID(RenderAssetFx.publish),render_asset_publish);
149193
io_eff(CID(RenderAssetFx.lookup),render_asset_lookup);
194+
io_eff(CID(RenderAssetFx.memo),render_asset_memo);

‎src/server/render-asset.bend‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,10 @@ def RenderAssetFx.publish(cwd: String, path: String, canonical: String, kind: St
1515
def RenderAssetFx.lookup(id: String) -> IO(String):
1616
import "./effects/render_asset.js"
1717

18+
# a published asset for this project, kind and exact revision, or ""
19+
def RenderAssetFx.memo(cwd: String, kind: String, revision: String) -> IO(String):
20+
import "./effects/render_asset.js"
21+
1822
def RenderAsset.error(s: String) -> IO(Unit):
1923
IO.print(J.Json.show(J.Json.obj([J.KV{"error", J.Str{s}}])))
2024

‎src/server/review.bend‎

Lines changed: 17 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -743,9 +743,24 @@ def RvJob.components(+cwd: String, spec: String) -> IO(Res):
743743
+o = Maybe.default(&2, J.Json, J.Json.parse(spec), J.Null{})
744744
IO.bind(P.Ran, Res, Rvc.run(["--review", "components-page", cwd, Nat.show(J.Json.get_nat(o, "offset", 0n)), J.Json.get_str(o, "sourceRevision", ""), J.Json.get_str(o, "schematicRevision", "")]), r => IO.pure(Res, RvJob.components.done(r)))
745745

746-
def RvJob.renderAsset(+cwd: String, path: String, +spec: String) -> IO(Res):
746+
# a warm request for an asset already published at this exact revision is
747+
# answered here (RenderAssetFx.memo re-checks the source's stamp and the
748+
# cached copy); anything else starts the child, which exports and publishes
749+
def RvJob.renderAsset.k(miss: Bool, +hit: String, +cwd: String, +path: String, +kind: String, +revision: String) -> IO(Res):
750+
match miss:
751+
case True{}:
752+
IO.bind(P.Ran, Res, Rvc.run(["--review", "renderasset", cwd, path, kind, revision]), r => IO.pure(Res, RvJob.components.done(r)))
753+
case False{}:
754+
IO.pure(Res, RvJob.components.done(P.Ran{0, hit}))
755+
756+
def RvJob.renderAsset.at(+hit: String, +cwd: String, +path: String, +kind: String, +revision: String) -> IO(Res):
757+
RvJob.renderAsset.k(String.is_empty(hit), hit, cwd, path, kind, revision)
758+
759+
def RvJob.renderAsset(+cwd: String, +path: String, +spec: String) -> IO(Res):
747760
+o = Maybe.default(&2, J.Json, J.Json.parse(spec), J.Null{})
748-
IO.bind(P.Ran, Res, Rvc.run(["--review", "renderasset", cwd, path, J.Json.get_str(o, "kind", "2"), J.Json.get_str(o, "revision", "")]), r => IO.pure(Res, RvJob.components.done(r)))
761+
+kind = J.Json.get_str(o, "kind", "2")
762+
+revision = J.Json.get_str(o, "revision", "")
763+
IO.bind(String, Res, RA.RenderAssetFx.memo(cwd, kind, revision), hit => RvJob.renderAsset.at(hit, cwd, path, kind, revision))
749764

750765
def RvJob.go(i: Nat, +thread: String, +cwd: String, +a: String, +b: String, +home: String) -> IO(Res):
751766
match i:

‎test/tools/render_asset_test.mjs‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,12 @@ try {
3636
assert.equal(one.asset.bytes,bytes.length);assert.equal(one.asset.sha256,digest(bytes));assert.equal(one.asset.orientation,'gltf-y-up');assert.equal(one.asset.revision,originalStat);
3737
const lookup=call('lookup',one.asset.sha256);assert.equal(lookup.size,bytes.length);assert.deepEqual(fs.readFileSync(lookup.path),bytes,'original authored metadata/binary bytes preserved');assert.equal(revision(source),originalStat,'source unchanged');assert.equal(fs.statSync(lookup.path).mode&0o222,0);assert.equal(fs.statSync(join(cache,'backplane-bend/render-assets')).mode&0o077,0);
3838
const firstRevision=revision(lookup.path),again=publish();assert.equal(again.asset.sha256,one.asset.sha256);assert.equal(revision(lookup.path),firstRevision,'cache hit does not overwrite immutable file');
39+
const memo=(k='glb',rev=revision(source))=>effects.get('RenderAssetFx.memo')(project,k,rev);
40+
assert.deepEqual(JSON.parse(memo()).asset,again.asset,'a published revision is answered from the memo');
41+
assert.equal(memo('mech'),'','another kind misses');assert.equal(memo('glb',''),'','no revision misses');assert.equal(memo('glb','0:0:0:0:0:0:0'),'','another revision misses');
42+
assert.equal(effects.get('RenderAssetFx.memo')(outside,'glb',revision(source)),'','another project misses');
43+
{const cached=join(cache,'backplane-bend','render-assets',again.asset.sha256+'.glb');fs.chmodSync(cached,0o644);assert.equal(memo(),'','a writable cached copy misses');fs.chmodSync(cached,0o444);assert.ok(memo());}
44+
{const before=revision(source);fs.utimesSync(source,new Date(),new Date(Date.now()+5000));assert.notEqual(revision(source),before);assert.equal(memo('glb',before),'','a source no longer at the revision misses');assert.equal(memo(),'','its new revision was never published');}
3945
const stepped=glb({extras:{unit:'mm',up:'Z'}},2);fs.writeFileSync(exported,stepped);const b=publish(step,exported,'mech');assert.equal(b.asset.orientation,'step-z-up-mm');assert.equal(b.asset.sha256,digest(stepped));
4046
for(const id of ['../source.glb','A'.repeat(64),'0'.repeat(63),one.asset.sha256+'.glb'])assert(call('lookup',id).error);
4147
assert(call('lookup','f'.repeat(64)).error,'missing id has no arbitrary path fallback');

0 commit comments

Comments
 (0)