Skip to content

Commit 60936de

Browse files
rwinchrwinch
committed
inject-collector-cache-config supports multiple scan dir
Closes gh-20
1 parent f561814 commit 60936de

8 files changed

Lines changed: 171 additions & 89 deletions

File tree

‎lib/cache-scandir/cache-scandir.js‎

Lines changed: 0 additions & 68 deletions
This file was deleted.

‎lib/cache-scandir/copy-recursive.js‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
'use strict'
2+
3+
const fs = require('fs')
4+
const path = require('path')
5+
6+
const copyRecursiveSync = function (src, dest) {
7+
const exists = fs.existsSync(src)
8+
const stats = exists && fs.statSync(src)
9+
const isDirectory = exists && stats.isDirectory()
10+
if (isDirectory) {
11+
if (!fs.existsSync(dest)) {
12+
fs.mkdirSync(dest)
13+
}
14+
fs.readdirSync(src).forEach(function (childItemName) {
15+
copyRecursiveSync(path.join(src, childItemName), path.join(dest, childItemName))
16+
})
17+
} else {
18+
fs.copyFileSync(src, dest)
19+
}
20+
}
21+
22+
function copyRecursive (scanDir, cacheDir) {
23+
copyRecursiveSync(scanDir, cacheDir)
24+
}
25+
26+
module.exports = copyRecursive

‎lib/cache-scandir/index.js‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
'use strict'
22

3-
const cacheScanDir = require('./cache-scandir')
3+
const copyRecursive = require('./copy-recursive')
44

55
const [, , ...args] = process.argv
6-
const [scanDir, cacheDir, zipFile] = args
6+
const [scanDir, cacheDir] = args
77

8-
cacheScanDir(scanDir, cacheDir, zipFile)
8+
copyRecursive(scanDir, cacheDir)

‎lib/inject-collector-cache-config-extension.js‎

Lines changed: 64 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,9 @@
22

33
const expandPath = require('@antora/expand-path-helper')
44
const ospath = require('path')
5-
const resolvedCacheScanDirIndexJs = require.resolve('@springio/antora-extensions/cache-scandir')
5+
const resolvedCopyRecursiveJs = require.resolve('@springio/antora-extensions/cache-scandir')
66
const { createHash } = require('crypto')
7+
const archiver = require('archiver')
78

89
module.exports.register = function ({ playbook, config = {} }) {
910
const logger = this.getLogger('inject-collector-cache-config-extension')
@@ -24,6 +25,7 @@ module.exports.register = function ({ playbook, config = {} }) {
2425
if (!fs.existsSync(outputDir)) {
2526
fs.mkdirSync(outputDir, { recursive: true })
2627
}
28+
const zipInfo = []
2729
this.once('contentAggregated', async ({ playbook, contentAggregate }) => {
2830
for (const { origins } of contentAggregate) {
2931
for (const origin of origins) {
@@ -73,18 +75,23 @@ module.exports.register = function ({ playbook, config = {} }) {
7375
const { scan: scanConfig = [] } = collector
7476
// cache the output of the build
7577
const scanDir = expandPath(scanConfig.dir, expandPathContext)
76-
logger.info(
77-
`Configuring collector to cache '${scanDir}' at '${cacheDir}' and zip the results at '${zipCacheFile}'`
78-
)
79-
const cachedCollectorConfig = createCachedCollectorConfig(scanDir, cacheDir, zipCacheFile)
78+
logger.info(`Configuring collector to cache '${scanDir}' at '${cacheDir}'`)
79+
const cachedCollectorConfig = createCachedCollectorConfig(scanDir, cacheDir)
8080
normalizedCollectorConfig.push.apply(normalizedCollectorConfig, cachedCollectorConfig)
81-
// add the zip of cache to be published
8281
})
82+
// add the zip of cache to be published
83+
zipInfo.push({ cacheDir, zipCacheFile })
8384
}
8485
}
8586
}
8687
}
8788
})
89+
this.once('beforePublish', async () => {
90+
for (const info of zipInfo) {
91+
console.log(JSON.stringify(info))
92+
await zip(fs, info.cacheDir, info.zipCacheFile)
93+
}
94+
})
8895
}
8996

9097
function download (get, url) {
@@ -110,12 +117,61 @@ function generateWorktreeFolderName ({ url, gitdir, worktree }) {
110117
return `${url.substr(url.lastIndexOf('/') + 1)}-${createHash('sha1').update(url).digest('hex')}`
111118
}
112119

113-
function createCachedCollectorConfig (scanDir, cacheDir, zipFileName, siteDir) {
120+
function createCachedCollectorConfig (scanDir, cacheDir) {
114121
return [
115122
{
116123
run: {
117-
command: `node '${resolvedCacheScanDirIndexJs}' '${scanDir}' '${cacheDir}' '${zipFileName}'`,
124+
command: `node '${resolvedCopyRecursiveJs}' '${scanDir}' '${cacheDir}'`,
118125
},
119126
},
120127
]
121128
}
129+
130+
const zip = async function (fs, src, destination) {
131+
const path = require('path')
132+
const destParent = path.dirname(destination)
133+
if (!fs.existsSync(destParent)) {
134+
fs.mkdirs(destParent, { recursive: true })
135+
}
136+
const output = fs.createWriteStream(destination)
137+
const archive = archiver('zip', {
138+
zlib: { level: 9 }, // Sets the compression level.
139+
})
140+
// listen for all archive data to be written
141+
// 'close' event is fired only when a file descriptor is involved
142+
output.on('close', function () {
143+
console.log(archive.pointer() + ' total bytes')
144+
console.log('archiver has been finalized and the output file descriptor has closed.')
145+
})
146+
147+
// This event is fired when the data source is drained no matter what was the data source.
148+
// It is not part of this library but rather from the NodeJS Stream API.
149+
// @see: https://nodejs.org/api/stream.html#stream_event_end
150+
output.on('end', function () {
151+
console.log('Data has been drained')
152+
})
153+
154+
// good practice to catch warnings (ie stat failures and other non-blocking errors)
155+
archive.on('warning', function (err) {
156+
if (err.code === 'ENOENT') {
157+
// log warning
158+
} else {
159+
// throw error
160+
throw err
161+
}
162+
})
163+
164+
// good practice to catch this error explicitly
165+
archive.on('error', function (err) {
166+
throw err
167+
})
168+
169+
// pipe archive data to the file
170+
archive.pipe(output)
171+
172+
archive.directory(src, false)
173+
174+
await archive.finalize()
175+
176+
console.log(`Saving ${src} into ${destination}`)
177+
}

‎package.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@
4141
"./tabs-migration-extension": "./lib/tabs-migration-extension.js",
4242
"./static-page-extension": "./lib/static-page-extension.js",
4343
"./cache-scandir": "./lib/cache-scandir/index.js",
44+
"./cache-scandir/copy-recursive": "./lib/cache-scandir/copy-recursive.js",
4445
"./set-algolia-env-extension": "./lib/set-algolia-env-extension.js",
4546
"./static-pages/search": "./lib/static/search.adoc",
4647
"./static-pages/spring-projects": "./lib/static/spring-projects.adoc",

test/cache-scandir-test.js renamed to test/copy-recursive-test.js

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ const { name: packageName } = require('#package')
66
const fs = require('fs')
77
const os = require('os')
88
const ospath = require('node:path')
9+
const copyRecursive = require(packageName + '/cache-scandir/copy-recursive')
910

1011
const FIXTURES_DIR = ospath.join(__dirname, 'fixtures')
1112
describe('cache-scandir-command', () => {
@@ -28,12 +29,20 @@ describe('cache-scandir-command', () => {
2829
it('caches the result', () => {
2930
const scanDir = ospath.join(FIXTURES_DIR, 'generated-antora-resources')
3031
const cacheDir = ospath.join(workSpaceDir, 'cache')
31-
const zipFile = ospath.join(FIXTURES_DIR, '.cache/6ca8fb4-1.0.0.zip')
32-
process.argv = ['', '', scanDir, cacheDir, zipFile]
32+
process.argv = ['', '', scanDir, cacheDir]
3333
require(packageName + '/cache-scandir')
34-
expect(fs.existsSync(zipFile)).to.eql(true)
3534
expect(fs.existsSync(ospath.join(cacheDir, 'antora.yml'))).to.eql(true)
3635
expect(fs.existsSync(ospath.join(cacheDir, 'modules/ROOT/pages/generated.adoc'))).to.eql(true)
3736
})
3837
})
38+
it('works when multiple scan_dir and contains existing dir', () => {
39+
const scanDir = ospath.join(FIXTURES_DIR, 'generated-antora-resources')
40+
const cacheDir = ospath.join(workSpaceDir, 'cache')
41+
copyRecursive(scanDir, cacheDir)
42+
const scanDir2 = ospath.join(FIXTURES_DIR, 'generated-antora-resources-2')
43+
copyRecursive(scanDir2, cacheDir)
44+
expect(fs.existsSync(ospath.join(cacheDir, 'antora.yml'))).to.eql(true)
45+
expect(fs.existsSync(ospath.join(cacheDir, 'modules/ROOT/pages/generated.adoc'))).to.eql(true)
46+
expect(fs.existsSync(ospath.join(cacheDir, 'modules/ROOT/pages/generated2.adoc'))).to.eql(true)
47+
})
3948
})

‎test/fixtures/generated-antora-resources-2/modules/ROOT/pages/generated2.adoc‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
= Generated
2+
3+
This is generated

‎test/inject-collector-cache-config-extension-test.js‎

Lines changed: 62 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -185,16 +185,75 @@ describe('inject-collector-cache-config-extension', () => {
185185
},
186186
{
187187
run: {
188-
command: `node '${resolvedCacheScanDirIndexJs}' '${scan}' '${cache}' '${zipFileName}'`,
188+
command: `node '${resolvedCacheScanDirIndexJs}' '${scan}' '${cache}'`,
189189
},
190190
},
191191
],
192192
}
193193
expect(actual).to.eql(expected)
194194
expect(generatorContext.messages).to.eql([
195195
`Unable to restore cache from ${httpServerUrl}/.cache/2c4fb2f-1.0.0.zip`,
196-
`Configuring collector to cache '${scan}' at '${cache}' and zip the results at '${zipFileName}'`,
196+
`Configuring collector to cache '${scan}' at '${cache}'`,
197197
])
198+
expect(fs.existsSync(zipFileName)).to.eql(false)
199+
await generatorContext.beforePublish()
200+
expect(fs.existsSync(zipFileName)).to.eql(true)
201+
})
202+
203+
it('cache not found multiple scans', async () => {
204+
const tag = createTag('1.0.0')
205+
tag.origins[0].refhash = tag.origins[0].refhash.split('').reverse().join('')
206+
// make multiple scan dirs
207+
tag.origins[0].descriptor = {
208+
ext: {
209+
collector: [{ scan: { dir: './build/antora-resources' } }, { scan: { dir: './build/antora-resources-2' } }],
210+
},
211+
}
212+
contentAggregate = [tag]
213+
ext.register.call(generatorContext, { playbook })
214+
await generatorContext.contentAggregated({ playbook, contentAggregate })
215+
expect(fs.existsSync(ospath.join(cacheDir, 'collector-cache/spring-security'))).to.equal(true)
216+
const actual = contentAggregate[0].origins[0].descriptor.ext
217+
const scan = ospath.join(cacheDir, 'collector/spring-security/build/antora-resources')
218+
const scan2 = ospath.join(cacheDir, 'collector/spring-security/build/antora-resources-2')
219+
const cache = ospath.join(cacheDir, 'collector-cache/spring-security/2c4fb2f-1.0.0')
220+
const zipFileName = ospath.join(
221+
playbookDir,
222+
'build/antora/inject-collector-cache-config-extension/.cache/2c4fb2f-1.0.0.zip'
223+
)
224+
const expected = {
225+
collector: [
226+
{
227+
scan: {
228+
dir: './build/antora-resources',
229+
},
230+
},
231+
{
232+
scan: {
233+
dir: './build/antora-resources-2',
234+
},
235+
},
236+
{
237+
run: {
238+
command: `node '${resolvedCacheScanDirIndexJs}' '${scan}' '${cache}'`,
239+
},
240+
},
241+
{
242+
run: {
243+
command: `node '${resolvedCacheScanDirIndexJs}' '${scan2}' '${cache}'`,
244+
},
245+
},
246+
],
247+
}
248+
expect(actual).to.eql(expected)
249+
expect(generatorContext.messages).to.eql([
250+
`Unable to restore cache from ${httpServerUrl}/.cache/2c4fb2f-1.0.0.zip`,
251+
`Configuring collector to cache '${scan}' at '${cache}'`,
252+
`Configuring collector to cache '${scan2}' at '${cache}'`,
253+
])
254+
expect(fs.existsSync(zipFileName)).to.eql(false)
255+
await generatorContext.beforePublish()
256+
expect(fs.existsSync(zipFileName)).to.eql(true)
198257
})
199258
it('cache downloaded', async () => {
200259
const zipFileName = ospath.join(
@@ -259,16 +318,12 @@ describe('inject-collector-cache-config-extension', () => {
259318
const url = playbook.site.url
260319
const scan = ospath.join(cacheDir, 'collector/spring-security/build/antora-resources')
261320
const cache = ospath.join(cacheDir, 'collector-cache/spring-security/6ca8fb4-1.0.0')
262-
const zipFileName = ospath.join(
263-
playbookDir,
264-
'build/antora/inject-collector-cache-config-extension/.cache/6ca8fb4-1.0.0.zip'
265-
)
266321
delete playbook.site.url
267322
ext.register.call(generatorContext, { playbook, config: { baseCacheUrl: url } })
268323
await generatorContext.contentAggregated({ playbook, contentAggregate })
269324
expect(generatorContext.messages).to.eql([
270325
`Unable to restore cache from ${httpServerUrl}/6ca8fb4-1.0.0.zip`,
271-
`Configuring collector to cache '${scan}' at '${cache}' and zip the results at '${zipFileName}'`,
326+
`Configuring collector to cache '${scan}' at '${cache}'`,
272327
])
273328
})
274329
})

0 commit comments

Comments
 (0)