Skip to content

Commit bc83425

Browse files
committed
fix(core): preserve deterministic worker and source ordering (nodejs#1183)
1 parent ec7751f commit bc83425

7 files changed

Lines changed: 174 additions & 37 deletions

File tree

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
---
2+
'@doc-kit/core': patch
3+
---
4+
5+
Keep worker results and discovered source files in deterministic order so repeated documentation builds produce stable output.
Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
import assert from 'node:assert/strict';
2+
import { mkdtemp, rm, writeFile } from 'node:fs/promises';
3+
import { tmpdir } from 'node:os';
4+
import { join, sep } from 'node:path';
5+
import { it } from 'node:test';
6+
7+
import { setConfig } from '#utils/configuration/index.mjs';
8+
9+
import { generate } from '../generate.mjs';
10+
11+
it('submits JavaScript files in sorted path order', async () => {
12+
const dir = await mkdtemp(join(tmpdir(), 'ast-js-order-'));
13+
14+
try {
15+
for (const name of ['z.js', 'a.js', 'm.js', 'ignore.md']) {
16+
await writeFile(join(dir, name), '');
17+
}
18+
19+
const input = join(dir, '*').split(sep).join('/');
20+
await setConfig({ target: ['ast-js'], input: [input] });
21+
22+
let received;
23+
const worker = {
24+
async *stream(items) {
25+
received = items;
26+
yield [];
27+
},
28+
};
29+
30+
for await (const chunk of generate(undefined, worker)) {
31+
assert.deepStrictEqual(chunk, []);
32+
}
33+
34+
assert.deepStrictEqual(
35+
received,
36+
['a.js', 'm.js', 'z.js'].map(name =>
37+
join(dir, name).split(sep).join('/')
38+
)
39+
);
40+
} finally {
41+
await rm(dir, { recursive: true, force: true });
42+
}
43+
});

‎packages/core/src/generators/ast-js/generate.mjs‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -44,9 +44,9 @@ export async function processChunk(inputSlice, itemIndices) {
4444
export async function* generate(_, worker) {
4545
const config = getConfig('ast-js');
4646

47-
const files = globSync(config.input, { ignore: config.ignore }).filter(
48-
p => extname(p) === '.js'
49-
);
47+
const files = globSync(config.input, { ignore: config.ignore })
48+
.filter(p => extname(p) === '.js')
49+
.sort();
5050

5151
// Parse the Javascript sources into ASTs in parallel using worker threads
5252
// source is both the items list and the fullInput since we use sliceInput

‎packages/core/src/generators/ast/__tests__/generate.test.mjs‎

Lines changed: 47 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,16 @@
11
import assert from 'node:assert/strict';
2-
import { mkdtemp, writeFile, rm } from 'node:fs/promises';
2+
import { mkdir, mkdtemp, writeFile, rm } from 'node:fs/promises';
33
import { tmpdir } from 'node:os';
44
import { join, sep } from 'node:path';
55
import { after, before, describe, it } from 'node:test';
66
import { pathToFileURL } from 'node:url';
77

88
import { loadGenerator } from '#generators/loader.mjs';
99
import { loadMarkdownPlugins } from '#utils/markdown/plugins.mjs';
10+
import { setConfig } from '#utils/configuration/index.mjs';
1011

1112
import { STABILITY_INDEX_URL } from '../constants.mjs';
12-
import { processChunk } from '../generate.mjs';
13+
import { generate, processChunk } from '../generate.mjs';
1314

1415
const ast = await loadGenerator(import.meta.resolve('../index.mjs'));
1516

@@ -216,3 +217,47 @@ describe('processChunk', () => {
216217
});
217218
});
218219
});
220+
221+
222+
describe('generate', () => {
223+
it('sorts files inside each pattern without reordering configured patterns', async () => {
224+
const first = join(dir, 'ordered-first');
225+
const second = join(dir, 'ordered-second');
226+
await mkdir(first);
227+
await mkdir(second);
228+
229+
for (const name of ['z.md', 'a.md', 'm.md']) {
230+
await writeFile(join(first, name), '# Title\n');
231+
await writeFile(join(second, name), '# Title\n');
232+
}
233+
234+
await setConfig({
235+
target: ['ast'],
236+
input: [
237+
toPosixPath(join(second, '*.md')),
238+
toPosixPath(join(first, '*.md')),
239+
],
240+
});
241+
242+
let received;
243+
const worker = {
244+
async *stream(items) {
245+
received = items;
246+
yield [];
247+
},
248+
};
249+
250+
for await (const chunk of generate(undefined, worker)) {
251+
assert.deepStrictEqual(chunk, []);
252+
}
253+
254+
assert.deepStrictEqual(
255+
received.map(([child]) => child),
256+
[second, first].flatMap(parent =>
257+
['a.md', 'm.md', 'z.md'].map(name =>
258+
toPosixPath(join(parent, name))
259+
)
260+
)
261+
);
262+
});
263+
});

‎packages/core/src/generators/ast/generate.mjs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -117,7 +117,7 @@ export async function* generate(_, worker) {
117117
const files = config.input.flatMap(input => {
118118
const parent = globParent(input);
119119

120-
return globSync(input, { ignore: config.ignore }).map(child => [
120+
return globSync(input, { ignore: config.ignore }).sort().map(child => [
121121
child,
122122
parent,
123123
]);

‎packages/core/src/threading/__tests__/parallel.test.mjs‎

Lines changed: 53 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import { deepStrictEqual, ok, strictEqual } from 'node:assert';
1+
import { deepStrictEqual, ok, rejects, strictEqual } from 'node:assert';
22
import { describe, it } from 'node:test';
33

44
import { loadGenerator } from '../../generators/loader.mjs';
@@ -177,4 +177,56 @@ describe('createParallelWorker', () => {
177177

178178
await pool.destroy();
179179
});
180+
it('yields concurrent worker results in submission order', async () => {
181+
const resolvers = [];
182+
const pool = {
183+
run: task =>
184+
new Promise(resolve => {
185+
resolvers.push(() => resolve(task.input));
186+
}),
187+
};
188+
const worker = createParallelWorker('ast-js', astJsGenerator, pool, {
189+
threads: 2,
190+
chunkSize: 1,
191+
});
192+
193+
const result = collectStream(worker.stream([0, 1, 2, 3, 4], {}));
194+
await Promise.resolve();
195+
196+
// All five tasks start before any result is yielded, but finish in reverse.
197+
strictEqual(resolvers.length, 5);
198+
for (const resolve of resolvers.toReversed()) {
199+
resolve();
200+
}
201+
202+
deepStrictEqual(await result, [0, 1, 2, 3, 4]);
203+
});
204+
205+
it('propagates a later worker failure without an unhandled rejection', async () => {
206+
const tasks = [];
207+
const pool = {
208+
run: task =>
209+
new Promise((resolve, reject) => {
210+
tasks.push({ resolve, reject, input: task.input });
211+
}),
212+
};
213+
const worker = createParallelWorker('ast-js', astJsGenerator, pool, {
214+
threads: 2,
215+
chunkSize: 1,
216+
});
217+
218+
const result = collectStream(worker.stream([0, 1, 2, 3], {}));
219+
await Promise.resolve();
220+
strictEqual(tasks.length, 4);
221+
222+
tasks[2].reject(new Error('chunk failed'));
223+
for (const [index, task] of tasks.entries()) {
224+
if (index !== 2) {
225+
task.resolve(task.input);
226+
}
227+
}
228+
229+
await rejects(result, /chunk failed/);
230+
});
231+
180232
});

‎packages/core/src/threading/parallel.mjs‎

Lines changed: 22 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -77,12 +77,12 @@ export default function createParallelWorker(
7777

7878
return {
7979
/**
80-
* Processes items in parallel, yielding results as chunks complete.
80+
* Processes items in parallel, yielding results in submission order.
8181
*
8282
* @template T
8383
* @param {T[]} items - Items to process
8484
* @param {object} extra - Extra options
85-
* @yields {R[]} Chunk results as they complete
85+
* @yields {R[]} Chunk results in input order
8686
*/
8787
async *stream(items, extra) {
8888
if (items.length === 0) {
@@ -98,42 +98,34 @@ export default function createParallelWorker(
9898

9999
const runInOneGo = threads <= 1 || items.length <= 2;
100100

101-
// Submit all tasks to Piscina - each promise resolves to itself for removal
102-
const pending = new Set(
103-
chunks.map(indices => {
104-
if (runInOneGo) {
105-
const promise = generator
106-
.processChunk(items, indices, extra)
107-
.then(result => ({ promise, result }));
108-
109-
return promise;
110-
}
111-
112-
const promise = pool
113-
.run(
101+
// Start every chunk immediately, but emit results in submission order.
102+
// Handle errors as each task settles so a later failure cannot become an
103+
// unhandled rejection while earlier chunks are still running.
104+
const tasks = chunks.map(indices => {
105+
const task = runInOneGo
106+
? generator.processChunk(items, indices, extra)
107+
: pool.run(
114108
createTask(items, indices, extra, configuration, specifier, name)
115-
)
116-
.then(result => ({ promise, result }));
117-
118-
return promise;
119-
})
120-
);
121-
122-
// Yield results as they complete (true parallel collection)
123-
let completed = 0;
109+
);
124110

125-
while (pending.size > 0) {
126-
const { promise, result } = await Promise.race(pending);
111+
return task.then(
112+
result => ({ result }),
113+
error => ({ error })
114+
);
115+
});
127116

128-
pending.delete(promise);
117+
for (const [index, task] of tasks.entries()) {
118+
const outcome = await task;
129119

130-
completed++;
120+
if ('error' in outcome) {
121+
throw outcome.error;
122+
}
131123

132-
parallelLogger.debug(`Chunk ${completed}/${chunks.length} completed`, {
124+
parallelLogger.debug(`Chunk ${index + 1}/${chunks.length} completed`, {
133125
generator: name,
134126
});
135127

136-
yield result;
128+
yield outcome.result;
137129
}
138130
},
139131
};

0 commit comments

Comments
 (0)