Skip to content

Commit ee156cc

Browse files
committed
feat: add improved lazy-chunking of command strings
1 parent baa7c98 commit ee156cc

6 files changed

Lines changed: 243 additions & 40 deletions

File tree

‎.changeset/better-loops-cheat.md‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
---
2+
'lint-staged': minor
3+
---
4+
5+
The chunking of tasks based on maximum command line argument length has been re-implemented to be more precise. Now the chunking happens based on the final generated command string, instead of just the list of staged files like previously. This benefits mainly Windows platforms and function commands like:
6+
7+
```js
8+
/** @type {import('lint-staged').Configuration} */
9+
export default {
10+
'*.ts': () => 'tsc', // Run "tsc" when any TS file is changed (for entire project)
11+
}
12+
```
13+
14+
Where the spawned command is literally `"tsc"` without any extra arguments. Previously, this was still chunked when a lot of files were staged. Now, it probably won't be chunked because the length of the command is just three letters.
15+
16+
Also, native JavaScript/Node.js function tasks won't be chunked at all, when previously they were run multiple times when chunked:
17+
18+
```js
19+
/** @type {import('lint-staged').Configuration} */
20+
export default {
21+
'*.js': {
22+
title: 'Log staged JS files to console',
23+
task: async (files) => {
24+
console.log('Staged JS files:', files)
25+
},
26+
},
27+
}
28+
```

‎lib/getSpawnedTasks.js‎

Lines changed: 125 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,104 @@ import { configurationError } from './messages.js'
44

55
const debugLog = createDebug('lint-staged:getSpawnedTasks')
66

7+
/**
8+
* Get the maximum length of a command-line argument string based on current platform
9+
*
10+
* https://serverfault.com/questions/69430/what-is-the-maximum-length-of-a-command-line-in-mac-os-x
11+
* https://support.microsoft.com/en-us/help/830473/command-prompt-cmd-exe-command-line-string-limitation
12+
* https://unix.stackexchange.com/a/120652
13+
*/
14+
export const getMaxArgLength = (platform = process.platform) => {
15+
switch (platform) {
16+
case 'darwin':
17+
return 262_144
18+
case 'win32':
19+
return 8_191
20+
default:
21+
return 131_072
22+
}
23+
}
24+
25+
/**
26+
* Chunk `files` so that when joined with `command`, the resulting string fits into `maxArgLength`.
27+
* For function commands, they are evaluated with the list of files and measured afterwards.
28+
*
29+
* @param {string | Function} command
30+
* @param {string[]} files
31+
* @param {number} [maxArgLength]
32+
* @returns {Promise<Array<{ command: string, files: string[] }>>}
33+
*/
34+
export const chunkFilesForCommand = async (command, files, maxArgLength) => {
35+
const isFn = typeof command === 'function'
36+
const resolved = isFn ? await command([...files]) : command
37+
const resolvedCommands = Array.isArray(resolved) ? resolved : [resolved]
38+
39+
if (isFn && resolvedCommands.some((command) => typeof command !== 'string')) {
40+
throw new Error(
41+
configurationError(
42+
'[Function]',
43+
'Function task should return a string or an array of strings',
44+
resolved
45+
)
46+
)
47+
}
48+
49+
if (!maxArgLength || maxArgLength === Infinity || files.length <= 1) {
50+
debugLog(
51+
'Skipping chunking because maxArgLength is %s and there are %s files',
52+
maxArgLength,
53+
files.length
54+
)
55+
56+
return resolvedCommands.map((resolvedCommand) => ({
57+
command: resolvedCommand,
58+
files,
59+
}))
60+
}
61+
62+
const fits = resolvedCommands.every((resolvedCommand) => {
63+
/**
64+
* Really the command is spawned using `tinyexec` and the files passed as an array of
65+
* arguments; but this approximates the length without quoting filenames.
66+
*/
67+
const finalCommandString = isFn ? resolvedCommand : resolvedCommand + ' ' + files.join(' ')
68+
69+
debugLog(
70+
'Resolved an argument string length of %d characters from %d files',
71+
finalCommandString.length,
72+
files.length
73+
)
74+
75+
return finalCommandString.length <= maxArgLength
76+
})
77+
78+
if (fits) {
79+
return resolvedCommands.map((resolvedCommand) => ({
80+
command: resolvedCommand,
81+
files,
82+
}))
83+
}
84+
85+
const middle = Math.ceil(files.length / 2)
86+
const left = files.slice(0, middle)
87+
const right = files.slice(middle)
88+
89+
debugLog(
90+
'Splitting %d files into chunks of %d and %d for maxArgLength of %d',
91+
files.length,
92+
left.length,
93+
right.length,
94+
maxArgLength
95+
)
96+
97+
const commands = await Promise.all([
98+
chunkFilesForCommand(command, left, maxArgLength),
99+
chunkFilesForCommand(command, right, maxArgLength),
100+
])
101+
102+
return commands.flat()
103+
}
104+
7105
/**
8106
* Creates and returns an array of tasks which map to the given commands.
9107
*
@@ -13,6 +111,7 @@ const debugLog = createDebug('lint-staged:getSpawnedTasks')
13111
* @param {string} options.continueOnError
14112
* @param {string} options.cwd
15113
* @param {import('./getStagedFiles.js').StagedFile[]} options.files
114+
* @param {number} [options.maxArgLength]
16115
* @param {string} options.topLevelDir
17116
* @param {Boolean} verbose
18117
*/
@@ -22,52 +121,41 @@ export const getSpawnedTasks = async ({
22121
continueOnError,
23122
cwd,
24123
files,
124+
maxArgLength,
25125
topLevelDir,
26126
verbose,
27127
}) => {
28128
debugLog('Creating tasks for commands %o', commands)
29-
const cmdTasks = []
30129

31130
const commandArray = Array.isArray(commands) ? commands : [commands]
32131

33132
const filepaths = files.map((f) => f.filepath)
34133

35-
for (const cmd of commandArray) {
36-
// command function may return array of commands that already include `stagedFiles`
37-
const isFn = typeof cmd === 'function'
38-
39-
/** Pass copy of file list to prevent mutation by function from config file. */
40-
const resolved = isFn ? await cmd([...filepaths]) : cmd
41-
42-
const resolvedArray = Array.isArray(resolved) ? resolved : [resolved] // Wrap non-array command as array
43-
44-
for (const command of resolvedArray) {
45-
// If the function linter didn't return string | string[] it won't work
46-
// Do the validation here instead of `validateConfig` to skip evaluating the function multiple times
47-
if (isFn && typeof command !== 'string') {
48-
throw new Error(
49-
configurationError(
50-
'[Function]',
51-
'Function task should return a string or an array of strings',
52-
resolved
53-
)
54-
)
55-
}
56-
57-
const task = getSpawnedTask({
58-
abortController,
59-
command,
60-
continueOnError,
61-
cwd,
62-
files: filepaths,
63-
topLevelDir,
64-
isFn,
65-
verbose,
66-
})
67-
68-
cmdTasks.push({ title: command, task })
69-
}
134+
if (filepaths.length === 0) {
135+
debugLog('Skipping task generation because no files matched', { commands })
136+
return []
70137
}
71138

72-
return cmdTasks
139+
const spawnedTasks = await Promise.all(
140+
commandArray.map(async (cmd) => {
141+
const isFn = typeof cmd === 'function'
142+
const chunkedCommands = await chunkFilesForCommand(cmd, filepaths, maxArgLength)
143+
144+
return chunkedCommands.map(({ command, files: chunkedFiles }) => ({
145+
title: command,
146+
task: getSpawnedTask({
147+
abortController,
148+
command,
149+
continueOnError,
150+
cwd,
151+
files: chunkedFiles,
152+
topLevelDir,
153+
isFn,
154+
verbose,
155+
}),
156+
}))
157+
})
158+
)
159+
160+
return spawnedTasks.flat()
73161
}

‎lib/index.js‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ import { assertGitVersion, MIN_GIT_VERSION } from './assertGitVersion.js'
22
import { enableColors } from './colors.js'
33
import { createDebug, enableDebug } from './debug.js'
44
import { execGit } from './execGit.js'
5+
import { getMaxArgLength } from './getSpawnedTasks.js'
56
import {
67
gitError,
78
minGitVersionRequired,
@@ -73,7 +74,7 @@ const lintStaged = async (
7374
hideAll = false,
7475
hideUnstaged = false,
7576
hidePartiallyStaged = !(hideAll || hideUnstaged),
76-
maxArgLength,
77+
maxArgLength = getMaxArgLength(),
7778
quiet = false,
7879
relative = false,
7980
// Stashing should be disabled by default when the `diff` option is used

‎lib/runAll.js‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -100,7 +100,7 @@ export const runAll = async (
100100
hideAll = false,
101101
hideUnstaged = false,
102102
hidePartiallyStaged = !(hideAll || hideUnstaged),
103-
maxArgLength: _maxArgLength,
103+
maxArgLength,
104104
quiet = false,
105105
relative = false,
106106
// Stashing should be disabled by default when the `diff` option is used
@@ -223,6 +223,7 @@ export const runAll = async (
223223
continueOnError,
224224
cwd: groupCwd,
225225
files: task.fileList,
226+
maxArgLength,
226227
topLevelDir,
227228
verbose,
228229
})
Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
import { describe, it, vi } from 'vitest'
2+
3+
import { chunkFilesForCommand } from '../../lib/getSpawnedTasks.js'
4+
5+
describe('chunkFilesForCommand', () => {
6+
it.for([
7+
[undefined, ['a.js', 'b.js']],
8+
[Infinity, ['a.js', 'b.js']],
9+
[1, ['a.js']],
10+
])(
11+
'does not chunk with maxArgLength $0 and files $1',
12+
async ([maxArgLength, files], { expect }) => {
13+
await expect(chunkFilesForCommand('lint', files, maxArgLength)).resolves.toEqual([
14+
{ command: 'lint', files },
15+
])
16+
}
17+
)
18+
19+
it('chunks string commands while preserving file order', async ({ expect }) => {
20+
const files = ['a.js', 'b.js', 'c.js', 'd.js', 'e.js']
21+
22+
await expect(chunkFilesForCommand('lint', files, 19)).resolves.toEqual([
23+
{ command: 'lint', files: ['a.js', 'b.js', 'c.js'] },
24+
{ command: 'lint', files: ['d.js', 'e.js'] },
25+
])
26+
})
27+
28+
it('reevaluates async function commands for each chunk', async ({ expect }) => {
29+
const files = ['a.js', 'b.js', 'c.js', 'd.js', 'e.js']
30+
const command = vi.fn(async (chunk) => `lint ${chunk.join(' ')}`)
31+
32+
await expect(chunkFilesForCommand(command, files, 19)).resolves.toEqual([
33+
{ command: 'lint a.js b.js c.js', files: ['a.js', 'b.js', 'c.js'] },
34+
{ command: 'lint d.js e.js', files: ['d.js', 'e.js'] },
35+
])
36+
expect(command).toHaveBeenCalledTimes(3)
37+
expect(command).toHaveBeenNthCalledWith(1, files)
38+
expect(command).toHaveBeenNthCalledWith(2, ['a.js', 'b.js', 'c.js'])
39+
expect(command).toHaveBeenNthCalledWith(3, ['d.js', 'e.js'])
40+
})
41+
42+
it('chunks when any command returned by a function is too long', async ({ expect }) => {
43+
const command = (files) => [`lint ${files.join(' ')}`, `format ${files.join(' ')}`]
44+
45+
await expect(chunkFilesForCommand(command, ['a.js', 'b.js'], 14)).resolves.toEqual([
46+
{ command: 'lint a.js', files: ['a.js'] },
47+
{ command: 'format a.js', files: ['a.js'] },
48+
{ command: 'lint b.js', files: ['b.js'] },
49+
{ command: 'format b.js', files: ['b.js'] },
50+
])
51+
})
52+
53+
it('passes a copy of the files to function commands', async ({ expect }) => {
54+
const files = ['a.js', 'b.js']
55+
const command = async (commandFiles) => {
56+
commandFiles.splice(0)
57+
return 'lint'
58+
}
59+
60+
await expect(chunkFilesForCommand(command, files)).resolves.toEqual([
61+
{ command: 'lint', files },
62+
])
63+
expect(files).toEqual(['a.js', 'b.js'])
64+
})
65+
66+
it.for([null, ['lint', null]])(
67+
'rejects an invalid function result: %j',
68+
async (result, { expect }) => {
69+
await expect(chunkFilesForCommand(() => result, ['a.js'])).rejects.toThrow(
70+
'Function task should return a string or an array of strings'
71+
)
72+
}
73+
)
74+
})

‎test/unit/getSpawnedTasks.spec.js‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ import { exec } from 'tinyexec'
22
import { beforeEach, describe, it, vi } from 'vitest'
33

44
import { getAbortController } from '../../lib/getAbortController.js'
5-
const { getSpawnedTasks } = await import('../../lib/getSpawnedTasks.js')
5+
const { getSpawnedTasks, getMaxArgLength } = await import('../../lib/getSpawnedTasks.js')
66

77
vi.mock('tinyexec', () => ({
88
exec: vi.fn().mockReturnValue({
@@ -187,3 +187,14 @@ describe('getSpawnedTasks', () => {
187187
expect(files).toEqual(['test.js'])
188188
})
189189
})
190+
191+
describe('getMaxArgLength', () => {
192+
it.for([
193+
['darwin', 262_144],
194+
['win32', 8_191],
195+
['linux', 131_072],
196+
['foobar', 131_072],
197+
])('should return $2 for $1', ([platform, expected], { expect }) => {
198+
expect(getMaxArgLength(platform)).toEqual(expected)
199+
})
200+
})

0 commit comments

Comments
 (0)