Skip to content

Commit 8004763

Browse files
author
Christian Budde Christensen
committed
feat: Do not fail on empty test suite
Add option to configuration-file and cli-args to server (single run) and runner, for disabling failure on empty test suite. Will instead display a warning. Closes #926
1 parent 7f10d81 commit 8004763

7 files changed

Lines changed: 95 additions & 14 deletions

File tree

‎lib/cli.js‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,10 @@ var processArgs = function (argv, options, fs, path) {
3737
options.colors = options.colors === 'true'
3838
}
3939

40+
if (helper.isString(options.failOnEmptyTestSuite)) {
41+
options.failOnEmptyTestSuite = options.failOnEmptyTestSuite === 'true'
42+
}
43+
4044
if (helper.isString(options.logLevel)) {
4145
var logConstant = constant['LOG_' + options.logLevel.toUpperCase()]
4246
if (helper.isDefined(logConstant)) {
@@ -165,6 +169,8 @@ var describeStart = function () {
165169
.describe('single-run', 'Run the test when browsers captured and exit.')
166170
.describe('no-single-run', 'Disable single-run.')
167171
.describe('report-slower-than', '<integer> Report tests that are slower than given time [ms].')
172+
.describe('fail-on-empty-test-suite', 'Fail on empty test suite.')
173+
.describe('no-fail-on-empty-test-suite', 'Do not fail on empty test suite.')
168174
.describe('help', 'Print usage and options.')
169175
}
170176

@@ -176,6 +182,8 @@ var describeRun = function () {
176182
' $0 run [<configFile>] [<options>] [ -- <clientArgs>]')
177183
.describe('port', '<integer> Port where the server is listening.')
178184
.describe('no-refresh', 'Do not re-glob all the patterns.')
185+
.describe('fail-on-empty-test-suite', 'Fail on empty test suite.')
186+
.describe('no-fail-on-empty-test-suite', 'Do not fail on empty test suite.')
179187
.describe('help', 'Print usage.')
180188
}
181189

‎lib/config.js‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -260,6 +260,7 @@ var Config = function () {
260260
this.browserDisconnectTolerance = 0
261261
this.browserNoActivityTimeout = 10000
262262
this.concurrency = Infinity
263+
this.failOnEmptyTestSuite = true
263264
}
264265

265266
var CONFIG_SYNTAX_HELP = ' module.exports = function(config) {\n' +

‎lib/middleware/runner.js‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,8 @@ var createRunnerMiddleware = function (emitter, fileList, capturedBrowsers, repo
4141
// clean up, close runner response
4242
emitter.once('run_complete', function (browsers, results) {
4343
reporter.removeAdapter(responseWrite)
44-
response.end(constant.EXIT_CODE + results.exitCode)
44+
var emptyTestSuite = (results.failed + results.success) === 0 ? 0 : 1
45+
response.end(constant.EXIT_CODE + emptyTestSuite + results.exitCode)
4546
})
4647
})
4748

‎lib/runner.js‎

Lines changed: 14 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -3,9 +3,11 @@ var http = require('http')
33
var constant = require('./constants')
44
var helper = require('./helper')
55
var cfg = require('./config')
6+
var logger = require('./logger')
7+
var log = logger.create('runner')
68

7-
var parseExitCode = function (buffer, defaultCode) {
8-
var tailPos = buffer.length - Buffer.byteLength(constant.EXIT_CODE) - 1
9+
var parseExitCode = function (buffer, defaultCode, failOnEmptyTestSuite) {
10+
var tailPos = buffer.length - Buffer.byteLength(constant.EXIT_CODE) - 2
911

1012
if (tailPos < 0) {
1113
return defaultCode
@@ -14,9 +16,15 @@ var parseExitCode = function (buffer, defaultCode) {
1416
// tail buffer which might contain the message
1517
var tail = buffer.slice(tailPos)
1618
var tailStr = tail.toString()
17-
if (tailStr.substr(0, tailStr.length - 1) === constant.EXIT_CODE) {
19+
if (tailStr.substr(0, tailStr.length - 2) === constant.EXIT_CODE) {
1820
tail.fill('\x00')
19-
return parseInt(tailStr.substr(-1), 10)
21+
var emptyInt = parseInt(tailStr.substr(-2, 1), 10)
22+
var exitCode = parseInt(tailStr.substr(-1), 10)
23+
if (failOnEmptyTestSuite === false && emptyInt === 0) {
24+
log.warn('Test suite was empty.')
25+
return 0
26+
}
27+
return exitCode
2028
}
2129

2230
return defaultCode
@@ -40,7 +48,7 @@ exports.run = function (config, done) {
4048

4149
var request = http.request(options, function (response) {
4250
response.on('data', function (buffer) {
43-
exitCode = parseExitCode(buffer, exitCode)
51+
exitCode = parseExitCode(buffer, exitCode, config.failOnEmptyTestSuite)
4452
process.stdout.write(buffer)
4553
})
4654

@@ -51,7 +59,7 @@ exports.run = function (config, done) {
5159

5260
request.on('error', function (e) {
5361
if (e.code === 'ECONNREFUSED') {
54-
console.error('There is no server listening on port %d', options.port)
62+
log.error('There is no server listening on port %d', options.port)
5563
done(1, e.code)
5664
} else {
5765
throw e

‎lib/server.js‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -251,8 +251,10 @@ Server.prototype._start = function (config, launcher, preprocess, fileList, webS
251251
var results = singleRunBrowsers.getResults()
252252
if (singleRunBrowserNotCaptured) {
253253
results.exitCode = 1
254+
} else if (results.success + results.failed === 0 && !config.failOnEmptyTestSuite) {
255+
results.exitCode = 0
256+
self.log.warn('Test suite was empty.')
254257
}
255-
256258
self.emit('run_complete', singleRunBrowsers, results)
257259
}
258260
}

‎test/unit/middleware/runner.spec.js‎

Lines changed: 49 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,7 @@ describe('middleware.runner', () => {
7373

7474
response.once('end', () => {
7575
expect(nextSpy).to.not.have.been.called
76-
expect(response).to.beServedAs(200, 'result\x1FEXIT0')
76+
expect(response).to.beServedAs(200, 'result\x1FEXIT10')
7777
done()
7878
})
7979

@@ -83,6 +83,54 @@ describe('middleware.runner', () => {
8383
emitter.emit('run_complete', capturedBrowsers, {exitCode: 0})
8484
})
8585

86+
it('should set the empty to 0 if empty results', (done) => {
87+
capturedBrowsers.add(new Browser())
88+
sinon.stub(capturedBrowsers, 'areAllReady', () => true)
89+
90+
response.once('end', () => {
91+
expect(nextSpy).to.not.have.been.called
92+
expect(response).to.beServedAs(200, 'result\x1FEXIT00')
93+
done()
94+
})
95+
96+
handler(new HttpRequestMock('/__run__'), response, nextSpy)
97+
98+
mockReporter.write('result')
99+
emitter.emit('run_complete', capturedBrowsers, {exitCode: 0, success: 0, failed: 0})
100+
})
101+
102+
it('should set the empty to 1 if successful tests', (done) => {
103+
capturedBrowsers.add(new Browser())
104+
sinon.stub(capturedBrowsers, 'areAllReady', () => true)
105+
106+
response.once('end', () => {
107+
expect(nextSpy).to.not.have.been.called
108+
expect(response).to.beServedAs(200, 'result\x1FEXIT10')
109+
done()
110+
})
111+
112+
handler(new HttpRequestMock('/__run__'), response, nextSpy)
113+
114+
mockReporter.write('result')
115+
emitter.emit('run_complete', capturedBrowsers, {exitCode: 0, success: 3, failed: 0})
116+
})
117+
118+
it('should set the empty to 1 if failed tests', (done) => {
119+
capturedBrowsers.add(new Browser())
120+
sinon.stub(capturedBrowsers, 'areAllReady', () => true)
121+
122+
response.once('end', () => {
123+
expect(nextSpy).to.not.have.been.called
124+
expect(response).to.beServedAs(200, 'result\x1FEXIT10')
125+
done()
126+
})
127+
128+
handler(new HttpRequestMock('/__run__'), response, nextSpy)
129+
130+
mockReporter.write('result')
131+
emitter.emit('run_complete', capturedBrowsers, {exitCode: 0, success: 0, failed: 6})
132+
})
133+
86134
it('should not run if there is no browser captured', (done) => {
87135
sinon.stub(fileListMock, 'refresh')
88136

‎test/unit/runner.spec.js‎

Lines changed: 18 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -12,14 +12,14 @@ describe('runner', () => {
1212
var EXIT = constant.EXIT_CODE
1313

1414
it('should return 0 exit code if present in the buffer', () => {
15-
expect(m.parseExitCode(new Buffer(`something\nfake${EXIT}0`))).to.equal(0)
15+
expect(m.parseExitCode(new Buffer(`something\nfake${EXIT}10`))).to.equal(0)
1616
})
1717

1818
it('should null the exit code part of the buffer', () => {
19-
var buffer = new Buffer(`some${EXIT}1`)
19+
var buffer = new Buffer(`some${EXIT}01`)
2020
m.parseExitCode(buffer)
2121

22-
expect(buffer.toString()).to.equal('some\0\0\0\0\0\0')
22+
expect(buffer.toString()).to.equal('some\0\0\0\0\0\0\0')
2323
})
2424

2525
it('should not touch buffer without exit code and return default', () => {
@@ -41,8 +41,21 @@ describe('runner', () => {
4141
})
4242

4343
it('should parse any single digit exit code', () => {
44-
expect(m.parseExitCode(new Buffer(`something\nfake${EXIT}1`))).to.equal(1)
45-
expect(m.parseExitCode(new Buffer(`something\nfake${EXIT}7`))).to.equal(7)
44+
expect(m.parseExitCode(new Buffer(`something\nfake${EXIT}01`))).to.equal(1)
45+
expect(m.parseExitCode(new Buffer(`something\nfake${EXIT}17`))).to.equal(7)
46+
})
47+
48+
it('should return exit code 0 if failOnEmptyTestSuite is false and and non-empty int is 0', () => {
49+
expect(m.parseExitCode(new Buffer(`something\nfake${EXIT}01`), undefined, false)).to.equal(0)
50+
})
51+
52+
it('should return exit code if failOnEmptyTestSuite is true', () => {
53+
expect(m.parseExitCode(new Buffer(`something\nfake${EXIT}00`), undefined, true)).to.equal(0)
54+
expect(m.parseExitCode(new Buffer(`something\nfake${EXIT}01`), undefined, true)).to.equal(1)
55+
expect(m.parseExitCode(new Buffer(`something\nfake${EXIT}07`), undefined, true)).to.equal(7)
56+
expect(m.parseExitCode(new Buffer(`something\nfake${EXIT}10`), undefined, true)).to.equal(0)
57+
expect(m.parseExitCode(new Buffer(`something\nfake${EXIT}11`), undefined, true)).to.equal(1)
58+
expect(m.parseExitCode(new Buffer(`something\nfake${EXIT}17`), undefined, true)).to.equal(7)
4659
})
4760
})
4861
})

0 commit comments

Comments
 (0)