Skip to content

Commit 34e1782

Browse files
Backport PR #19126 on branch 4.6.x (Fix spurious 3s kernel restart delay) (#19160)
Backport PR #19126: Fix spurious 3s kernel restart delay Co-authored-by: Afshin Taylor Darian <[email protected]>
1 parent 57e5afd commit 34e1782

2 files changed

Lines changed: 132 additions & 18 deletions

File tree

‎packages/services/src/kernel/default.ts‎

Lines changed: 24 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,6 @@ import { PageConfig } from '@jupyterlab/coreutils';
3232
declare let requirejs: any;
3333

3434
export const DEFAULT_KERNEL_INFO_TIMEOUT = 3000;
35-
const RESTARTING_KERNEL_SESSION = '_RESTARTING_';
3635
const STARTING_KERNEL_SESSION = '';
3736

3837
/**
@@ -473,11 +472,9 @@ export class KernelConnection implements Kernel.IKernelConnection {
473472
// If we have a kernel_info_request and we are starting or restarting, send the
474473
// kernel_info_request immediately if we can, and if not throw an error so
475474
// we can retry later. On restarting we do this because we must get at least one message
476-
// from the kernel to reset the kernel session (thus clearing the restart
477-
// status sentinel).
475+
// from the kernel before sending pending messages.
478476
if (
479-
(this._kernelSession === STARTING_KERNEL_SESSION ||
480-
this._kernelSession === RESTARTING_KERNEL_SESSION) &&
477+
(this._kernelSession === STARTING_KERNEL_SESSION || this._isRestarting) &&
481478
KernelMessage.isInfoRequestMsg(msg)
482479
) {
483480
if (this.connectionStatus === 'connected') {
@@ -497,10 +494,7 @@ export class KernelConnection implements Kernel.IKernelConnection {
497494
}
498495

499496
// Send if the ws allows it, otherwise queue the message.
500-
if (
501-
this.connectionStatus === 'connected' &&
502-
this._kernelSession !== RESTARTING_KERNEL_SESSION
503-
) {
497+
if (this.connectionStatus === 'connected' && !this._isRestarting) {
504498
this._ws!.send(
505499
this.serverSettings.serializer.serialize(msg, this._ws!.protocol)
506500
);
@@ -556,11 +550,16 @@ export class KernelConnection implements Kernel.IKernelConnection {
556550
}
557551
this._updateStatus('restarting');
558552
this._clearKernelState();
559-
this._kernelSession = RESTARTING_KERNEL_SESSION;
560-
await this._kernelAPIClient.restart(this.id);
561-
// Reconnect to the kernel to address cases where kernel ports
562-
// have changed during the restart.
563-
await this.reconnect();
553+
this._isRestarting = true;
554+
try {
555+
await this._kernelAPIClient.restart(this.id);
556+
// Reconnect to the kernel to address cases where kernel ports
557+
// have changed during the restart.
558+
await this.reconnect();
559+
} catch (error) {
560+
this._isRestarting = false;
561+
throw error;
562+
}
564563
this.hasPendingInput = false;
565564
}
566565

@@ -1289,7 +1288,7 @@ export class KernelConnection implements Kernel.IKernelConnection {
12891288
// stop sending messages.
12901289
while (
12911290
this.connectionStatus === 'connected' &&
1292-
this._kernelSession !== RESTARTING_KERNEL_SESSION &&
1291+
!this._isRestarting &&
12931292
this._pendingMessages.length > 0
12941293
) {
12951294
this._sendMessage(this._pendingMessages[0], false);
@@ -1549,7 +1548,7 @@ export class KernelConnection implements Kernel.IKernelConnection {
15491548

15501549
if (this.status !== 'dead') {
15511550
if (connectionStatus === 'connected') {
1552-
let restarting = this._kernelSession === RESTARTING_KERNEL_SESSION;
1551+
let restarting = this._isRestarting;
15531552

15541553
// Send a kernel info request to make sure we send at least one
15551554
// message to get kernel status back. Always request kernel info
@@ -1567,20 +1566,26 @@ export class KernelConnection implements Kernel.IKernelConnection {
15671566
return;
15681567
}
15691568
sendPendingCalled = true;
1570-
if (restarting && this._kernelSession === RESTARTING_KERNEL_SESSION) {
1569+
if (restarting && this._isRestarting) {
15711570
// We were restarting and a message didn't arrive to set the
15721571
// session, but we just assume the restart succeeded and send any
15731572
// pending messages.
15741573

15751574
// FIXME: it would be better to retry the kernel_info_request here
1575+
this._isRestarting = false;
15761576
this._kernelSession = '';
15771577
}
15781578
clearTimeout(timeoutHandle);
15791579
if (this._pendingMessages.length > 0) {
15801580
this._sendPending();
15811581
}
15821582
};
1583-
void p.then(sendPendingOnce);
1583+
void p.then(() => {
1584+
if (restarting) {
1585+
this._isRestarting = false;
1586+
}
1587+
sendPendingOnce();
1588+
});
15841589
// FIXME: if sent while zmq subscriptions are not established,
15851590
// kernelInfo may not resolve, so use a timeout to ensure we don't hang forever.
15861591
// It may be preferable to retry kernelInfo rather than give up after one timeout.
@@ -1842,6 +1847,7 @@ export class KernelConnection implements Kernel.IKernelConnection {
18421847
private _status: KernelMessage.Status = 'unknown';
18431848
private _connectionStatus: Kernel.ConnectionStatus = 'connecting';
18441849
private _kernelSession = '';
1850+
private _isRestarting = false;
18451851
private _clientId: string;
18461852
private _isDisposed = false;
18471853
/**

‎packages/services/test/kernel/ikernel.spec.ts‎

Lines changed: 108 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,10 +4,64 @@
44
import { PageConfig } from '@jupyterlab/coreutils';
55
import { JupyterServer, testEmission } from '@jupyterlab/testing';
66
import { PromiseDelegate, UUID } from '@lumino/coreutils';
7+
import type WebSocket from 'ws';
78
import type { Kernel, KernelManager, KernelSpec } from '../../src';
89
import { KernelMessage, KernelSpecAPI } from '../../src';
10+
import { deserialize } from '../../src/kernel/serialize';
911
import { FakeKernelManager, handleRequest, KernelTester } from '../utils';
1012

13+
class RequestOrderKernelTester extends KernelTester {
14+
requestOrder: string[] = [];
15+
16+
waitForNextRequest(): Promise<string> {
17+
const delegate = new PromiseDelegate<string>();
18+
this._nextRequestWaiters.push(delegate);
19+
return delegate.promise;
20+
}
21+
22+
waitForRequestType(msgType: string): Promise<void> {
23+
if (this.requestOrder.includes(msgType)) {
24+
return Promise.resolve();
25+
}
26+
const delegate = new PromiseDelegate<void>();
27+
this._requestTypeWaiters.push({ msgType, delegate });
28+
return delegate.promise;
29+
}
30+
31+
protected onSocket(sock: WebSocket): void {
32+
super.onSocket(sock);
33+
sock.on('message', (msg: any) => {
34+
if (msg instanceof Buffer) {
35+
msg = new Uint8Array(msg).buffer;
36+
}
37+
const data = deserialize(
38+
msg,
39+
KernelMessage.supportedKernelWebSocketProtocols
40+
.v1KernelWebsocketJupyterOrg
41+
);
42+
const msgType = data.header.msg_type;
43+
this.requestOrder.push(msgType);
44+
45+
for (const delegate of this._nextRequestWaiters.splice(0)) {
46+
delegate.resolve(msgType);
47+
}
48+
this._requestTypeWaiters = this._requestTypeWaiters.filter(waiter => {
49+
if (waiter.msgType === msgType) {
50+
waiter.delegate.resolve();
51+
return false;
52+
}
53+
return true;
54+
});
55+
});
56+
}
57+
58+
private _nextRequestWaiters: PromiseDelegate<string>[] = [];
59+
private _requestTypeWaiters: Array<{
60+
msgType: string;
61+
delegate: PromiseDelegate<void>;
62+
}> = [];
63+
}
64+
1165
describe('Kernel.IKernel', () => {
1266
let defaultKernel: Kernel.IKernelConnection;
1367
let specs: KernelSpec.ISpecModels;
@@ -691,6 +745,60 @@ describe('Kernel.IKernel', () => {
691745
expect(comm.isDisposed).toBe(true);
692746
await kernel.shutdown();
693747
});
748+
749+
it('should prioritize kernel info after late messages during restart', async () => {
750+
const tester = new RequestOrderKernelTester();
751+
const kernel = await tester.start();
752+
let future:
753+
| Kernel.IShellFuture<
754+
KernelMessage.IExecuteRequestMsg,
755+
KernelMessage.IExecuteReplyMsg
756+
>
757+
| undefined;
758+
759+
try {
760+
(kernel as any)._kernelInfoTimeout = 100;
761+
tester.requestOrder = [];
762+
763+
const restartResponse = new PromiseDelegate<Response>();
764+
const oldFetch = tester.serverSettings.fetch;
765+
(tester.serverSettings as any).fetch = () => {
766+
(tester.serverSettings as any).fetch = oldFetch;
767+
return restartResponse.promise;
768+
};
769+
770+
const firstRequest = tester.waitForNextRequest();
771+
const restart = kernel.restart();
772+
future = kernel.requestExecute({ code: 'foo' });
773+
774+
const staleMessageId = UUID.uuid4();
775+
const staleMessage = testEmission(kernel.anyMessage, {
776+
find: (_, args) =>
777+
args.direction === 'recv' &&
778+
args.msg.header.msg_id === staleMessageId
779+
});
780+
tester.sendStatus(staleMessageId, 'idle');
781+
await staleMessage;
782+
783+
restartResponse.resolve(
784+
new Response(JSON.stringify({ id: kernel.id, name: kernel.name }), {
785+
status: 200
786+
})
787+
);
788+
789+
expect(await firstRequest).toBe('kernel_info_request');
790+
await restart;
791+
await tester.waitForRequestType('execute_request');
792+
expect(tester.requestOrder.slice(0, 2)).toEqual([
793+
'kernel_info_request',
794+
'execute_request'
795+
]);
796+
} finally {
797+
future?.dispose();
798+
await tester.shutdown();
799+
tester.dispose();
800+
}
801+
});
694802
});
695803

696804
describe('#reconnect()', () => {

0 commit comments

Comments
 (0)