Skip to content

Commit c24a015

Browse files
committed
test(android): name the crash in failed-step evidence
An Android step that ends on the launcher is ambiguous in CI. The `wait for Catalog` failure on the Expo 58 bump uploaded a two-line daemon log and a snapshot showing Play Store, Gmail, and the fixture icon, which reads the same whether the process crashed or navigated somewhere unexpected. The artifact carries no logcat, so the only way to tell them apart was to bisect majors across several CI rounds. Adds three probes to the platform-owned device evidence: `pidof` for the fixture app, the resumed activity, and the crash buffer. A dead pid next to a launcher `mResumedActivity` is a crash, and the crash buffer then names the library in the same run. The resumed-activity selector keeps the ActivityRecord hash with the component name, so the same hash on a new component is a navigation and a new hash is a restart. The adb read moves out of the harness into `device-evidence.ts`, where the seam is the query itself: tests replay a device instead of needing an emulator. The group bound rises with the probe count, because the bound protects the screenshot and losing it whole would hide the rotation facts the existing probes exist to carry.
1 parent d688926 commit c24a015

4 files changed

Lines changed: 224 additions & 73 deletions

File tree

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
import assert from 'node:assert/strict';
2+
import test from 'node:test';
3+
4+
import { readAndroidDeviceEvidence, type AndroidAdbRead } from './device-evidence.ts';
5+
6+
const TARGET = { appId: 'com.callstack.agentdevicelab', serial: 'emulator-5554' };
7+
8+
type FakeDevice = {
9+
crashBuffer?: string;
10+
pid?: string;
11+
resumedActivity?: string;
12+
};
13+
14+
function replayDevice(device: FakeDevice): AndroidAdbRead {
15+
return async (args) => {
16+
const query = args.join(' ');
17+
if (query.includes('pidof')) return device.pid ?? '';
18+
if (query.includes('dumpsys activity')) return device.resumedActivity ?? '';
19+
if (query.includes('-b crash')) return device.crashBuffer ?? '';
20+
if (query.includes('dumpsys display')) return ' mCurrentOrientation=0\n mRotation=0\n';
21+
if (query.includes('logcat')) return '';
22+
return '0';
23+
};
24+
}
25+
26+
test('a dropped process with a launcher activity reads as a crash and names the library', async () => {
27+
const evidence = await readAndroidDeviceEvidence(
28+
TARGET,
29+
replayDevice({
30+
pid: '',
31+
resumedActivity: [
32+
' * Task{b3f2a1 #33 type=standard A=10247:com.google.android.apps.nexuslauncher}',
33+
' topResumedActivity=ActivityRecord{7c1d9e2 u0 com.google.android.apps.nexuslauncher/.NexusLauncherActivity}',
34+
' mFocusedApp=ActivityRecord{7c1d9e2 u0 com.google.android.apps.nexuslauncher/.NexusLauncherActivity}',
35+
].join('\n'),
36+
crashBuffer: [
37+
'E/AndroidRuntime(12345): FATAL EXCEPTION: mqt_native_modules',
38+
'E/AndroidRuntime(12345): Process: com.callstack.agentdevicelab, PID: 12345',
39+
'E/AndroidRuntime(12345): java.lang.UnsatisfiedLinkError: dlopen failed: libgesturehandler.so',
40+
'E/AndroidRuntime(12345): \tat com.swmansion.gesturehandler.RNGestureHandlerModule.<clinit>(RNGestureHandlerModule.kt:14)',
41+
].join('\n'),
42+
}),
43+
);
44+
45+
const appProcess = section(evidence, 'app process');
46+
assert.equal(appProcess, '', 'a dead process must read as an empty pid, not as noise');
47+
assert.match(section(evidence, 'resumed activity'), /nexuslauncher\/\.NexusLauncherActivity/);
48+
assert.match(section(evidence, 'crash buffer'), /FATAL EXCEPTION: mqt_native_modules/);
49+
assert.match(section(evidence, 'crash buffer'), /RNGestureHandlerModule/);
50+
});
51+
52+
test('an alive process keeps its record hash so a navigation is not mistaken for a restart', async () => {
53+
const evidence = await readAndroidDeviceEvidence(
54+
TARGET,
55+
replayDevice({
56+
pid: '12345',
57+
resumedActivity: [
58+
' * Task{a1b2c3 #41 type=standard A=10248:com.callstack.agentdevicelab}',
59+
' mResumedActivity: ActivityRecord{5f6a7b8 u0 com.callstack.agentdevicelab/.MainActivity t41}',
60+
].join('\n'),
61+
}),
62+
);
63+
64+
const activity = section(evidence, 'resumed activity');
65+
assert.equal(section(evidence, 'app process'), '12345');
66+
assert.match(
67+
activity,
68+
/ActivityRecord\{5f6a7b8 u0 com\.callstack\.agentdevicelab\/\.MainActivity/,
69+
);
70+
assert.ok(!activity.includes('* Task{'), 'task rows carry no focus information');
71+
});
72+
73+
test('one failing probe records its failure without taking the rest of the document down', async () => {
74+
const read: AndroidAdbRead = async (args) => {
75+
if (args.join(' ').includes('-b crash')) throw new Error('adb: device offline');
76+
return '0';
77+
};
78+
79+
const evidence = await readAndroidDeviceEvidence(TARGET, read);
80+
81+
assert.match(section(evidence, 'crash buffer'), /\(failed: adb: device offline\)/);
82+
assert.equal(section(evidence, 'user_rotation'), '0');
83+
assert.match(section(evidence, 'app process'), /^0$/);
84+
});
85+
86+
function section(evidence: string, title: string): string {
87+
const match = new RegExp(`## ${title}\\n([\\s\\S]*?)(?=\\n## |$)`).exec(evidence);
88+
assert.ok(match, `evidence is missing the "${title}" section`);
89+
// Sections are joined with a blank line, so the capture always keeps the separator's newline.
90+
return match[1]!.replace(/\n$/, '');
91+
}
Lines changed: 120 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,120 @@
1+
/**
2+
* What the OS says about an Android device when an e2e step fails. Read through adb, not
3+
* agent-device, so the facts still stand when the CLI path is the thing that failed.
4+
*/
5+
import { execFile } from 'node:child_process';
6+
import { promisify } from 'node:util';
7+
8+
const execFileAsync = promisify(execFile);
9+
10+
const DEVICE_PROBE_TIMEOUT_MS = 5_000;
11+
/** Lines read back from logcat; the rotation decisions of the last few minutes fit comfortably. */
12+
const ROTATION_LOG_TAIL = 4_000;
13+
const ROTATION_LOG_LINES = 60;
14+
const ROTATION_LOG_LINE_LENGTH = 240;
15+
/** The crash buffer holds only recent crashes, so a tail is enough to carry a stack. */
16+
const CRASH_LOG_TAIL = 400;
17+
const RESUMED_ACTIVITY_LINES = 12;
18+
19+
/** The device a failed step ran against. */
20+
export type AndroidDeviceEvidenceTarget = {
21+
appId: string;
22+
serial: string;
23+
};
24+
25+
/** One adb query answered with its stdout. Tests replay a device through this. */
26+
export type AndroidAdbRead = (args: readonly string[]) => Promise<string>;
27+
28+
/**
29+
* Rotation (the two settings `orientation` writes, the display's current rotation, and the
30+
* WindowManager decisions logcat still holds) plus whether the app is still running at all. Those
31+
* last three probes are what separates "the app crashed" from "the app navigated somewhere
32+
* unexpected": a `pidof` that returns nothing next to a launcher `mResumedActivity` is a crash, and
33+
* the crash buffer then names the library. The caller bounds this read so it never delays the
34+
* screenshot that follows.
35+
*/
36+
export async function readAndroidDeviceEvidence(
37+
target: AndroidDeviceEvidenceTarget,
38+
read: AndroidAdbRead = createAdbRead(target.serial),
39+
): Promise<string> {
40+
const probes: readonly [string, readonly string[]][] = [
41+
['accelerometer_rotation', ['shell', 'settings', 'get', 'system', 'accelerometer_rotation']],
42+
['user_rotation', ['shell', 'settings', 'get', 'system', 'user_rotation']],
43+
['display rotation', ['shell', 'dumpsys', 'display']],
44+
// The tail, not the whole buffer: the emulator keeps 2MB and a loaded one takes seconds to
45+
// dump it, which is what the group bound is for, not this read.
46+
['logcat rotation decisions', ['logcat', '-d', '-v', 'time', '-t', String(ROTATION_LOG_TAIL)]],
47+
['app process', ['shell', 'pidof', target.appId]],
48+
['resumed activity', ['shell', 'dumpsys', 'activity', 'activities']],
49+
['crash buffer', ['logcat', '-d', '-b', 'crash', '-v', 'time']],
50+
];
51+
const sections: string[] = [];
52+
for (const [title, args] of probes) {
53+
try {
54+
sections.push(`## ${title}\n${selectDeviceEvidenceLines(title, await read(args))}`);
55+
} catch (error) {
56+
sections.push(
57+
`## ${title}\n(failed: ${error instanceof Error ? error.message : String(error)})`,
58+
);
59+
}
60+
}
61+
return `${sections.join('\n\n')}\n`;
62+
}
63+
64+
function createAdbRead(serial: string): AndroidAdbRead {
65+
return async (args) => {
66+
const { stdout } = await execFileAsync('adb', ['-s', serial, ...args], {
67+
maxBuffer: 64 * 1024 * 1024,
68+
timeout: DEVICE_PROBE_TIMEOUT_MS,
69+
});
70+
return stdout;
71+
};
72+
}
73+
74+
function selectDeviceEvidenceLines(title: string, output: string): string {
75+
if (title === 'display rotation') {
76+
return output
77+
.split('\n')
78+
.filter((line) =>
79+
/mCurrentOrientation|mRotation=|installOrientation|\brotation \d/.test(line),
80+
)
81+
.map((line) => line.trim().slice(0, ROTATION_LOG_LINE_LENGTH))
82+
.slice(0, 8)
83+
.join('\n');
84+
}
85+
if (title === 'logcat rotation decisions') {
86+
return output
87+
.split('\n')
88+
.filter(
89+
(line) =>
90+
/(WindowManager|DisplayRotation|WindowOrientationListener|RotationResolver|DisplayContent|SensorService)/.test(
91+
line,
92+
) && /rotat|orient/i.test(line),
93+
)
94+
.slice(-ROTATION_LOG_LINES)
95+
.map((line) => line.slice(0, ROTATION_LOG_LINE_LENGTH))
96+
.join('\n');
97+
}
98+
if (title === 'resumed activity') {
99+
// Keeps the ActivityRecord hash with the component name: the same hash on a different
100+
// component is a navigation, a new hash is a process restart.
101+
return output
102+
.split('\n')
103+
.filter((line) =>
104+
/mResumedActivity|topResumedActivity|mFocusedApp|mStoppingActivity|mPausingActivity/.test(
105+
line,
106+
),
107+
)
108+
.slice(0, RESUMED_ACTIVITY_LINES)
109+
.map((line) => line.trim().slice(0, ROTATION_LOG_LINE_LENGTH))
110+
.join('\n');
111+
}
112+
if (title === 'crash buffer') {
113+
return output
114+
.split('\n')
115+
.slice(-CRASH_LOG_TAIL)
116+
.map((line) => line.slice(0, ROTATION_LOG_LINE_LENGTH))
117+
.join('\n');
118+
}
119+
return output.trim();
120+
}

‎test/integration/android-emulator-e2e/live-harness.ts‎

Lines changed: 5 additions & 70 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,5 @@
1-
import { execFile } from 'node:child_process';
21
import fs from 'node:fs';
32
import path from 'node:path';
4-
import { promisify } from 'node:util';
53

64
import {
75
createLiveDeviceContext,
@@ -11,6 +9,7 @@ import {
119
} from '../live-device-e2e/runtime.ts';
1210
import type { AndroidEmulatorBehaviorId } from './behavior-coverage.ts';
1311
import { liveCommandsForScenario } from './coverage.ts';
12+
import { readAndroidDeviceEvidence } from './device-evidence.ts';
1413
import { liveBehaviorsForScenario, writeCoverageReport } from './live-coverage-report.ts';
1514

1615
export { assertCoverageComplete, writeCoverageReport } from './live-coverage-report.ts';
@@ -39,77 +38,13 @@ export function createContext(): LiveContext {
3938
};
4039
}
4140

42-
const execFileAsync = promisify(execFile);
43-
44-
const ROTATION_PROBE_TIMEOUT_MS = 5_000;
45-
/** Lines read back from logcat; the rotation decisions of the last few minutes fit comfortably. */
46-
const ROTATION_LOG_TAIL = 4_000;
47-
const ROTATION_LOG_LINES = 60;
48-
const ROTATION_LOG_LINE_LENGTH = 240;
49-
50-
/**
51-
* What the OS says about rotation when a step fails: the two settings `orientation` writes, the
52-
* display's current rotation, and the WindowManager rotation decisions logcat still holds (with
53-
* the reason it gives). Read through adb, not agent-device, so it stands even when the CLI path
54-
* is what failed; the shared collector bounds the whole read so it never delays the screenshot.
55-
*/
56-
async function readAndroidRotationEvidence(context: LiveContext): Promise<string> {
57-
const probes: readonly [string, string[]][] = [
58-
['accelerometer_rotation', ['shell', 'settings', 'get', 'system', 'accelerometer_rotation']],
59-
['user_rotation', ['shell', 'settings', 'get', 'system', 'user_rotation']],
60-
['display rotation', ['shell', 'dumpsys', 'display']],
61-
// The tail, not the whole buffer: the emulator keeps 2MB and a loaded one takes seconds to
62-
// dump it, which is what the group bound is for, not this read.
63-
['logcat rotation decisions', ['logcat', '-d', '-v', 'time', '-t', String(ROTATION_LOG_TAIL)]],
64-
];
65-
const sections: string[] = [];
66-
for (const [title, args] of probes) {
67-
try {
68-
const { stdout } = await execFileAsync('adb', ['-s', context.serial, ...args], {
69-
maxBuffer: 64 * 1024 * 1024,
70-
timeout: ROTATION_PROBE_TIMEOUT_MS,
71-
});
72-
sections.push(`## ${title}\n${selectRotationLines(title, stdout)}`);
73-
} catch (error) {
74-
sections.push(
75-
`## ${title}\n(failed: ${error instanceof Error ? error.message : String(error)})`,
76-
);
77-
}
78-
}
79-
return `${sections.join('\n\n')}\n`;
80-
}
81-
82-
function selectRotationLines(title: string, output: string): string {
83-
if (title === 'display rotation') {
84-
return output
85-
.split('\n')
86-
.filter((line) =>
87-
/mCurrentOrientation|mRotation=|installOrientation|\brotation \d/.test(line),
88-
)
89-
.map((line) => line.trim().slice(0, ROTATION_LOG_LINE_LENGTH))
90-
.slice(0, 8)
91-
.join('\n');
92-
}
93-
if (title === 'logcat rotation decisions') {
94-
return output
95-
.split('\n')
96-
.filter(
97-
(line) =>
98-
/(WindowManager|DisplayRotation|WindowOrientationListener|RotationResolver|DisplayContent|SensorService)/.test(
99-
line,
100-
) && /rotat|orient/i.test(line),
101-
)
102-
.slice(-ROTATION_LOG_LINES)
103-
.map((line) => line.slice(0, ROTATION_LOG_LINE_LENGTH))
104-
.join('\n');
105-
}
106-
return output.trim();
107-
}
108-
10941
const harness = createLiveDeviceHarness<LiveContext, AndroidEmulatorBehaviorId>({
11042
behaviorsForScenario: liveBehaviorsForScenario,
11143
commandsForScenario: liveCommandsForScenario,
112-
deviceEvidence: readAndroidRotationEvidence,
44+
deviceEvidence: (context) => readAndroidDeviceEvidence(context),
45+
// Seven adb reads, two of them dumpsys: the default bound would drop the whole file,
46+
// including the crash stack, on a loaded runner.
47+
deviceEvidenceTimeoutMs: 25_000,
11348
commonFlags: (context, args) => [
11449
...args,
11550
'--platform',

‎test/integration/live-device-e2e/runtime.ts‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -51,11 +51,13 @@ type HarnessOptions<Context, BehaviorId extends string> = {
5151
options?: { timeoutMs?: number },
5252
) => Promise<CliJsonResult>;
5353
/**
54-
* Platform-owned device facts for a failed step (rotation state, system logs), read outside
55-
* agent-device so they describe the device even when the CLI path is what failed. Best-effort:
56-
* a throw or undefined records nothing.
54+
* Platform-owned device facts for a failed step (rotation state, whether the app process is
55+
* alive, crash logs), read outside agent-device so they describe the device even when the CLI
56+
* path is what failed. Best-effort: a throw or undefined records nothing.
5757
*/
5858
deviceEvidence?: (context: Context) => Promise<string | undefined>;
59+
/** Bound for `deviceEvidence` as a group. A platform that adds probes raises this with them. */
60+
deviceEvidenceTimeoutMs?: number;
5961
writeCoverageReport: (context: Context) => void;
6062
};
6163

@@ -200,6 +202,9 @@ export function createLiveDeviceHarness<
200202
stem: path.join(context.artifactDir, `failed-step-${context.stepHistory.length}`),
201203
runCli: (args) => runCli(options.commonFlags(context, args), context.env),
202204
...(deviceEvidence ? { deviceEvidence: () => deviceEvidence(context) } : {}),
205+
...(options.deviceEvidenceTimeoutMs === undefined
206+
? {}
207+
: { deviceEvidenceTimeoutMs: options.deviceEvidenceTimeoutMs }),
203208
});
204209
}
205210

0 commit comments

Comments
 (0)