Skip to content

Commit cac56d9

Browse files
koala73koala73
committed
fix(seed-bigmac): parallelize the 50-country EXA loop to stop 240s-deadline crashes (#4994)
The Big Mac seeder fetched all 50 countries SEQUENTIALLY under runSeed's 240s fetch-phase deadline. Whenever EXA latency exceeded ~4.8s/country (240s/50) the run breached the deadline and exited 75 — a spurious "Deploy Crashed!" alert every tick, no data lost but 240s of compute burned. Run the country loop with BOUNDED CONCURRENCY (6) via the existing allSettledWithConcurrency helper (now exported from _seed-utils.mjs). Worst case is ceil(50/6)=9 waves x 15s ~= 135s, comfortably under the deadline. Per-country failures already degrade to available:false, so the run now exits 0 with partial data; only a total EXA outage trips the graceful path. (The old 150ms inter-call throttle is dropped — the concurrency cap of 6 is the rate limiter.) Also guards the seeder's top-level execution behind an isMain check so the core is importable, and adds tests/bigmac-seed.test.mjs (concurrency regression + country-order preservation + single-country-failure isolation). Claude-Session: https://claude.ai/code/session_01XKZZy7bzUNTZeJDVWgXbjj
1 parent 240baa0 commit cac56d9

3 files changed

Lines changed: 167 additions & 80 deletions

File tree

‎scripts/_seed-utils.mjs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -128,7 +128,7 @@ export async function fetchCoinPaprikaTickersById(paprikaIds, options = {}) {
128128
return tickers;
129129
}
130130

131-
async function allSettledWithConcurrency(items, concurrency, mapper) {
131+
export async function allSettledWithConcurrency(items, concurrency, mapper) {
132132
const results = new Array(items.length);
133133
let nextIndex = 0;
134134
const workerCount = Math.min(concurrency, items.length);

‎scripts/seed-bigmac.mjs‎

Lines changed: 101 additions & 79 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,10 @@
11
#!/usr/bin/env node
22

3-
import { loadEnvFile, CHROME_UA, runSeed, sleep, readSeedSnapshot, getSharedFxRates, SHARED_FX_FALLBACKS } from './_seed-utils.mjs';
4-
5-
loadEnvFile(import.meta.url);
3+
import { loadEnvFile, CHROME_UA, runSeed, readSeedSnapshot, getSharedFxRates, SHARED_FX_FALLBACKS, allSettledWithConcurrency } from './_seed-utils.mjs';
64

75
const CANONICAL_KEY = 'economic:bigmac:v1';
86
const CACHE_TTL = 864000; // 10 days — weekly seed with 3-day cron-drift buffer
9-
const EXA_DELAY_MS = 150;
7+
const EXA_CONCURRENCY = 6; // in-flight EXA searches; bounds the 50-country loop under runSeed's 240s deadline (see fetchBigMacPrices / #4994)
108

119
const FX_FALLBACKS = SHARED_FX_FALLBACKS;
1210

@@ -18,7 +16,7 @@ const WOW_ANOMALY_THRESHOLD = 20; // % change that signals a data bug
1816
const USD_MIN = 1.50;
1917
const USD_MAX = 12.00;
2018

21-
const COUNTRIES = [
19+
export const COUNTRIES = [
2220
// Americas
2321
{ code: 'US', name: 'United States', currency: 'USD', flag: '🇺🇸' },
2422
{ code: 'CA', name: 'Canada', currency: 'CAD', flag: '🇨🇦' },
@@ -129,71 +127,89 @@ async function searchExa(query, includeDomains = null) {
129127
return resp.json();
130128
}
131129

132-
async function fetchBigMacPrices(prevSnapshot) {
133-
const fxRates = await getSharedFxRates(FX_SYMBOLS, FX_FALLBACKS);
134-
const results = [];
135-
136-
for (const country of COUNTRIES) {
137-
await sleep(EXA_DELAY_MS);
138-
console.log(`\n Processing ${country.flag} ${country.name} (${country.currency})...`);
139-
140-
const fxRate = fxRates[country.currency] ?? FX_FALLBACKS[country.currency] ?? null;
141-
let localPrice = null;
142-
let usdPrice = null;
143-
let sourceSite = '';
144-
145-
try {
146-
// Include currency code in query — helps EXA find per-country specialist pages
147-
const query = `Big Mac price ${country.name} ${country.currency}`;
148-
const SPECIALIST_SITES = ['theburgerindex.com', 'eatmyindex.com'];
149-
150-
// Specialist Big Mac Index sites only — clean, verified per-country data
151-
const exaResult = await searchExa(query, SPECIALIST_SITES);
152-
await sleep(EXA_DELAY_MS);
153-
154-
if (exaResult?.results?.length) {
155-
for (const result of exaResult.results) {
156-
const summary = result?.summary;
157-
if (!summary || typeof summary !== 'string') continue;
158-
const hit = matchPrice(summary, result.url || '');
159-
if (hit?.currency === country.currency) {
160-
localPrice = hit.price;
161-
sourceSite = hit.source;
162-
break;
163-
}
130+
// Resolve one country's Big Mac price. NEVER throws for an upstream/EXA failure —
131+
// a failed lookup yields an `available: false` row so a single flaky country can
132+
// never fail the whole run. Called concurrently (see fetchBigMacPrices).
133+
async function processCountry(country, fxRates, searchExaFn) {
134+
const fxRate = fxRates[country.currency] ?? FX_FALLBACKS[country.currency] ?? null;
135+
let localPrice = null;
136+
let usdPrice = null;
137+
let sourceSite = '';
138+
139+
try {
140+
// Include currency code in query — helps EXA find per-country specialist pages
141+
const query = `Big Mac price ${country.name} ${country.currency}`;
142+
const SPECIALIST_SITES = ['theburgerindex.com', 'eatmyindex.com'];
143+
144+
// Specialist Big Mac Index sites only — clean, verified per-country data
145+
const exaResult = await searchExaFn(query, SPECIALIST_SITES);
146+
147+
if (exaResult?.results?.length) {
148+
for (const result of exaResult.results) {
149+
const summary = result?.summary;
150+
if (!summary || typeof summary !== 'string') continue;
151+
const hit = matchPrice(summary, result.url || '');
152+
if (hit?.currency === country.currency) {
153+
localPrice = hit.price;
154+
sourceSite = hit.source;
155+
break;
164156
}
165157
}
166-
} catch (err) {
167-
console.warn(` [${country.code}] EXA error: ${err.message}`);
168-
}
169-
170-
if (usdPrice === null) {
171-
usdPrice = localPrice !== null && fxRate ? +(localPrice * fxRate).toFixed(4) : null;
172158
}
159+
} catch (err) {
160+
console.warn(` [${country.code}] EXA error: ${err.message}`);
161+
}
173162

174-
// Sanity check: Big Mac USD price must be in a plausible global range
175-
if (usdPrice !== null && (usdPrice < USD_MIN || usdPrice > USD_MAX)) {
176-
console.warn(` [PRICE] ANOMALY ${country.flag} ${country.name}: $${usdPrice} out of range [$${USD_MIN}-$${USD_MAX}] — dropping price`);
177-
usdPrice = null;
178-
localPrice = null;
179-
}
163+
usdPrice = localPrice !== null && fxRate ? +(localPrice * fxRate).toFixed(4) : null;
180164

181-
const status = localPrice !== null ? `${localPrice} ${country.currency} = $${usdPrice}` : 'N/A';
182-
console.log(` Big Mac: ${status}`);
183-
184-
results.push({
185-
code: country.code,
186-
name: country.name,
187-
currency: country.currency,
188-
flag: country.flag,
189-
localPrice: localPrice !== null ? +localPrice.toFixed(4) : null,
190-
usdPrice,
191-
fxRate: fxRate || 0,
192-
sourceSite,
193-
available: usdPrice !== null,
194-
});
165+
// Sanity check: Big Mac USD price must be in a plausible global range
166+
if (usdPrice !== null && (usdPrice < USD_MIN || usdPrice > USD_MAX)) {
167+
console.warn(` [PRICE] ANOMALY ${country.flag} ${country.name}: $${usdPrice} out of range [$${USD_MIN}-$${USD_MAX}] — dropping price`);
168+
usdPrice = null;
169+
localPrice = null;
195170
}
196171

172+
const status = localPrice !== null ? `${localPrice} ${country.currency} = $${usdPrice}` : 'N/A';
173+
console.log(` ${country.flag} ${country.name} (${country.currency}): ${status}`);
174+
175+
return {
176+
code: country.code,
177+
name: country.name,
178+
currency: country.currency,
179+
flag: country.flag,
180+
localPrice: localPrice !== null ? +localPrice.toFixed(4) : null,
181+
usdPrice,
182+
fxRate: fxRate || 0,
183+
sourceSite,
184+
available: usdPrice !== null,
185+
};
186+
}
187+
188+
export async function fetchBigMacPrices(prevSnapshot, { searchExaFn = searchExa, getFxRatesFn = getSharedFxRates, concurrency = EXA_CONCURRENCY } = {}) {
189+
const fxRates = await getFxRatesFn(FX_SYMBOLS, FX_FALLBACKS);
190+
191+
// Fetch the 50 countries with BOUNDED CONCURRENCY. The runner (runSeed) caps the
192+
// whole fetch phase at 240s; a sequential loop of 50 EXA calls (≤15s each) breaches
193+
// that deadline the moment average latency exceeds 240/50 ≈ 4.8s, crashing the run
194+
// with a spurious exit-75 "Deploy Crashed!" alert (issue #4994). At concurrency 6
195+
// the worst case is ⌈50/6⌉ = 9 waves × 15s ≈ 135s — comfortably under the deadline.
196+
const settled = await allSettledWithConcurrency(
197+
COUNTRIES,
198+
concurrency,
199+
(country) => processCountry(country, fxRates, searchExaFn),
200+
);
201+
const results = settled.map((s, i) => {
202+
if (s.status === 'fulfilled') return s.value;
203+
const c = COUNTRIES[i];
204+
console.warn(` [${c.code}] processing failed: ${s.reason?.message || s.reason}`);
205+
return {
206+
code: c.code, name: c.name, currency: c.currency, flag: c.flag,
207+
localPrice: null, usdPrice: null,
208+
fxRate: fxRates[c.currency] ?? FX_FALLBACKS[c.currency] ?? 0,
209+
sourceSite: '', available: false,
210+
};
211+
});
212+
197213
const withData = results.filter(r => r.usdPrice != null);
198214
const cheapest = withData.length ? withData.reduce((a, b) => a.usdPrice < b.usdPrice ? a : b).code : '';
199215
const mostExpensive = withData.length ? withData.reduce((a, b) => a.usdPrice > b.usdPrice ? a : b).code : '';
@@ -263,24 +279,30 @@ async function fetchBigMacPrices(prevSnapshot) {
263279
};
264280
}
265281

266-
const prevSnapshot = await readSeedSnapshot(CANONICAL_KEY);
267-
268282
export function declareRecords(data) {
269283
return data?.countries?.filter(c => c.available).length || 0;
270284
}
271285

272-
await runSeed('economic', 'bigmac', CANONICAL_KEY, () => fetchBigMacPrices(prevSnapshot), {
273-
ttlSeconds: CACHE_TTL,
274-
validateFn: (data) => data?.countries?.length > 0,
275-
recordCount: (data) => data?.countries?.filter(c => c.available).length || 0,
276-
declareRecords,
277-
sourceVersion: 'economist-bigmac-v1',
278-
schemaVersion: 1,
279-
maxStaleMin: 10080,
280-
extraKeys: prevSnapshot ? [{
281-
key: `${CANONICAL_KEY}:prev`,
282-
transform: () => prevSnapshot, // write PRE-overwrite snapshot; ignore new data
283-
ttl: CACHE_TTL * 2,
286+
// Only run the seed when invoked directly (`node seed-bigmac.mjs`), not when
287+
// imported by a test — matches the repo-wide seeder main-guard idiom.
288+
const isMain = process.argv[1]?.endsWith('seed-bigmac.mjs');
289+
if (isMain) {
290+
loadEnvFile(import.meta.url);
291+
const prevSnapshot = await readSeedSnapshot(CANONICAL_KEY);
292+
293+
await runSeed('economic', 'bigmac', CANONICAL_KEY, () => fetchBigMacPrices(prevSnapshot), {
294+
ttlSeconds: CACHE_TTL,
295+
validateFn: (data) => data?.countries?.length > 0,
296+
recordCount: (data) => data?.countries?.filter(c => c.available).length || 0,
284297
declareRecords,
285-
}] : undefined,
286-
});
298+
sourceVersion: 'economist-bigmac-v1',
299+
schemaVersion: 1,
300+
maxStaleMin: 10080,
301+
extraKeys: prevSnapshot ? [{
302+
key: `${CANONICAL_KEY}:prev`,
303+
transform: () => prevSnapshot, // write PRE-overwrite snapshot; ignore new data
304+
ttl: CACHE_TTL * 2,
305+
declareRecords,
306+
}] : undefined,
307+
});
308+
}

‎tests/bigmac-seed.test.mjs‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
import { describe, it } from 'node:test';
2+
import assert from 'node:assert/strict';
3+
4+
import { fetchBigMacPrices, COUNTRIES } from '../scripts/seed-bigmac.mjs';
5+
6+
// Reproduces #4994: the 50-country EXA loop used to run STRICTLY SEQUENTIALLY
7+
// under runSeed's 240s fetch-phase deadline, so it crashed (exit-75 "Deploy
8+
// Crashed!" alert) the moment average EXA latency crept over 240/50 ≈ 4.8s.
9+
// The fix runs the loop with bounded concurrency. These tests pin that in.
10+
11+
const PER_CALL_MS = 25;
12+
13+
// A fake EXA that sleeps PER_CALL_MS then returns a parseable price whose
14+
// currency matches the queried country (query ends with the currency code).
15+
function makeFakeExa({ failCurrencies = new Set() } = {}) {
16+
return async (query) => {
17+
await new Promise((r) => setTimeout(r, PER_CALL_MS));
18+
const ccy = query.trim().split(/\s+/).pop();
19+
if (failCurrencies.has(ccy)) throw new Error(`simulated EXA failure for ${ccy}`);
20+
return { results: [{ summary: `A Big Mac costs 5.00 ${ccy}`, url: 'https://test.example' }] };
21+
};
22+
}
23+
24+
const fakeFx = async () => Object.fromEntries(COUNTRIES.map((c) => [c.currency, 1]));
25+
26+
describe('seed-bigmac fetchBigMacPrices', () => {
27+
it('runs the country loop concurrently — much faster than sequential (regression for #4994)', async () => {
28+
const searchExaFn = makeFakeExa();
29+
30+
const seqStart = Date.now();
31+
await fetchBigMacPrices(null, { searchExaFn, getFxRatesFn: fakeFx, concurrency: 1 });
32+
const seqMs = Date.now() - seqStart;
33+
34+
const conStart = Date.now();
35+
await fetchBigMacPrices(null, { searchExaFn, getFxRatesFn: fakeFx, concurrency: 6 });
36+
const conMs = Date.now() - conStart;
37+
38+
// Concurrency 6 should be well under half the sequential wall-clock. (Ideal
39+
// ratio is ~1/6; assert < 1/3 for CI-scheduler headroom.) BEFORE the fix the
40+
// only path was `concurrency: 1` — this comparison would be ~1.0 and fail.
41+
assert.ok(
42+
conMs < seqMs / 3,
43+
`concurrent run (${conMs}ms) should be < 1/3 of sequential (${seqMs}ms)`,
44+
);
45+
});
46+
47+
it('preserves country order and returns one row per country', async () => {
48+
const data = await fetchBigMacPrices(null, { searchExaFn: makeFakeExa(), getFxRatesFn: fakeFx, concurrency: 6 });
49+
assert.equal(data.countries.length, COUNTRIES.length);
50+
for (let i = 0; i < COUNTRIES.length; i += 1) {
51+
assert.equal(data.countries[i].code, COUNTRIES[i].code, `row ${i} must stay aligned with COUNTRIES order`);
52+
}
53+
// All fakes return a valid in-range price → every country available.
54+
assert.ok(data.countries.every((c) => c.available && c.usdPrice === 5), 'every country resolves a price');
55+
});
56+
57+
it('a single failing country degrades to available:false, never crashing the run', async () => {
58+
const failCurrencies = new Set([COUNTRIES[3].currency]); // one country's EXA throws
59+
const data = await fetchBigMacPrices(null, { searchExaFn: makeFakeExa({ failCurrencies }), getFxRatesFn: fakeFx, concurrency: 6 });
60+
assert.equal(data.countries.length, COUNTRIES.length);
61+
const failed = data.countries.find((c) => c.currency === COUNTRIES[3].currency);
62+
assert.equal(failed.available, false, 'failed country is marked unavailable');
63+
assert.ok(data.countries.some((c) => c.available), 'other countries still resolve');
64+
});
65+
});

0 commit comments

Comments
 (0)