From 952e4412c4813f2c532e713f69d07f4abb6bda0a Mon Sep 17 00:00:00 2001 From: Brandon Li Date: Tue, 27 Oct 2026 14:14:43 -0500 Subject: [PATCH] Catalog: a build that learns a platform fetches the firms an older one skipped A firm on a platform the build didn't know was skipped, yet the cache was stamped current, so after updating to a build that knows the platform the sync sent ?since= and the master answered "unchanged": the firm (Vest, on machines that synced before 96fdd9c) stayed missing until an unrelated catalog edit. The parser version goes to 3, so those caches fetch in full once, and a payload now names the platforms it skipped (skippedPlatforms): a cache whose skipped platform this build knows is no longer current. Kept rule ids still report from such a cache. scripts/check-catalog.mjs syncs against a stand-in master on a local port. Co-Authored-By: Claude Opus 5.5 --- lib/firm-catalog.ts | 22 ++++++-- lib/master-protocol.ts | 3 ++ package.json | 2 +- scripts/check-catalog.mjs | 107 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 129 insertions(+), 5 deletions(-) create mode 100644 scripts/check-catalog.mjs diff --git a/lib/firm-catalog.ts b/lib/firm-catalog.ts index e71ac6b..3fda3ab 100644 --- a/lib/firm-catalog.ts +++ b/lib/firm-catalog.ts @@ -49,8 +49,11 @@ const COLD_RETRY_MAX_MS = 60_000; * first sync on a newer parser fetches the whole catalog. * 2: kept rule sets (keptAccountRules) and the firm's maxConcurrentAccounts. * Stamped only on a payload from a master that sends keptAccountRules: one - * from an older master can't vouch for kept sets it never sent. */ -export const CATALOG_PARSER_VERSION = 2; + * from an older master can't vouch for kept sets it never sent. + * 3: Vest firms, which a build from before the platform skipped. A platform + * learned later needs no bump: the cache names the platforms it skipped + * (skippedPlatforms), and a build that knows one fetches in full. */ +export const CATALOG_PARSER_VERSION = 3; /** A Date header further off than this is somebody else's clock, not the master's. */ const MAX_MASTER_OFFSET_MS = 365 * 24 * 60 * 60 * 1000; @@ -103,6 +106,10 @@ function cache(): CatalogCache { const parser = (stored as { parser?: unknown }).parser; if (typeof parser === 'number') payload.parser = parser; else delete payload.parser; + // Likewise the platforms it skipped: their firms aren't in the blob. + const skipped = (stored as { skippedPlatforms?: unknown }).skippedPlatforms; + if (Array.isArray(skipped)) payload.skippedPlatforms = skipped.filter((p): p is string => typeof p === 'string'); + else delete payload.skippedPlatforms; } catch { console.error('[catalog] cached payload is unreadable — ignoring it'); } @@ -283,6 +290,8 @@ function validatePayload(raw: unknown): FirmCatalogPayload { if (!Array.isArray(body.firms)) throw new Error('missing firms'); const firms: CatalogFirm[] = []; + // Stamped on the payload, so a build that learns one refetches its firms. + const skippedPlatforms = new Set(); body.firms.forEach((f: unknown, i: number) => { if (typeof f !== 'object' || f === null) throw new Error(`firm ${i} is not an object`); const firm = f as Record; @@ -297,6 +306,7 @@ function validatePayload(raw: unknown): FirmCatalogPayload { const platform = firm.platform === undefined ? 'tradovate' : firm.platform; if (!isFirmPlatform(platform)) { warnUnknownPlatform(firm.id, firm.name, platform); + skippedPlatforms.add(String(platform)); return; } // Percent commission and every-day trading are settings of the @@ -349,6 +359,7 @@ function validatePayload(raw: unknown): FirmCatalogPayload { return { version: body.version, updatedAt: body.updatedAt, firms, ...(trading ? { trading } : {}), ...(keptAware ? { parser: CATALOG_PARSER_VERSION } : {}), + ...(skippedPlatforms.size > 0 ? { skippedPlatforms: [...skippedPlatforms] } : {}), }; } @@ -424,10 +435,13 @@ export async function syncCatalog(): Promise<{ ok: boolean; changed: boolean; ve // `since` only against the master the cache came from: two masters' // version numbers can collide, and an 'unchanged' from the wrong one // would keep the other's catalog. Nor for a cache an older parser wrote: - // it lacks what that parser dropped, and 'unchanged' would keep it so. + // it lacks what that parser dropped, and 'unchanged' would keep it so — + // fields, or firms on a platform it didn't know and this build does. const hasCache = getCachedCatalog().length > 0; const source = url.origin; - const cacheCurrent = (cache().payload?.parser ?? 0) >= CATALOG_PARSER_VERSION; + const cached = cache().payload; + const cacheCurrent = (cached?.parser ?? 0) >= CATALOG_PARSER_VERSION + && !(cached?.skippedPlatforms ?? []).some((p) => isFirmPlatform(p)); if (hasCache && cacheCurrent && getSetting('catalog_source_url') === source) url.searchParams.set('since', String(version)); try { diff --git a/lib/master-protocol.ts b/lib/master-protocol.ts index 897ccf9..e062ecb 100644 --- a/lib/master-protocol.ts +++ b/lib/master-protocol.ts @@ -508,6 +508,9 @@ export interface FirmCatalogPayload { * payload. An older parser dropped fields it didn't know, so a cache * without it, or below the current one, isn't trusted to hold them. */ parser?: number; + /** Instance-local, like parser: the platforms of the firms that parser + * skipped as unknown to it. A build that knows one fetches in full. */ + skippedPlatforms?: string[]; } export interface FirmCatalogUnchanged { diff --git a/package.json b/package.json index 90e5328..9cab0cc 100644 --- a/package.json +++ b/package.json @@ -7,7 +7,7 @@ "build": "next build", "start": "next start", "lint": "eslint", - "check": "node --experimental-strip-types --disable-warning=MODULE_TYPELESS_PACKAGE_JSON --test scripts/check-trading-logic.mjs scripts/check-instruments.mjs scripts/check-account-status.mjs scripts/check-kept-rules.mjs scripts/check-request-user-agent.mjs scripts/check-vest.mjs scripts/check-trades.mjs" + "check": "node --experimental-strip-types --disable-warning=MODULE_TYPELESS_PACKAGE_JSON --test scripts/check-trading-logic.mjs scripts/check-instruments.mjs scripts/check-account-status.mjs scripts/check-kept-rules.mjs scripts/check-request-user-agent.mjs scripts/check-vest.mjs scripts/check-trades.mjs scripts/check-catalog.mjs" }, "dependencies": { "@microsoft/signalr": "^10.0.11", diff --git a/scripts/check-catalog.mjs b/scripts/check-catalog.mjs new file mode 100644 index 0000000..28c4d55 --- /dev/null +++ b/scripts/check-catalog.mjs @@ -0,0 +1,107 @@ +// Checks for the firm catalog sync in lib/firm-catalog.ts, against a stand-in +// master on a local port and a throwaway database — never the live DB or the +// real master. +// +// npm run check +import { after, test } from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { registerHooks } from 'node:module'; +import { createServer } from 'node:http'; +import { once } from 'node:events'; + +// Resolve the extensionless local imports used by Next, without a build. +function resolveTypes(specifier, context, nextResolve) { + try { + return nextResolve(specifier, context); + } catch (err) { + if (err.code !== 'ERR_MODULE_NOT_FOUND' || !specifier.startsWith('.')) throw err; + return nextResolve(`${specifier}.ts`, context); + } +} +registerHooks({ resolve: resolveTypes }); + +const directory = mkdtempSync(join(tmpdir(), 'autofirmer-catalog-')); +process.env.AUTOFIRMER_DB = join(directory, 'test.sqlite'); +after(() => rmSync(directory, { recursive: true, force: true })); + +const { getSetting, setSetting } = await import('../lib/db.ts'); +const { syncCatalog, getCachedCatalog, getKeptRuleIdsForFirm, CATALOG_PARSER_VERSION } = await import('../lib/firm-catalog.ts'); + +const firm = (id, name, platform) => ({ + id, name, platform, keptAccountRules: [], updatedAt: '2026-10-26T19:00:00.000Z', + accountTypes: [{ + id: `${id}-50k`, firmId: id, label: '50k', prefix: `${id.toUpperCase()}-`, updatedAt: '2026-10-26T19:00:00.000Z', + accountSize: 50000, profitTarget: 3000, consistency: 0, minDayPnL: 0, minTradingDays: 0, maxLoss: 3000, + maxPositionSize: 0, withdrawalStages: [], + }], +}); +const apex = firm('apex', 'Apex', 'tradovate'); +const vest = firm('vest', 'VestMarkets', 'vest'); + +// The master's catalog, answering ?since= as app/api/firms/catalog does, and +// the `since` each sync sent (null = a full fetch). +let served = { version: 41, updatedAt: '2026-10-26T19:20:00.000Z', firms: [apex, vest] }; +const sinceSent = []; +const server = createServer((req, res) => { + const since = new URL(req.url, 'http://master').searchParams.get('since'); + sinceSent.push(since); + res.setHeader('Content-Type', 'application/json'); + res.end(JSON.stringify(since !== null && Number(since) === served.version + ? { version: served.version, updatedAt: served.updatedAt, unchanged: true } + : served)); +}); +server.listen(0, '127.0.0.1'); +await once(server, 'listening'); +after(() => server.close()); +const master = `http://127.0.0.1:${server.address().port}`; + +/** This machine's cache as some build stored it, read afresh as after a restart. */ +function storedCache(payload) { + setSetting('master_dashboard_url', master); + setSetting('catalog_source_url', master); + setSetting('catalog_version', String(payload.version)); + setSetting('catalog_payload', JSON.stringify(payload)); + globalThis.__firmCatalog = undefined; + sinceSent.length = 0; +} + +const names = () => getCachedCatalog().map((f) => f.name); + +test('a cache a pre-Vest build stored is fetched in full by a build that knows Vest', async (t) => { + t.mock.method(console, 'log', () => {}); + // Parser 2 skipped the Vest firm and said nothing of it; the master's + // catalog hasn't changed since, so ?since= would answer "unchanged". + storedCache({ version: 41, updatedAt: served.updatedAt, firms: [apex], parser: 2 }); + assert.equal((await syncCatalog()).ok, true); + assert.deepEqual(sinceSent, [null]); + assert.deepEqual(names(), ['Apex', 'VestMarkets']); + // Current from then on: the next sync asks ?since= again. + await syncCatalog(); + assert.deepEqual(sinceSent, [null, '41']); +}); + +test('a skipped platform is named in the cache, and a build that knows it fetches in full', async (t) => { + t.mock.method(console, 'log', () => {}); + t.mock.method(console, 'warn', () => {}); + // The master adds a firm on a platform this build doesn't know. + served = { version: 42, updatedAt: '2026-10-27T15:00:00.000Z', firms: [apex, vest, firm('px', 'PX Firm', 'projectx')] }; + storedCache({ version: 41, updatedAt: '2026-10-26T19:20:00.000Z', firms: [apex, vest], parser: CATALOG_PARSER_VERSION }); + await syncCatalog(); + assert.deepEqual(names(), ['Apex', 'VestMarkets']); + assert.deepEqual(JSON.parse(getSetting('catalog_payload')).skippedPlatforms, ['projectx']); + // While this build still doesn't know it, syncs stay incremental, and the + // cache still vouches for the kept rule sets of the firms it holds. + globalThis.__firmCatalog = undefined; + sinceSent.length = 0; + await syncCatalog(); + assert.deepEqual(sinceSent, ['42']); + assert.deepEqual(getKeptRuleIdsForFirm('apex'), []); + // A build that knows the platform it skipped (Vest here) fetches in full. + storedCache({ version: 42, updatedAt: served.updatedAt, firms: [apex], parser: CATALOG_PARSER_VERSION, skippedPlatforms: ['vest'] }); + await syncCatalog(); + assert.deepEqual(sinceSent, [null]); + assert.ok(names().includes('VestMarkets')); +});