Catalog: a build that learns a platform fetches the firms an older one skipped

A firm on a platform the build didn't know was skipped, yet the cache was
stamped current, so after updating to a build that knows the platform the
sync sent ?since= and the master answered "unchanged": the firm (Vest, on
machines that synced before 96fdd9c) stayed missing until an unrelated
catalog edit. The parser version goes to 3, so those caches fetch in full
once, and a payload now names the platforms it skipped (skippedPlatforms):
a cache whose skipped platform this build knows is no longer current. Kept
rule ids still report from such a cache. scripts/check-catalog.mjs syncs
against a stand-in master on a local port.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

lib/firm-catalog.ts

@@ -49,8 +49,11 @@ const COLD_RETRY_MAX_MS = 60_000;
  *  first sync on a newer parser fetches the whole catalog.
  *  2: kept rule sets (keptAccountRules) and the firm's maxConcurrentAccounts.
  *  Stamped only on a payload from a master that sends keptAccountRules: one
- *  from an older master can't vouch for kept sets it never sent. */
+ *  from an older master can't vouch for kept sets it never sent.
-export const CATALOG_PARSER_VERSION = 2;
+ *  3: Vest firms, which a build from before the platform skipped. A platform
+ *  learned later needs no bump: the cache names the platforms it skipped
+ *  (skippedPlatforms), and a build that knows one fetches in full. */
+export const CATALOG_PARSER_VERSION = 3;
 
 /** A Date header further off than this is somebody else's clock, not the master's. */
 const MAX_MASTER_OFFSET_MS = 365 * 24 * 60 * 60 * 1000;
@@ -103,6 +106,10 @@ function cache(): CatalogCache {
             const parser = (stored as { parser?: unknown }).parser;
             if (typeof parser === 'number') payload.parser = parser;
             else delete payload.parser;
+            // Likewise the platforms it skipped: their firms aren't in the blob.
+            const skipped = (stored as { skippedPlatforms?: unknown }).skippedPlatforms;
+            if (Array.isArray(skipped)) payload.skippedPlatforms = skipped.filter((p): p is string => typeof p === 'string');
+            else delete payload.skippedPlatforms;
         } catch {
             console.error('[catalog] cached payload is unreadable — ignoring it');
         }
@@ -283,6 +290,8 @@ function validatePayload(raw: unknown): FirmCatalogPayload {
     if (!Array.isArray(body.firms)) throw new Error('missing firms');
 
     const firms: CatalogFirm[] = [];
+    // Stamped on the payload, so a build that learns one refetches its firms.
+    const skippedPlatforms = new Set<string>();
     body.firms.forEach((f: unknown, i: number) => {
         if (typeof f !== 'object' || f === null) throw new Error(`firm ${i} is not an object`);
         const firm = f as Record<string, unknown>;
@@ -297,6 +306,7 @@ function validatePayload(raw: unknown): FirmCatalogPayload {
         const platform = firm.platform === undefined ? 'tradovate' : firm.platform;
         if (!isFirmPlatform(platform)) {
             warnUnknownPlatform(firm.id, firm.name, platform);
+            skippedPlatforms.add(String(platform));
             return;
         }
         // Percent commission and every-day trading are settings of the
@@ -349,6 +359,7 @@ function validatePayload(raw: unknown): FirmCatalogPayload {
     return {
         version: body.version, updatedAt: body.updatedAt, firms, ...(trading ? { trading } : {}),
         ...(keptAware ? { parser: CATALOG_PARSER_VERSION } : {}),
+        ...(skippedPlatforms.size > 0 ? { skippedPlatforms: [...skippedPlatforms] } : {}),
     };
 }
 
@@ -424,10 +435,13 @@ export async function syncCatalog(): Promise<{ ok: boolean; changed: boolean; ve
     // `since` only against the master the cache came from: two masters'
     // version numbers can collide, and an 'unchanged' from the wrong one
     // would keep the other's catalog. Nor for a cache an older parser wrote:
-    // it lacks what that parser dropped, and 'unchanged' would keep it so.
+    // it lacks what that parser dropped, and 'unchanged' would keep it so —
+    // fields, or firms on a platform it didn't know and this build does.
     const hasCache = getCachedCatalog().length > 0;
     const source = url.origin;
-    const cacheCurrent = (cache().payload?.parser ?? 0) >= CATALOG_PARSER_VERSION;
+    const cached = cache().payload;
+    const cacheCurrent = (cached?.parser ?? 0) >= CATALOG_PARSER_VERSION
+        && !(cached?.skippedPlatforms ?? []).some((p) => isFirmPlatform(p));
     if (hasCache && cacheCurrent && getSetting('catalog_source_url') === source) url.searchParams.set('since', String(version));
 
     try {

lib/master-protocol.ts

@@ -508,6 +508,9 @@ export interface FirmCatalogPayload {
      *  payload. An older parser dropped fields it didn't know, so a cache
      *  without it, or below the current one, isn't trusted to hold them. */
     parser?: number;
+    /** Instance-local, like parser: the platforms of the firms that parser
+     *  skipped as unknown to it. A build that knows one fetches in full. */
+    skippedPlatforms?: string[];
 }
 
 export interface FirmCatalogUnchanged {

package.json

@@ -7,7 +7,7 @@
     "build": "next build",
     "start": "next start",
     "lint": "eslint",
-    "check": "node --experimental-strip-types --disable-warning=MODULE_TYPELESS_PACKAGE_JSON --test scripts/check-trading-logic.mjs scripts/check-instruments.mjs scripts/check-account-status.mjs scripts/check-kept-rules.mjs scripts/check-request-user-agent.mjs scripts/check-vest.mjs scripts/check-trades.mjs"
+    "check": "node --experimental-strip-types --disable-warning=MODULE_TYPELESS_PACKAGE_JSON --test scripts/check-trading-logic.mjs scripts/check-instruments.mjs scripts/check-account-status.mjs scripts/check-kept-rules.mjs scripts/check-request-user-agent.mjs scripts/check-vest.mjs scripts/check-trades.mjs scripts/check-catalog.mjs"
   },
   "dependencies": {
     "@microsoft/signalr": "^10.0.11",

scripts/check-catalog.mjs

@@ -0,0 +1,107 @@
+// Checks for the firm catalog sync in lib/firm-catalog.ts, against a stand-in
+// master on a local port and a throwaway database — never the live DB or the
+// real master.
+//
+//   npm run check
+import { after, test } from 'node:test';
+import assert from 'node:assert/strict';
+import { mkdtempSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { registerHooks } from 'node:module';
+import { createServer } from 'node:http';
+import { once } from 'node:events';
+
+// Resolve the extensionless local imports used by Next, without a build.
+function resolveTypes(specifier, context, nextResolve) {
+    try {
+        return nextResolve(specifier, context);
+    } catch (err) {
+        if (err.code !== 'ERR_MODULE_NOT_FOUND' || !specifier.startsWith('.')) throw err;
+        return nextResolve(`${specifier}.ts`, context);
+    }
+}
+registerHooks({ resolve: resolveTypes });
+
+const directory = mkdtempSync(join(tmpdir(), 'autofirmer-catalog-'));
+process.env.AUTOFIRMER_DB = join(directory, 'test.sqlite');
+after(() => rmSync(directory, { recursive: true, force: true }));
+
+const { getSetting, setSetting } = await import('../lib/db.ts');
+const { syncCatalog, getCachedCatalog, getKeptRuleIdsForFirm, CATALOG_PARSER_VERSION } = await import('../lib/firm-catalog.ts');
+
+const firm = (id, name, platform) => ({
+    id, name, platform, keptAccountRules: [], updatedAt: '2026-10-26T19:00:00.000Z',
+    accountTypes: [{
+        id: `${id}-50k`, firmId: id, label: '50k', prefix: `${id.toUpperCase()}-`, updatedAt: '2026-10-26T19:00:00.000Z',
+        accountSize: 50000, profitTarget: 3000, consistency: 0, minDayPnL: 0, minTradingDays: 0, maxLoss: 3000,
+        maxPositionSize: 0, withdrawalStages: [],
+    }],
+});
+const apex = firm('apex', 'Apex', 'tradovate');
+const vest = firm('vest', 'VestMarkets', 'vest');
+
+// The master's catalog, answering ?since= as app/api/firms/catalog does, and
+// the `since` each sync sent (null = a full fetch).
+let served = { version: 41, updatedAt: '2026-10-26T19:20:00.000Z', firms: [apex, vest] };
+const sinceSent = [];
+const server = createServer((req, res) => {
+    const since = new URL(req.url, 'http://master').searchParams.get('since');
+    sinceSent.push(since);
+    res.setHeader('Content-Type', 'application/json');
+    res.end(JSON.stringify(since !== null && Number(since) === served.version
+        ? { version: served.version, updatedAt: served.updatedAt, unchanged: true }
+        : served));
+});
+server.listen(0, '127.0.0.1');
+await once(server, 'listening');
+after(() => server.close());
+const master = `http://127.0.0.1:${server.address().port}`;
+
+/** This machine's cache as some build stored it, read afresh as after a restart. */
+function storedCache(payload) {
+    setSetting('master_dashboard_url', master);
+    setSetting('catalog_source_url', master);
+    setSetting('catalog_version', String(payload.version));
+    setSetting('catalog_payload', JSON.stringify(payload));
+    globalThis.__firmCatalog = undefined;
+    sinceSent.length = 0;
+}
+
+const names = () => getCachedCatalog().map((f) => f.name);
+
+test('a cache a pre-Vest build stored is fetched in full by a build that knows Vest', async (t) => {
+    t.mock.method(console, 'log', () => {});
+    // Parser 2 skipped the Vest firm and said nothing of it; the master's
+    // catalog hasn't changed since, so ?since= would answer "unchanged".
+    storedCache({ version: 41, updatedAt: served.updatedAt, firms: [apex], parser: 2 });
+    assert.equal((await syncCatalog()).ok, true);
+    assert.deepEqual(sinceSent, [null]);
+    assert.deepEqual(names(), ['Apex', 'VestMarkets']);
+    // Current from then on: the next sync asks ?since= again.
+    await syncCatalog();
+    assert.deepEqual(sinceSent, [null, '41']);
+});
+
+test('a skipped platform is named in the cache, and a build that knows it fetches in full', async (t) => {
+    t.mock.method(console, 'log', () => {});
+    t.mock.method(console, 'warn', () => {});
+    // The master adds a firm on a platform this build doesn't know.
+    served = { version: 42, updatedAt: '2026-10-27T15:00:00.000Z', firms: [apex, vest, firm('px', 'PX Firm', 'projectx')] };
+    storedCache({ version: 41, updatedAt: '2026-10-26T19:20:00.000Z', firms: [apex, vest], parser: CATALOG_PARSER_VERSION });
+    await syncCatalog();
+    assert.deepEqual(names(), ['Apex', 'VestMarkets']);
+    assert.deepEqual(JSON.parse(getSetting('catalog_payload')).skippedPlatforms, ['projectx']);
+    // While this build still doesn't know it, syncs stay incremental, and the
+    // cache still vouches for the kept rule sets of the firms it holds.
+    globalThis.__firmCatalog = undefined;
+    sinceSent.length = 0;
+    await syncCatalog();
+    assert.deepEqual(sinceSent, ['42']);
+    assert.deepEqual(getKeptRuleIdsForFirm('apex'), []);
+    // A build that knows the platform it skipped (Vest here) fetches in full.
+    storedCache({ version: 42, updatedAt: served.updatedAt, firms: [apex], parser: CATALOG_PARSER_VERSION, skippedPlatforms: ['vest'] });
+    await syncCatalog();
+    assert.deepEqual(sinceSent, [null]);
+    assert.ok(names().includes('VestMarkets'));
+});