diff --git a/app/api/health/route.ts b/app/api/health/route.ts new file mode 100644 index 0000000..64f0718 --- /dev/null +++ b/app/api/health/route.ts @@ -0,0 +1,72 @@ +import { readFileSync } from 'node:fs'; +import path from 'node:path'; +import { NextResponse } from 'next/server'; +import { getClients } from '@/lib/clients'; +import { getTradableFirms } from '@/lib/firm-catalog'; +import { getSchedulerHealth, getSchedulerStatus } from '@/lib/auto-trade'; +import { getBuildVersion } from '@/lib/version'; +import { errorMessage, isClientReady } from '@/lib/platform-client'; + +// Liveness has to be computed now, never served from a cache. +export const dynamic = 'force-dynamic'; + +/** A one-line marker file, or null when it isn't there. */ +function readMarker(...segments: string[]): string | null { + try { + return readFileSync(path.join(process.cwd(), ...segments), 'utf8').trim() || null; + } catch { + return null; + } +} + +/** + * GET /api/health — is this instance able to trade, and if not, why. + * + * The numbers to read before restarting a broken box, and what start-all + * prints after boot. Always 200 when the handler runs: `ok` in the body is the + * verdict (every tradable firm that has a client is synced, and synced + * recently), so a caller can tell "unhealthy" from "not answering". + * + * Calls getClients(), so hitting it also boots an instance that somehow + * hasn't been. + * + * Built field by field on purpose — firm rows carry passwords and API keys, + * and nothing here may echo a credential or a token. + */ +export async function GET() { + try { + const now = Date.now(); + const clients = getClients(); + + const entries = getTradableFirms().map((firm) => { + const base = { firmId: firm.id, firm: firm.name, platform: firm.platform }; + const client = clients.get(firm.id); + if (!client) return { ...base, hasClient: false, ready: false, accounts: 0 }; + + const ready = isClientReady(client, now); + const accounts = client.accountList.length; + try { + return { ...base, hasClient: true, ready, accounts, ...client.health() }; + } catch (err) { + // One client's broken health() must not hide the rest. + return { ...base, hasClient: true, ready, accounts, healthError: errorMessage(err) }; + } + }); + + return NextResponse.json({ + ok: entries.every((e) => !e.hasClient || e.ready), + now, + uptimeSec: Math.round(process.uptime()), + version: getBuildVersion(), + // Read from disk on every call: the build .next holds now (the + // running one, unless a build landed since this process started), + // and the commit the updater last finished building. + buildId: readMarker('.next', 'BUILD_ID'), + builtCommit: readMarker('scripts', '.built-commit'), + scheduler: { ...getSchedulerStatus(), ...getSchedulerHealth() }, + clients: entries, + }); + } catch (err) { + return NextResponse.json({ ok: false, error: errorMessage(err) }, { status: 500 }); + } +} diff --git a/app/api/trade/route.ts b/app/api/trade/route.ts index f08d6cd..f5e90cc 100644 --- a/app/api/trade/route.ts +++ b/app/api/trade/route.ts @@ -1,5 +1,6 @@ import { NextRequest, NextResponse } from 'next/server'; import { runTrade, startScheduler } from '@/lib/auto-trade'; +import { errorMessage } from '@/lib/platform-client'; import { POINT_VALUES } from '@/lib/trading-logic'; export async function POST(req: NextRequest) { @@ -15,15 +16,19 @@ export async function POST(req: NextRequest) { return NextResponse.json({ error: `Unknown symbol: ${symbol}` }, { status: 400 }); } + // (Re)start the scheduler with this action + symbol first, so a batch + // that throws or runs long can never leave it unstarted. It can't + // double-fire against the batch below: the tick waits for flat + // positions, skips while this batch holds the trade mutex, and never + // picks an account whose order is still in flight. + startScheduler(action, symbol, stopAfterAll ?? false); + // Execute trade immediately for all eligible accounts const results = await runTrade(action, symbol); - // (Re)start the scheduler with this action + symbol - startScheduler(action, symbol, stopAfterAll ?? false); - return NextResponse.json(results); - } catch (err: any) { - console.error('[POST /api/trade]', err); - return NextResponse.json({ error: err?.message ?? 'Trade failed' }, { status: 500 }); + } catch (err: unknown) { + console.error(`[POST /api/trade] ${errorMessage(err)}`); + return NextResponse.json({ error: errorMessage(err) || 'Trade failed' }, { status: 500 }); } } diff --git a/instrumentation.ts b/instrumentation.ts new file mode 100644 index 0000000..cc4cd20 --- /dev/null +++ b/instrumentation.ts @@ -0,0 +1,40 @@ +/** + * Server start hook — Next runs register() once per server process, before + * it serves the first request (a root instrumentation.ts is picked up with no + * config in Next 15+). + * + * It boots the trading background work eagerly. getClients() in lib/clients.ts + * starts the broker clients, the contract resolver, the reporter, the master + * link, the client supervisor and the persisted-schedule resume — and it used + * to run only on the first HTTP request. So a restarted instance (a crash, an + * update, pm2 after a reboot) sat idle, not trading and invisible to the + * master, until someone opened the dashboard or start-all's warm-up curl + * happened to reach it. + * + * The boot must never take the server down with it: a failure is logged, and + * if the boot never got going the first request still starts it through the + * same lazy getClients() path as before. + * + * This module is its own server bundle, so it can hold its own instances of + * lib/ modules, separate from the route handlers'. Anything the two must share + * (the client pool, the scheduler, timers) lives on globalThis for that reason. + */ + +export async function register(): Promise { + // The nodejs check is the documented shape for a Node-only import, kept + // exactly: the edge compile (Turbopack always builds one) inlines + // NEXT_RUNTIME and drops the block, so lib/clients — sqlite, sockets — + // never reaches the edge bundle, where it would fail the build. The build + // check is defense in depth (Next itself doesn't call register() during + // `next build`) — a build must never dial brokers or resume the scheduler. + if (process.env.NEXT_RUNTIME === 'nodejs') { + if (process.env.NEXT_PHASE === 'phase-production-build') return; + try { + const { getClients } = await import('./lib/clients'); + getClients(); + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + console.error(`[instrumentation] eager boot failed — clients will start on the first request: ${message}`); + } + } +} diff --git a/lib/auto-trade.ts b/lib/auto-trade.ts index ac32a71..eda0935 100644 --- a/lib/auto-trade.ts +++ b/lib/auto-trade.ts @@ -18,39 +18,302 @@ import { type DayPlan, } from './trading-logic'; import { getSetting, setSetting } from './db'; +import { errorMessage, isClientReady, isOrderNotPlaced, type PlatformClient } from './platform-client'; + +// ── process-wide state ─────────────────────────────────────────────────────── + +/** + * Everything the batches share, held on globalThis like the scheduler state + * further down. Next can bundle this module more than once — the scheduler + * tick runs from whichever bundle booted the clients, POST /api/trade and + * /api/copy-trade from their own route bundles — and plain module-level state + * would split into one private copy per bundle: a leg recorded by the tick + * would be invisible to a copy-trade, the trade mutex would lock nothing, and + * the session-open jitter could be rolled twice in one day. + */ +interface TradeRuntime { + /** In-memory runaway guard: legs fired per account per trade date, plus + * the sum of their targets — realized P&L falling short of that sum is + * how a stop-out is detected. Resets on restart, which is fine — day + * completion is derived from realized PnL, not from this counter. */ + legCounts: Map; + /** Today's session-open delay, keyed by Central calendar day (see openJitterMinutes). */ + sessionOpenJitter: Map; + /** Accounts whose orders are being placed right now. Their position only + * shows once the fill comes back over the socket, so until then this is + * what keeps them out of every other batch. */ + accountsInFlight: Set; + /** Firm id → when the tick first saw its client not ready (see UNREADY_GRACE_MS). */ + unreadySince: Map; + /** Firm id → when the tick last logged it as quarantined (rate limit). */ + quarantineLoggedAt: Map; + /** When the batch holding the trade mutex took it (0 = no batch in flight). */ + tradeInFlightSince: number; + // What getSchedulerHealth reports. + lastTickAt: number; + lastTickOutcome: TickOutcome; + lastFireAt: number; + holding: string[]; + quarantined: string[]; +} + +/** What the scheduler's last tick decided (getSchedulerHealth). */ +export type TickOutcome = + | 'not-running' | 'holding-for-sync' | 'positions-open' | 'closed' + | 'no-eligible' | 'fired' | 'skipped-in-flight' | `error: ${string}`; + +const _tradeGlobal = globalThis as typeof globalThis & { __autoTradeRuntime?: TradeRuntime }; + +function runtime(): TradeRuntime { + if (!_tradeGlobal.__autoTradeRuntime) { + _tradeGlobal.__autoTradeRuntime = { + legCounts: new Map(), + sessionOpenJitter: new Map(), + accountsInFlight: new Set(), + unreadySince: new Map(), + quarantineLoggedAt: new Map(), + tradeInFlightSince: 0, + lastTickAt: 0, + lastTickOutcome: 'not-running', + lastFireAt: 0, + holding: [], + quarantined: [], + }; + } + return _tradeGlobal.__autoTradeRuntime; +} + +/** How long the tick holds every firm for one whose client isn't ready. Long + * enough for a restart or a reconnect to finish syncing; past it the client + * is presumed dead and its firm is quarantined (see assessReadiness). */ +const UNREADY_GRACE_MS = 10 * 60 * 1000; +/** How often a quarantined firm is re-announced in the log. */ +const QUARANTINE_LOG_EVERY_MS = 5 * 60 * 1000; +/** A batch that has held the trade mutex this long is presumed wedged. */ +const TRADE_WEDGED_MS = 10 * 60 * 1000; // ── helpers ────────────────────────────────────────────────────────────────── +/** How many accounts one batch may fire, never below 1. The settings PATCH + * stores whatever the form sends, and a cleared field arrives as '' — + * parseInt('') is NaN, which slice() takes as 0: every batch would fire + * nobody while the tick reported 'fired'. Non-numeric falls back to the + * default 5, like a missing setting; the form's own floor is 1. */ +function maxConcurrentAccounts(): number { + const configured = parseInt(getSetting('max_concurrent_accounts') ?? '5', 10); + return Math.max(1, Number.isFinite(configured) ? configured : 5); +} + function isAccountDead(amount: number, autoLiqThreshold: number): boolean { // Tradovate floors are positive balances; reconstructed TopstepX MLL // floors are negative (or +$0.01 once locked at breakeven). 0 = no floor. return autoLiqThreshold !== 0 && amount <= autoLiqThreshold; } -// In-memory runaway guard: legs fired per account per trade date, plus the sum -// of their targets — realized P&L falling short of that sum is how a stop-out -// is detected. Resets on restart, which is fine — day completion is derived -// from realized PnL, not from this counter. -const _legCounts = new Map(); - function legsToday(accountId: number, tradeDate: string): number { - const entry = _legCounts.get(accountId); + const entry = runtime().legCounts.get(accountId); return entry && entry.date === tradeDate ? entry.count : 0; } function legSumToday(accountId: number, tradeDate: string): number { - const entry = _legCounts.get(accountId); + const entry = runtime().legCounts.get(accountId); return entry && entry.date === tradeDate ? entry.sum : 0; } function recordLeg(accountId: number, tradeDate: string, legAmount: number): void { - _legCounts.set(accountId, { + runtime().legCounts.set(accountId, { date: tradeDate, count: legsToday(accountId, tradeDate) + 1, sum: Math.round((legSumToday(accountId, tradeDate) + legAmount) * 100) / 100, }); } +/** Take back a leg recordLeg counted for an entry that never reached the + * broker. Left standing, it would bench a stop-type account for the day — + * realized P&L "falls short" of a leg that never traded — and burn one of + * the day's MAX_LEGS_PER_DAY. */ +function unrecordLeg(accountId: number, tradeDate: string, legAmount: number): void { + const legCounts = runtime().legCounts; + const entry = legCounts.get(accountId); + if (!entry || entry.date !== tradeDate || entry.count <= 0) return; + if (entry.count === 1) { + legCounts.delete(accountId); + return; + } + legCounts.set(accountId, { + date: tradeDate, + count: entry.count - 1, + sum: Math.round((entry.sum - legAmount) * 100) / 100, + }); +} + +/** + * Send a leg's market entry. The leg was recorded before this — the runaway + * guard has to see it while the batch is still in flight — so it is taken + * back here only when the client KNOWS the order never reached the broker + * (OrderNotPlacedError). Anything ambiguous, like an ack or fill timeout, + * keeps the leg: the order may well have filled, and a leg counted that never + * happened only costs a leg, while one forgotten that did can trade the + * account twice. + */ +async function sendEntry( + tag: string, + // eslint-disable-next-line @typescript-eslint/no-explicit-any + client: any, + acc: { id: number; name: string }, + contractName: string, + contracts: number, + action: 'Buy' | 'Sell', + tradeDate: string, + legAmount: number, +) { + try { + return await client.sendOrder(acc.id, contractName, contracts, action, 'Market'); + } catch (err) { + if (isOrderNotPlaced(err)) { + unrecordLeg(acc.id, tradeDate, legAmount); + console.warn(`${tag} ${acc.name}: entry never reached the broker — leg un-counted`); + } else { + console.warn(`${tag} ${acc.name}: entry outcome unknown — leg kept (it may have filled)`); + } + throw err; + } +} + +/** Run one account's part of a batch with the account marked in flight, so + * no other batch picks it while its orders are still being placed. */ +async function withAccountInFlight(accountId: number, fire: () => Promise): Promise { + const inFlight = runtime().accountsInFlight; + inFlight.add(accountId); + try { + return await fire(); + } finally { + inFlight.delete(accountId); + } +} + +// ── readiness ──────────────────────────────────────────────────────────────── + +type TradableFirm = ReturnType[number]; +interface FirmClient { firm: TradableFirm; client: PlatformClient } + +/** + * Tradable firms that have a client, split by isClientReady. Everything that + * trades or gates works from `ready` only: a client whose sync is missing or + * stale holds a frozen snapshot, whose positions may be long gone and which + * may be missing ones opened since — it must neither trade nor gate. + */ +function firmClients(now = Date.now()): { ready: FirmClient[]; unready: FirmClient[] } { + const clients = getClients(); + const ready: FirmClient[] = []; + const unready: FirmClient[] = []; + for (const firm of getTradableFirms()) { + const client = clients.get(firm.id); + if (!client) continue; + (isClientReady(client, now) ? ready : unready).push({ firm, client }); + } + return { ready, unready }; +} + +function countOpenPositions(ready: FirmClient[]): number { + return ready.reduce( + (count, { client }) => count + client.accountList.filter((acc) => !!client.positions[acc.id]).length, + 0, + ); +} + +/** + * Which firms hold the tick, and which it trades without. + * + * A firm whose client isn't ready might hold positions nobody can see, so for + * the first UNREADY_GRACE_MS the whole tick waits for it, as it always has — + * right for a restart or a reconnect, which settle within a minute or two. + * Past the grace period the client is presumed dead (broker maintenance, a + * login that keeps failing) and the firm is quarantined: the other firms + * trade without it, rather than one broken session stopping every firm on the + * machine. A quarantined firm trades nothing — runTrade and copyTrade only + * ever see ready clients — and rejoins on its first good sync. + * + * A client that has synced and found no accounts is ignored outright: it + * cannot hold positions or trade, so there is nothing to wait for. + */ +function assessReadiness( + ready: FirmClient[], + unready: FirmClient[], + now: number, +): { holding: TradableFirm[]; quarantined: { firm: TradableFirm; unreadyMs: number }[] } { + const rt = runtime(); + const holding: TradableFirm[] = []; + const quarantined: { firm: TradableFirm; unreadyMs: number }[] = []; + const tracked = new Set(); + + for (const { firm, client } of unready) { + if (client.lastSyncAt > 0 && client.accountList.length === 0) continue; + tracked.add(firm.id); + const since = rt.unreadySince.get(firm.id) ?? now; + rt.unreadySince.set(firm.id, since); + const unreadyMs = now - since; + if (unreadyMs < UNREADY_GRACE_MS) holding.push(firm); + else quarantined.push({ firm, unreadyMs }); + } + + // Forget firms that are ready again (or gone), and say so for any that + // had been traded around. + for (const id of [...rt.unreadySince.keys()]) { + if (tracked.has(id)) continue; + const back = ready.find((r) => r.firm.id === id); + if (back && rt.quarantineLoggedAt.has(id)) { + console.log(`[scheduler] ${back.firm.name} synced again — trading it with the others`); + } + rt.unreadySince.delete(id); + rt.quarantineLoggedAt.delete(id); + } + + return { holding, quarantined }; +} + +// ── trade mutex ────────────────────────────────────────────────────────────── + +/** + * One batch at a time, instance-wide. The tick, a master "trade" command, + * POST /api/trade and /api/copy-trade can all fire a batch, and two batches + * collecting at once would each see the other's accounts as still flat — the + * position gate only trips once a fill has come back over the socket. + * + * A batch that has held the mutex past TRADE_WEDGED_MS is presumed stuck on a + * broker call that will never answer, and the next one goes ahead without it + * rather than nothing trading for the rest of the process's life. The stuck + * batch's accounts stay in accountsInFlight, so they are never traded twice, + * and if the stuck batch does wake up it finds it no longer holds the mutex + * and abandons its now-stale selection (holdsTradeMutex). + * + * Returns the lease to hand to releaseTradeMutex, or null to skip. + */ +function acquireTradeMutex(tag: string): number | null { + const rt = runtime(); + const now = Date.now(); + if (rt.tradeInFlightSince > 0) { + const heldMs = now - rt.tradeInFlightSince; + if (heldMs < TRADE_WEDGED_MS) { + console.log(`${tag} another batch in flight for ${Math.round(heldMs / 1000)}s — skipping`); + return null; + } + console.warn(`${tag} batch in flight for ${Math.round(heldMs / 60_000)}m looks wedged — proceeding without it`); + } + rt.tradeInFlightSince = now; + return now; +} + +function holdsTradeMutex(lease: number): boolean { + return runtime().tradeInFlightSince === lease; +} + +/** Only the holder unlocks: a wedged batch that finally returns must not + * release the batch that took over from it. */ +function releaseTradeMutex(lease: number): void { + if (holdsTradeMutex(lease)) runtime().tradeInFlightSince = 0; +} + /** Obfuscation stop for the current leg, in dollars. Infinity = no stop. * Day 1 of a cycle carries the $1,000 account stop (firstDayStop), min-day * (small) trades a 2×-target stop (minDayStop) — and the day after a losing @@ -134,6 +397,9 @@ function evaluateAccountForTrade(client: any, acc: { id: number; name: string; a if (isAccountDead(cash.amount, autoLiqThreshold)) return { ...base, eligible: false, reason: 'dead' }; if (!acc.active) return { ...base, eligible: false, reason: 'inactive (DLL)' }; if (client.positions[acc.id]) return { ...base, eligible: false, reason: 'already in trade' }; + // Another batch is placing this account's order right now; its position + // won't show until the fill comes back, so the check above can't see it. + if (runtime().accountsInFlight.has(acc.id)) return { ...base, eligible: false, reason: 'order in flight' }; // Obfuscation stops: a stop-out shows as realized P&L falling short of the // legs fired today — the day is booked; the make-back ride is tomorrow. @@ -210,9 +476,9 @@ function evaluateAccountForTrade(client: any, acc: { id: number; name: string; a // Random 1–20 minute delay added to the session open, picked once per Central // calendar day. Re-rolling on every check would move the open boundary between // polls, letting a trade slip through a moment the next check would reject. -const sessionOpenJitter = new Map(); - +// Held in runtime() so every bundle of this module agrees on the day's roll. function openJitterMinutes(centralDay: string): number { + const sessionOpenJitter = runtime().sessionOpenJitter; let jitter = sessionOpenJitter.get(centralDay); if (jitter === undefined) { jitter = 1 + Math.floor(Math.random() * 20); // 1–20 inclusive @@ -290,23 +556,46 @@ function isInNoTradeWindow(): boolean { // ── core trade logic ────────────────────────────────────────────────────────── -export async function runTrade(action: 'Buy' | 'Sell' | 'Auto', symbol: string) { +/** One account's outcome in a batch: its fill summary, or { status: 'error', reason }. */ +type LegResult = { status: string; [key: string]: unknown }; +type FirmTradeResults = { firm: string; results: LegResult[] }; +type TradeOutcome = Extract; + +export async function runTrade(action: 'Buy' | 'Sell' | 'Auto', symbol: string): Promise { + return (await runTradeBatch(action, symbol)).results; +} + +/** runTrade plus what it decided, which the tick records for getSchedulerHealth. */ +async function runTradeBatch( + action: 'Buy' | 'Sell' | 'Auto', + symbol: string, +): Promise<{ outcome: TradeOutcome; results: FirmTradeResults[] }> { + const lease = acquireTradeMutex('[auto-trade]'); + if (lease === null) return { outcome: 'skipped-in-flight', results: [] }; + try { + return await runTradeLocked(action, symbol, lease); + } finally { + releaseTradeMutex(lease); + } +} + +async function runTradeLocked( + action: 'Buy' | 'Sell' | 'Auto', + symbol: string, + lease: number, +): Promise<{ outcome: TradeOutcome; results: FirmTradeResults[] }> { if (isInNoTradeWindow()) { console.log('[auto-trade] CME market closed — skipping'); - return []; + return { outcome: 'closed', results: [] }; } - // Block if any account still has an open position (same gate as the scheduler tick) - const allClients = getClients(); - const allFirms = getTradableFirms(); - const openPositions = allFirms.reduce((count, firm) => { - const c = allClients.get(firm.id); - if (!c) return count; - return count + c.accountList.filter(acc => !!c.positions[acc.id]).length; - }, 0); + // Block if any account still has an open position (same gate as the scheduler tick). + // Ready clients only, like everything below: a stale snapshot must neither trade nor gate. + const { ready } = firmClients(); + const openPositions = countOpenPositions(ready); if (openPositions > 0) { console.log(`[auto-trade] ${openPositions} position(s) still open — skipping`); - return []; + return { outcome: 'positions-open', results: [] }; } // Resolve 'Auto' symbol once per batch so all accounts trade the same symbol @@ -324,9 +613,7 @@ export async function runTrade(action: 'Buy' | 'Sell' | 'Auto', symbol: string) if (!pointValue) throw new Error(`Unknown symbol: ${resolvedSymbol}`); const tradeDate = currentTradeDateCT(); - const maxConcurrent = parseInt(getSetting('max_concurrent_accounts') ?? '5', 10); - const firms = getTradableFirms(); - const clients = getClients(); + const maxConcurrent = maxConcurrentAccounts(); // ── Phase 1: collect ALL eligible accounts across ALL firms in parallel ── type EligibleItem = { @@ -346,9 +633,8 @@ export async function runTrade(action: 'Buy' | 'Sell' | 'Auto', symbol: string) const allEligible: EligibleItem[] = []; - await Promise.all(firms.map(async (firm) => { - const client = clients.get(firm.id); - if (!client || client.accountList.length === 0) return; + await Promise.all(ready.map(async ({ firm, client }) => { + if (client.accountList.length === 0) return; // Skip this firm entirely if the symbol is banned for it if (firm.bannedSymbols.includes(resolvedSymbol)) { @@ -356,26 +642,42 @@ export async function runTrade(action: 'Buy' | 'Sell' | 'Auto', symbol: string) return; } + // One firm's trouble (a contract lookup that throws, a malformed + // snapshot) skips that firm, never the batch — a rejection here would + // fail the Promise.all and every other firm would sit the tick out. + try { + const contract = await client.findFrontMonthContract(resolvedSymbol); + if (!contract) return; - const contract = await client.findFrontMonthContract(resolvedSymbol); - if (!contract) return; + const found: EligibleItem[] = []; + for (const acc of client.accountList) { + const ev = evaluateAccountForTrade(client, acc, tradeDate); + if (!ev || !ev.eligible || !ev.plan) continue; - for (const acc of client.accountList) { - const ev = evaluateAccountForTrade(client, acc, tradeDate); - if (!ev || !ev.eligible || !ev.plan) continue; - - allEligible.push({ - firmName: firm.name, client, acc, contract, - cash: ev.cash, dailyPnL: ev.dailyPnL, daysTraded: ev.daysTraded, - plan: ev.plan, isExtraDay: ev.isExtraDay, drain: ev.drain, - prefix: getAccountConfig(acc.id).accountType?.prefix ?? '', - }); + found.push({ + firmName: firm.name, client, acc, contract, + cash: ev.cash, dailyPnL: ev.dailyPnL, daysTraded: ev.daysTraded, + plan: ev.plan, isExtraDay: ev.isExtraDay, drain: ev.drain, + prefix: getAccountConfig(acc.id).accountType?.prefix ?? '', + }); + } + allEligible.push(...found); + } catch (err) { + console.error(`[auto-trade] ${firm.name}: skipped — ${errorMessage(err)}`); } })); if (allEligible.length === 0) { console.log('[auto-trade] no eligible accounts found'); - return []; + return { outcome: 'no-eligible', results: [] }; + } + + // Collecting took so long that another batch presumed this one wedged and + // took the mutex. That batch traded on fresher numbers than these, so + // firing now could trade an account twice. + if (!holdsTradeMutex(lease)) { + console.warn('[auto-trade] superseded while collecting accounts — abandoning this batch'); + return { outcome: 'skipped-in-flight', results: [] }; } // Take only the first batch — all fired simultaneously, no rolling pool. @@ -395,9 +697,12 @@ export async function runTrade(action: 'Buy' | 'Sell' | 'Auto', symbol: string) .sort((a, b) => b[1].length - a[1].length || a[0].localeCompare(b[0]))[0]; const batch = batchGroup.slice(0, maxConcurrent); console.log(`[auto-trade] ${allEligible.length} eligible account(s) across ${groups.size} prefix group(s) — firing batch of ${batch.length} ${batchPrefix || '(no prefix)'}`); + runtime().lastFireAt = Date.now(); // ── Phase 2: fire the batch simultaneously ── - const tradeResults = await Promise.allSettled(batch.map(async (item) => { + // Each account is marked in flight before anything is placed and released + // once its orders are done, so no overlapping batch can pick it meanwhile. + const tradeResults = await Promise.allSettled(batch.map((item) => withAccountInFlight(item.acc.id, async () => { const { client, acc, contract } = item; const cfg = getAccountConfig(acc.id).accountType!; @@ -443,15 +748,19 @@ export async function runTrade(action: 'Buy' | 'Sell' | 'Auto', symbol: string) minLegs - legsToday(acc.id, tradeDate), ) : item.plan.remainingToday; + // Counted before anything is sent so the runaway guard sees it while + // the batch is in flight; taken back only if the entry never reached + // the broker (here, or in sendEntry). recordLeg(acc.id, tradeDate, legAmount); - const inst = await legInstrument(client, item.drain, contract, pointValue); + const inst = await legInstrument(client, item.drain, contract, pointValue) + .catch((err: unknown) => { unrecordLeg(acc.id, tradeDate, legAmount); throw err; }); const legContract = inst.contract; const pv = inst.pointValue; const rawContracts = Math.max(1, Math.ceil(legAmount / 1000)); const contracts = cfg.maxPositionSize > 0 ? Math.min(rawContracts, cfg.maxPositionSize) : rawContracts; - const fill = await client.sendOrder(acc.id, legContract.name, contracts, resolvedAction, 'Market'); + const fill = await sendEntry('[auto-trade]', client, acc, legContract.name, contracts, resolvedAction, tradeDate, legAmount); // Wait briefly for the cash balance WebSocket update to reflect entry commission await new Promise(r => setTimeout(r, 1000)); @@ -481,9 +790,18 @@ export async function runTrade(action: 'Buy' | 'Sell' | 'Auto', symbol: string) try { exitOrder = await client.placeOcoExit(acc.id, legContract.name, contracts, exitAction, exitPrice, stopPrice); } catch (err) { - // The position is already open — never leave it exit-less. Fall - // back to the classic lone limit and shout about it. - console.error(`[auto-trade] ${acc.name}: placeoco REJECTED (${err instanceof Error ? err.message : err}) — falling back to lone limit exit, NO STOP`); + // The position is already open — never leave it exit-less when + // the pair never went in (refused, or not sent): fall back to + // the classic lone limit and shout about it. An unknown outcome + // (the ack timed out, the connection went after the send) is + // different: the pair may be working, and a lone limit on top + // doubles the exit — both filling flips the account into an + // opposite position nothing manages. Place nothing more then. + if (!isOrderNotPlaced(err)) { + console.error(`[auto-trade] ${acc.name}: placeoco outcome UNKNOWN (${errorMessage(err)}) — NOT placing a second exit; check this position's exits at the broker`); + throw new Error(`entry filled @ ${fill.price} but the exit's outcome is unknown — ${errorMessage(err)}`); + } + console.error(`[auto-trade] ${acc.name}: placeoco REJECTED (${errorMessage(err)}) — falling back to lone limit exit, NO STOP`); stopPrice = null; exitOrder = await client.placeOrderNoWait(acc.id, legContract.name, contracts, exitAction, 'Limit', exitPrice); } @@ -508,22 +826,28 @@ export async function runTrade(action: 'Buy' | 'Sell' | 'Auto', symbol: string) exitPrice, commission: entryCommission, }; - })); + }))); // Group results by firm for the response - const firmResultsMap = new Map(); + const firmResultsMap = new Map(); for (let i = 0; i < batch.length; i++) { const firmName = batch[i].firmName; if (!firmResultsMap.has(firmName)) firmResultsMap.set(firmName, []); const r = tradeResults[i]; + if (r.status === 'rejected') { + console.error(`[auto-trade] ${batch[i].acc.name} (${firmName}): failed — ${errorMessage(r.reason)}`); + } firmResultsMap.get(firmName)!.push( r.status === 'fulfilled' ? r.value - : { status: 'error', reason: (r.reason as any)?.message ?? String(r.reason) } + : { status: 'error', reason: errorMessage(r.reason) } ); } - return Array.from(firmResultsMap.entries()).map(([firm, results]) => ({ firm, results })); + return { + outcome: 'fired', + results: Array.from(firmResultsMap.entries()).map(([firm, results]) => ({ firm, results })), + }; } // ── copy trade ─────────────────────────────────────────────────────────────── @@ -535,14 +859,30 @@ export async function runTrade(action: 'Buy' | 'Sell' | 'Auto', symbol: string) */ export async function copyTrade() { console.log('[copy-trade] called'); + // Shares runTrade's mutex: a copy landing mid-batch would see the batch's + // accounts as still flat and could fill past max_concurrent_accounts. + const lease = acquireTradeMutex('[copy-trade]'); + if (lease === null) return []; + try { + return await copyTradeLocked(lease); + } finally { + releaseTradeMutex(lease); + } +} + +async function copyTradeLocked(lease: number): Promise { if (isInNoTradeWindow()) { console.log('[copy-trade] outside trading hours — skipping'); return []; } - const firms = getTradableFirms(); - const clients = getClients(); - const maxConcurrent = Math.max(1, parseInt(getSetting('max_concurrent_accounts') ?? '5', 10)); + // Ready clients only, as in runTrade: a stale snapshot must neither be + // copied from, counted against the slots, nor traded. + const { ready, unready } = firmClients(); + if (unready.length > 0) { + console.log(`[copy-trade] not synced — leaving out: ${unready.map((u) => u.firm.name).join(', ')}`); + } + const maxConcurrent = maxConcurrentAccounts(); // Find all accounts with open positions to determine direction + symbol let resolvedAction: 'Buy' | 'Sell' | null = null; @@ -554,9 +894,7 @@ export async function copyTrade() { let positionedClient: any = null; const positionedPrefixes = new Set(); - for (const firm of firms) { - const client = clients.get(firm.id); - if (!client) continue; + for (const { client } of ready) { for (const acc of client.accountList) { const pos = client.positions[acc.id]; if (!pos) continue; @@ -585,10 +923,15 @@ export async function copyTrade() { if (positionedClient) { for (const sym of enabledSymbols) { - const contract = await positionedClient.findFrontMonthContract(sym); - if (contract && contract.id === positionContractId) { - resolvedSymbol = sym; - break; + // A lookup that throws rules out that one symbol, not the copy. + try { + const contract = await positionedClient.findFrontMonthContract(sym); + if (contract && contract.id === positionContractId) { + resolvedSymbol = sym; + break; + } + } catch (err) { + console.error(`[copy-trade] ${sym}: contract lookup failed — ${errorMessage(err)}`); } } } @@ -629,10 +972,9 @@ export async function copyTrade() { }; const eligible: CopyItem[] = []; - await Promise.all(firms.map(async (firm) => { - const client = clients.get(firm.id); - if (!client || client.accountList.length === 0) { - console.log(`[copy-trade] ${firm.name}: no client or empty account list — skip firm`); + await Promise.all(ready.map(async ({ firm, client }) => { + if (client.accountList.length === 0) { + console.log(`[copy-trade] ${firm.name}: empty account list — skip firm`); return; } if (firm.bannedSymbols.includes(resolvedSymbol!)) { @@ -640,24 +982,31 @@ export async function copyTrade() { return; } - const contract = await client.findFrontMonthContract(resolvedSymbol!); - if (!contract) { - console.log(`[copy-trade] ${firm.name}: cannot resolve front-month contract for ${resolvedSymbol} — skip firm`); - return; - } + // Same isolation as runTrade: one firm's throw skips that firm only. + try { + const contract = await client.findFrontMonthContract(resolvedSymbol!); + if (!contract) { + console.log(`[copy-trade] ${firm.name}: cannot resolve front-month contract for ${resolvedSymbol} — skip firm`); + return; + } - for (const acc of client.accountList) { - const ev = evaluateAccountForTrade(client, acc, tradeDate); - if (!ev) { console.log(`[copy-trade] ${acc.name}: no matching config prefix — skip`); continue; } - if (!ev.eligible || !ev.plan) { console.log(`[copy-trade] ${acc.name}: ${ev.reason} — skip`); continue; } - // copyTrade has no extra-day branch — those accounts wait for the scheduler - if (ev.isExtraDay) { console.log(`[copy-trade] ${acc.name}: extra-day account — skip`); continue; } + const found: CopyItem[] = []; + for (const acc of client.accountList) { + const ev = evaluateAccountForTrade(client, acc, tradeDate); + if (!ev) { console.log(`[copy-trade] ${acc.name}: no matching config prefix — skip`); continue; } + if (!ev.eligible || !ev.plan) { console.log(`[copy-trade] ${acc.name}: ${ev.reason} — skip`); continue; } + // copyTrade has no extra-day branch — those accounts wait for the scheduler + if (ev.isExtraDay) { console.log(`[copy-trade] ${acc.name}: extra-day account — skip`); continue; } - eligible.push({ - firmName: firm.name, client, acc, contract, cash: ev.cash, dailyPnL: ev.dailyPnL, plan: ev.plan, - drain: ev.drain, - prefix: getAccountConfig(acc.id).accountType?.prefix ?? '', - }); + found.push({ + firmName: firm.name, client, acc, contract, cash: ev.cash, dailyPnL: ev.dailyPnL, plan: ev.plan, + drain: ev.drain, + prefix: getAccountConfig(acc.id).accountType?.prefix ?? '', + }); + } + eligible.push(...found); + } catch (err) { + console.error(`[copy-trade] ${firm.name}: skipped — ${errorMessage(err)}`); } })); @@ -671,10 +1020,17 @@ export async function copyTrade() { return []; } - console.log(`[copy-trade] copying ${resolvedAction} ${resolvedSymbol} to ${batch.length} account(s)`); + // Superseded while collecting (see runTrade) — this selection is stale. + if (!holdsTradeMutex(lease)) { + console.warn('[copy-trade] superseded while collecting accounts — abandoning this copy'); + return []; + } - // Fire orders (same as Phase 2 of runTrade) - const tradeResults = await Promise.allSettled(batch.map(async (item) => { + console.log(`[copy-trade] copying ${resolvedAction} ${resolvedSymbol} to ${batch.length} account(s)`); + runtime().lastFireAt = Date.now(); + + // Fire orders (same as Phase 2 of runTrade, accounts marked in flight the same way) + const tradeResults = await Promise.allSettled(batch.map((item) => withAccountInFlight(item.acc.id, async () => { const { client, acc, contract } = item; const cfg = getAccountConfig(acc.id).accountType!; @@ -686,15 +1042,17 @@ export async function copyTrade() { minLegs - legsToday(acc.id, tradeDate), ) : item.plan.remainingToday; + // Counted before sending, taken back only if the entry never reached the broker (see runTrade). recordLeg(acc.id, tradeDate, legAmount); - const inst = await legInstrument(client, item.drain, contract, pointValue); + const inst = await legInstrument(client, item.drain, contract, pointValue) + .catch((err: unknown) => { unrecordLeg(acc.id, tradeDate, legAmount); throw err; }); const legContract = inst.contract; const pv = inst.pointValue; const rawContracts = Math.max(1, Math.ceil(legAmount / 1000)); const contracts = cfg.maxPositionSize > 0 ? Math.min(rawContracts, cfg.maxPositionSize) : rawContracts; - const fill = await client.sendOrder(acc.id, legContract.name, contracts, resolvedAction!, 'Market'); + const fill = await sendEntry('[copy-trade]', client, acc, legContract.name, contracts, resolvedAction!, tradeDate, legAmount); await new Promise(r => setTimeout(r, 1000)); const updatedCash = client.accountCashBalances[acc.id] ?? { amount: 0, realizedPnL: 0 }; @@ -720,7 +1078,12 @@ export async function copyTrade() { try { exitOrder = await client.placeOcoExit(acc.id, legContract.name, contracts, exitAction, exitPrice, stopPrice); } catch (err) { - console.error(`[copy-trade] ${acc.name}: placeoco REJECTED (${err instanceof Error ? err.message : err}) — falling back to lone limit exit, NO STOP`); + // Lone-limit fallback only for a pair that never went in (see runTrade). + if (!isOrderNotPlaced(err)) { + console.error(`[copy-trade] ${acc.name}: placeoco outcome UNKNOWN (${errorMessage(err)}) — NOT placing a second exit; check this position's exits at the broker`); + throw new Error(`entry filled @ ${fill.price} but the exit's outcome is unknown — ${errorMessage(err)}`); + } + console.error(`[copy-trade] ${acc.name}: placeoco REJECTED (${errorMessage(err)}) — falling back to lone limit exit, NO STOP`); stopPrice = null; exitOrder = await client.placeOrderNoWait(acc.id, legContract.name, contracts, exitAction, 'Limit', exitPrice); } @@ -745,15 +1108,18 @@ export async function copyTrade() { exitPrice, commission: entryCommission, }; - })); + }))); const results: unknown[] = []; for (let i = 0; i < batch.length; i++) { const r = tradeResults[i]; + if (r.status === 'rejected') { + console.error(`[copy-trade] ${batch[i].acc.name} (${batch[i].firmName}): failed — ${errorMessage(r.reason)}`); + } results.push( r.status === 'fulfilled' ? r.value - : { status: 'error', reason: (r.reason as any)?.message ?? String(r.reason) } + : { status: 'error', reason: errorMessage(r.reason) } ); } return results; @@ -763,13 +1129,31 @@ export async function copyTrade() { /** Returns true if any configured account could still trade today (eligible for a next leg / extra-day, or has an open position). */ function hasRemainingConfiguredAccounts(): boolean { - const firms = getTradableFirms(); - const clients = getClients(); + const { ready, unready } = firmClients(); const tradeDate = currentTradeDateCT(); - for (const firm of firms) { - const client = clients.get(firm.id); - if (!client || client.accountList.length === 0) continue; + // A client that isn't ready can't say whether its accounts are done — its + // snapshot is missing or stale — so while one might hold any of ours, the + // scheduler stays up for it to rejoin. Only a client that never synced + // and knows no accounts is an unknown in full; one whose (stale) list + // shows none of ours can be ruled out. + // + // Only while the session is open, though. Outside it the unready firm + // couldn't trade anyway, and after the close its day is over. Unbounded, + // one firm that never comes back (a wrong password, a TopstepX key with + // no username) would keep a stop-after-all scheduler up for good, trading + // the ready firms every session after — exactly what the operator opted + // out of. (Before an open with every ready account already done, this + // stops rather than wait on the unready firm: the safe side.) + if (!isInNoTradeWindow()) { + for (const { client } of unready) { + if (client.lastSyncAt === 0 && client.accountList.length === 0) return true; + if (client.accountList.some((acc) => getAccountConfig(acc.id).accountType)) return true; + } + } + + for (const { client } of ready) { + if (client.accountList.length === 0) continue; for (const acc of client.accountList) { const cfg = getAccountConfig(acc.id).accountType; @@ -777,6 +1161,8 @@ function hasRemainingConfiguredAccounts(): boolean { // Account with open position = still in play if (client.positions[acc.id]) return true; + // Order being placed right now = still in play (evaluation calls it ineligible) + if (runtime().accountsInFlight.has(acc.id)) return true; const ev = evaluateAccountForTrade(client, acc, tradeDate); if (ev?.eligible) return true; @@ -810,6 +1196,21 @@ function getState(): SchedulerState { export function startScheduler(action: 'Buy' | 'Sell' | 'Auto', symbol: string, stopAfterAll: boolean = false) { const state = getState(); + // The settings reads and writes are the part that can throw, so they go + // before anything in memory changes: a failure then leaves whatever + // schedule was running as it was, never one marked running with its + // interval cleared and nothing scheduled. A non-numeric setting falls back + // to 60s — setInterval takes NaN as 1ms. + const configuredSecs = parseInt(getSetting('tick_interval_seconds') ?? '60', 10); + const intervalSecs = Math.max(5, Number.isFinite(configuredSecs) ? configuredSecs : 60); + + // Mirror to the settings table so a restart can pick the schedule back up. + // The interval itself is in-memory only; resumeSchedulerIfPersisted() recreates it. + setSetting('scheduler_running', '1'); + setSetting('scheduler_action', action); + setSetting('scheduler_symbol', symbol); + setSetting('scheduler_stop_after_all', stopAfterAll ? '1' : '0'); + // Clear any existing interval if (state.intervalId !== null) { clearInterval(state.intervalId); @@ -820,43 +1221,44 @@ export function startScheduler(action: 'Buy' | 'Sell' | 'Auto', symbol: string, state.running = true; state.stopAfterAll = stopAfterAll; - // Mirror to the settings table so a restart can pick the schedule back up. - // The interval itself is in-memory only; resumeSchedulerIfPersisted() recreates it. - setSetting('scheduler_running', '1'); - setSetting('scheduler_action', action); - setSetting('scheduler_symbol', symbol); - setSetting('scheduler_stop_after_all', stopAfterAll ? '1' : '0'); - const tick = async () => { if (!state.running) return; state.lastRun = new Date(); - - // Skip this tick until every client has completed its initial sync (positions are populated) - const clients = getClients(); - const firms = getTradableFirms(); - const notReady = firms.filter(f => { - const c = clients.get(f.id); - return c && !c.syncComplete; - }); - if (notReady.length > 0) { - console.log(`[scheduler] waiting for sync: ${notReady.map(f => f.name).join(', ')}`); - return; - } - - // Skip this tick if any account still has an open position from the previous batch - const openPositions = firms.reduce((count, firm) => { - const client = clients.get(firm.id); - if (!client) return count; - return count + client.accountList.filter(acc => !!client.positions[acc.id]).length; - }, 0); - if (openPositions > 0) { - console.log(`[scheduler] ${openPositions} position(s) still open — skipping tick`); - return; - } + const rt = runtime(); + const now = Date.now(); + rt.lastTickAt = now; try { - const results = await runTrade(state.action, state.symbol); - const filled = results.flatMap((r: any) => r.results ?? []).filter((r: any) => r.status === 'filled').length; + // Hold the tick while any firm's client is still inside its sync + // grace period (positions may not be populated yet); trade around + // firms that have been out past it (see assessReadiness). + const { ready, unready } = firmClients(now); + const { holding, quarantined } = assessReadiness(ready, unready, now); + rt.holding = holding.map((f) => f.name); + rt.quarantined = quarantined.map((q) => q.firm.name); + if (holding.length > 0) { + rt.lastTickOutcome = 'holding-for-sync'; + console.log(`[scheduler] waiting for sync: ${rt.holding.join(', ')}`); + return; + } + for (const { firm, unreadyMs } of quarantined) { + if (now - (rt.quarantineLoggedAt.get(firm.id) ?? 0) < QUARANTINE_LOG_EVERY_MS) continue; + rt.quarantineLoggedAt.set(firm.id, now); + console.warn(`[scheduler] ${firm.name} not synced for ${Math.round(unreadyMs / 60_000)}m — trading the other firms without it`); + } + + // Skip this tick if any account still has an open position from the + // previous batch — one batch fires only when every (ready) account is flat. + const openPositions = countOpenPositions(ready); + if (openPositions > 0) { + rt.lastTickOutcome = 'positions-open'; + console.log(`[scheduler] ${openPositions} position(s) still open — skipping tick`); + return; + } + + const { outcome, results } = await runTradeBatch(state.action, state.symbol); + rt.lastTickOutcome = outcome; + const filled = results.flatMap((r) => r.results).filter((r) => r.status === 'filled').length; if (filled > 0) console.log(`[scheduler] tick: ${filled} account(s) filled`); // Auto-stop if user opted in and no configured accounts can trade anymore @@ -865,11 +1267,11 @@ export function startScheduler(action: 'Buy' | 'Sell' | 'Auto', symbol: string, stopScheduler(); } } catch (err) { - console.error('[scheduler] tick error:', err); + rt.lastTickOutcome = `error: ${errorMessage(err)}`; + console.error(`[scheduler] tick error: ${errorMessage(err)}`); } }; - const intervalSecs = Math.max(5, parseInt(getSetting('tick_interval_seconds') ?? '60', 10)); state.intervalId = setInterval(tick, intervalSecs * 1_000); console.log(`[scheduler] started — ${action} ${symbol} every ${intervalSecs}s`); } @@ -881,6 +1283,17 @@ export function stopScheduler() { state.intervalId = null; } state.running = false; + + // Nothing watches readiness while stopped, so the grace clocks would be + // stale by the next start — that start gives every firm a fresh grace. + // Reset before the settings write, which can throw. + const rt = runtime(); + rt.unreadySince.clear(); + rt.quarantineLoggedAt.clear(); + rt.holding = []; + rt.quarantined = []; + rt.lastTickOutcome = 'not-running'; + setSetting('scheduler_running', '0'); console.log('[scheduler] stopped'); } @@ -888,9 +1301,10 @@ export function stopScheduler() { /** * Restart the schedule that was running before the process went down. * - * No sync-wait here on purpose: tick() already skips while any client reports - * !syncComplete, and again while any account holds an open position. So the - * worst case is a few logged no-op ticks until the clients finish syncing. + * No sync-wait here on purpose: tick() already holds while any client isn't + * ready (for up to UNREADY_GRACE_MS each), and again while any account holds + * an open position. So the worst case is a few logged no-op ticks until the + * clients finish syncing. */ export function resumeSchedulerIfPersisted(): void { if (getSetting('scheduler_running') !== '1') return; @@ -915,3 +1329,31 @@ export function getSchedulerStatus() { sessionOpenDelayMin: todaysSessionOpenDelay(), }; } + +/** + * Why the scheduler is or isn't trading, for health checks — separate from + * getSchedulerStatus, whose shape is what the master receives. `holding` are + * the firms the last tick waited on (inside their sync grace period); + * `quarantined` the ones past it, traded around. Timestamps are epoch ms, + * 0 = never; lastFireAt counts batches from any caller, not just the tick. + * Until the first tick after a start, the outcome still reads 'not-running'. + */ +export function getSchedulerHealth(): { + lastTickAt: number; + lastTickOutcome: TickOutcome; + lastFireAt: number; + holding: string[]; + quarantined: string[]; + tradeInFlightSince: number; +} { + const running = getState().running; + const rt = runtime(); + return { + lastTickAt: rt.lastTickAt, + lastTickOutcome: running ? rt.lastTickOutcome : 'not-running', + lastFireAt: rt.lastFireAt, + holding: running ? [...rt.holding] : [], + quarantined: running ? [...rt.quarantined] : [], + tradeInFlightSince: rt.tradeInFlightSince, + }; +} diff --git a/lib/clients.ts b/lib/clients.ts index 8c3211b..f4d3de3 100644 --- a/lib/clients.ts +++ b/lib/clients.ts @@ -1,23 +1,32 @@ import { TradovateClient } from './tradovate-class'; import { TopstepXClient } from './topstepx-class'; -import type { PlatformClient } from './platform-client'; +import { errorMessage, isClientReady, type PlatformClient } from './platform-client'; import { getInstruments } from './db'; import { getTradableFirms, startCatalogSync } from './firm-catalog'; import { resolveFirmAssignments } from './account-type-resolver'; import { resolveContracts } from './contract-resolver'; import { startReporter, sendReport } from './reporter'; import { startBackupSync } from './backup-sync'; -import { applyBrokerProxy } from './outbound-proxy'; +import { applyBrokerHttpDefaults, applyBrokerProxy } from './outbound-proxy'; import { startMasterLink } from './master-link'; import { resumeSchedulerIfPersisted } from './auto-trade'; // Use global to persist the client pool across HMR reloads in dev mode. // Keyed by the master's firm id, which is a uuid string. +// +// It is also what keeps the pool single in production: instrumentation.ts +// boots it from its own server bundle, and the route handlers can each hold a +// separate instance of this module. Registry state therefore lives here, never +// in module-level variables. const g = global as typeof globalThis & { __tradovateClients?: Map; __tradovateClientsInitialized?: boolean; __tradovateClientCreds?: Map; __contractResolverTimer?: ReturnType; + __contractResolveRetryTimer?: ReturnType; + __contractResolveRetries?: number; + __clientSupervisorTimer?: ReturnType; + __clientSupervision?: Map; }; function ensureMap(): Map { @@ -74,7 +83,14 @@ export function initClient( export function removeClient(id: string): void { const client = ensureMap().get(id); - client?.disconnect(); + // A throwing disconnect() must still take the client out of the pool: + // left in, half torn down, it would never reconnect — and the supervisor + // rebuilding it would fail the same way every time. + try { + client?.disconnect(); + } catch (err) { + console.error(`[clients] disconnect failed for firm ${id}: ${errorMessage(err)}`); + } ensureMap().delete(id); credsMap().delete(id); } @@ -127,11 +143,21 @@ export function reconcileClients(): void { export function getClients(): Map { const map = ensureMap(); + + // `next build` imports route modules to collect page data. Nothing there + // should call this, but a build that did must never dial the brokers or + // resume the scheduler — so hand back the empty pool and leave the flag + // unset, as if nothing had happened. + if (process.env.NEXT_PHASE === 'phase-production-build') return map; + if (!g.__tradovateClientsInitialized) { g.__tradovateClientsInitialized = true; try { - // Before anything dials out: put the operator's proxy (if any) - // under the broker clients. Master traffic stays direct by design. + // Before anything dials out: a timeout on every broker HTTP call, + // and the operator's proxy (if any) under the broker clients. + // Master traffic stays direct by design. (outbound-proxy applies + // both when it loads, too; these are idempotent.) + applyBrokerHttpDefaults(); applyBrokerProxy(); // Pull the catalog before anything else — without it there are no @@ -158,6 +184,9 @@ export function getClients(): Map { // Schedule daily resolve at midnight scheduleDailyResolve(); + // Rebuild any client that has been wedged for too long. + startClientSupervisor(); + // Start master dashboard reporter startReporter(); @@ -180,30 +209,67 @@ export function getClients(): Map { // ── Contract auto-resolve ──────────────────────────────────────────────────── +// With no authenticated Tradovate client yet — a cold catalog, a slow login, +// broker maintenance at boot — keep checking once a minute for half an hour +// instead of giving up until midnight. Missing it leaves the contract cache +// cold: the settings page shows nothing, TopstepX loses the resolver's +// front-month names, and the first trade pays for the lookups itself. +const CONTRACT_RESOLVE_RETRY_MS = 60_000; +const CONTRACT_RESOLVE_MAX_RETRIES = 30; + function triggerContractResolve(): void { + // A midnight run supersedes a pending retry; either way only one chain. + if (g.__contractResolveRetryTimer) clearTimeout(g.__contractResolveRetryTimer); + g.__contractResolveRetryTimer = undefined; + const map = ensureMap(); // Find first client with a valid access token (the Yahoo-driven resolver // confirms contract ids against Tradovate; TopstepX clients resolve their // own contracts but still reuse the resolver's front-month names). + // A token alone isn't enough: a client whose session died keeps its + // expired one, and resolving with that fails with nothing left to retry. let accessToken: string | null = null; for (const [, c] of map) { // eslint-disable-next-line @typescript-eslint/no-explicit-any const token = (c as any).accessInfo?.accessToken; - if (token) { accessToken = token; break; } + if (!token) continue; + let authenticated = true; + try { authenticated = c.health().authenticated; } catch { /* no health: trust the token */ } + if (authenticated) { accessToken = token; break; } } if (!accessToken) { - console.log('[contract-resolver] No authenticated Tradovate client yet — skipping resolve'); + const retries = (g.__contractResolveRetries ?? 0) + 1; + if (retries > CONTRACT_RESOLVE_MAX_RETRIES) { + console.log( + `[contract-resolver] No authenticated Tradovate client after ${CONTRACT_RESOLVE_MAX_RETRIES} retries` + + ' — giving up until the midnight resolve', + ); + g.__contractResolveRetries = 0; + return; + } + if (retries === 1) { + console.log('[contract-resolver] No authenticated Tradovate client yet — retrying every 60s'); + } + g.__contractResolveRetries = retries; + g.__contractResolveRetryTimer = setTimeout(() => triggerContractResolve(), CONTRACT_RESOLVE_RETRY_MS); return; } + g.__contractResolveRetries = 0; - const symbols = getInstruments().filter((i) => i.enabled).map((i) => i.symbol); - console.log(`[contract-resolver] Resolving ${symbols.length} instruments...`); - resolveContracts(symbols, accessToken) - .then((results) => { - const rolled = Object.entries(results).filter(([, v]) => v?.alternative); - console.log(`[contract-resolver] Done — ${rolled.length} contract(s) rolled via price match`); - }) - .catch((err) => console.error('[contract-resolver] Resolve failed:', err)); + // Runs from timers: a throw here (the instruments read) would escape as + // an uncaught exception. + try { + const symbols = getInstruments().filter((i) => i.enabled).map((i) => i.symbol); + console.log(`[contract-resolver] Resolving ${symbols.length} instruments...`); + resolveContracts(symbols, accessToken) + .then((results) => { + const rolled = Object.entries(results).filter(([, v]) => v?.alternative); + console.log(`[contract-resolver] Done — ${rolled.length} contract(s) rolled via price match`); + }) + .catch((err) => console.error(`[contract-resolver] Resolve failed: ${errorMessage(err)}`)); + } catch (err) { + console.error(`[contract-resolver] Resolve failed: ${errorMessage(err)}`); + } } function scheduleDailyResolve(): void { @@ -215,7 +281,179 @@ function scheduleDailyResolve(): void { const now = new Date(); if (now.getHours() === 0 && now.getMinutes() === 0) { console.log('[contract-resolver] Midnight resolve triggered'); + // Its own half hour of retries, not what's left of a chain it cuts off. + g.__contractResolveRetries = 0; triggerContractResolve(); } }, 60_000); } + +// ── Client supervision ─────────────────────────────────────────────────────── + +/** + * Rebuilds a client that has been unable to trade for a long time. + * + * The clients reconnect themselves, but only along the paths their authors + * foresaw. A re-login that fails in an unexpected way, an await that never + * settles, a socket that dies without a close event — any of those used to + * leave a client wedged for the life of the process, and because the + * scheduler waits on every firm, one wedged firm stopped them all. After a + * weekend of broker maintenance that meant a pm2 restart. A fresh client is + * exactly what the restart gave it, so the registry hands one out itself: + * same firm, same credentials, new instance. + * + * "Unready" is isClientReady() saying no — the same rule the scheduler and + * /api/health use: not synced, or no good sync in the last 5 minutes. + * + * Deliberately slow — 20 minutes unready before a rebuild, and at most one + * rebuild per firm per 20 minutes. The clients' own retries get the first go, + * and a broker that is really down, or rate-limiting logins, must not be + * hammered with new sessions. Some clients are never replaced while unready + * (see rebuildHold): one whose own login retry is pending, and a TopstepX + * one holding a position. + */ +const SUPERVISOR_INTERVAL_MS = 60_000; +const REBUILD_AFTER_UNREADY_MS = 20 * 60_000; +const REBUILD_COOLDOWN_MS = 20 * 60_000; + +/** The message TopstepXClient records when a firm has an API key but no + * username. That client never even tries to log in, so a rebuild changes + * nothing — adding the username changes the credentials, and + * reconcileClients rebuilds it then. */ +const TOPSTEPX_USERNAME_MISSING = /username missing/i; + +interface SupervisionEntry { + /** The instance being judged. A different one (rebuilt here, or replaced + * by reconcileClients after a credentials change) starts a fresh clock. */ + client: PlatformClient; + /** When it was first seen unready (0 = ready at the last check). */ + unhealthySince: number; + lastRebuildAt: number; + /** Last time a due rebuild was held back and said so (log rate limit). */ + holdLoggedAt: number; +} + +function supervisionMap(): Map { + if (!g.__clientSupervision) g.__clientSupervision = new Map(); + return g.__clientSupervision; +} + +function startClientSupervisor(): void { + // HMR-safe, and safe when resetClients() re-runs the bootstrap. + if (g.__clientSupervisorTimer) clearInterval(g.__clientSupervisorTimer); + g.__clientSupervisorTimer = setInterval(superviseClients, SUPERVISOR_INTERVAL_MS); +} + +/** Why a client isn't ready, short enough for one log line. Never the token. */ +function describeUnready(client: PlatformClient, now: number): string { + const parts: string[] = []; + try { + const health = client.health(); + parts.push(`transport ${health.transport}`); + if (health.lastLoginError) parts.push(`login error: ${health.lastLoginError.slice(0, 160)}`); + } catch { + parts.push('health unavailable'); + } + parts.push(client.lastSyncAt > 0 + ? `last good sync ${Math.round((now - client.lastSyncAt) / 60_000)}m ago` + : 'never synced'); + return parts.join(', '); +} + +/** + * Why replacing this client now would do more harm than good, or null. + * + * - Its login keeps failing and its own retry is already scheduled. Nothing + * in the instance is broken — the broker is down, refusing the + * credentials, or rate-limiting — and a fresh instance would fail the same + * way while starting its backoff over at seconds: a burst of logins after + * every rebuild instead of one every few minutes, which is how a login + * rate-limit turns into a captcha lockout. + * - TopstepX with a position open. The gateway has no OCO, so the client + * itself cancels the surviving exit leg when the other fills; a fresh + * instance knows nothing of those pairs, and the survivor would be left + * working at the broker after the position closes — an order that can open + * a position nobody asked for. It waits until the account is flat. + */ +function rebuildHold(platform: string, client: PlatformClient, now: number): string | null { + try { + const health = client.health(); + if (health.consecutiveLoginFailures > 0 && health.nextReconnectAt > now) { + return `its login retry is due in ${Math.max(1, Math.round((health.nextReconnectAt - now) / 60_000))}m` + + ` (${health.consecutiveLoginFailures} failure(s) in a row)`; + } + } catch { /* no health: judge it on the rest */ } + if (platform === 'topstepx' && Object.keys(client.positions ?? {}).length > 0) { + return 'it holds an open position (TopstepX exit brackets are tracked only inside the client)'; + } + return null; +} + +function superviseClients(): void { + // Runs inside setInterval: nothing may escape, or the process logs an + // uncaught exception and the supervisor keeps firing into the same fault. + try { + const map = ensureMap(); + const book = supervisionMap(); + const now = Date.now(); + + // A firm that left the registry takes its bookkeeping with it. + for (const id of [...book.keys()]) { + if (!map.has(id)) book.delete(id); + } + + let firms: ReturnType | null = null; + for (const [id, client] of [...map]) { + try { + let entry = book.get(id); + if (!entry) { + entry = { client, unhealthySince: 0, lastRebuildAt: 0, holdLoggedAt: 0 }; + book.set(id, entry); + } else if (entry.client !== client) { + entry.client = client; + entry.unhealthySince = 0; + } + + if (isClientReady(client, now)) { + entry.unhealthySince = 0; + continue; + } + if (!entry.unhealthySince) { + entry.unhealthySince = now; + continue; + } + if (now - entry.unhealthySince < REBUILD_AFTER_UNREADY_MS) continue; + if (now - entry.lastRebuildAt < REBUILD_COOLDOWN_MS) continue; + + // Only read when a rebuild is actually due — it hits the DB. + firms ??= getTradableFirms(); + const firm = firms.find((f) => f.id === id); + if (!firm) continue; // no longer tradable: reconcileClients drops it + if (firm.platform === 'topstepx' + && TOPSTEPX_USERNAME_MISSING.test(client.lastFetchErrors?.['login'] ?? '')) continue; + + const minutes = Math.round((now - entry.unhealthySince) / 60_000); + const hold = rebuildHold(firm.platform, client, now); + if (hold) { + if (now - entry.holdLoggedAt >= REBUILD_COOLDOWN_MS) { + entry.holdLoggedAt = now; + console.log(`[clients] ${firm.name} unhealthy for ${minutes}m — not rebuilding while ${hold}`); + } + continue; + } + console.log( + `[clients] ${firm.name} unhealthy for ${minutes}m (${describeUnready(client, now)})` + + ' — rebuilding client', + ); + entry.lastRebuildAt = now; + removeClient(id); + entry.client = initClient(firm.id, firm.name, firm.platform, firm); + entry.unhealthySince = now; + } catch (err) { + console.error(`[clients] supervisor check failed for firm ${id}: ${errorMessage(err)}`); + } + } + } catch (err) { + console.error(`[clients] supervisor tick failed: ${errorMessage(err)}`); + } +} diff --git a/lib/contract-resolver.ts b/lib/contract-resolver.ts index 3071f34..df6986d 100644 --- a/lib/contract-resolver.ts +++ b/lib/contract-resolver.ts @@ -13,6 +13,7 @@ */ import axios from 'axios'; +import { errorMessage } from './platform-client'; interface ContractInfo { id: number; @@ -31,7 +32,13 @@ const PROBE_WINDOW = 6; // ── Cache ──────────────────────────────────────────────────────────────────── -const cache = new Map(); +// On globalThis, not module scope: instrumentation.ts boots the process from its +// own bundle, so the boot and midnight resolves would otherwise fill a cache the +// route handlers (settings page, manual trades) never see. +const _g = globalThis as typeof globalThis & { + __contractCache?: Map; +}; +const cache = (_g.__contractCache ??= new Map()); const CACHE_TTL_MS = 30 * 60 * 1000; // 30 minutes export function getCachedContract(symbol: string): ResolvedContract | null { @@ -205,7 +212,8 @@ export async function resolveContracts( console.log(`[contract-resolver] ${sym}: ${match.name} resolved`); } } catch (err) { - console.error(`[contract-resolver] ${sym}: Tradovate lookup failed`, err); + // Never the whole error: an AxiosError carries the Bearer token in its config. + console.error(`[contract-resolver] ${sym}: Tradovate lookup failed — ${errorMessage(err)}`); results[sym] = null; } })); diff --git a/lib/master-link.ts b/lib/master-link.ts index b8f92c7..7f379f1 100644 --- a/lib/master-link.ts +++ b/lib/master-link.ts @@ -99,14 +99,18 @@ async function execute(command: Command): Promise { if (command.symbol !== 'Auto' && !POINT_VALUES[command.symbol]) { throw new Error(`Unknown symbol: ${command.symbol}`); } - // Same order as POST /api/trade: fire once now, then leave the - // scheduler running to pick up accounts that were busy. - const results = await runTrade(command.action, command.symbol); + // Same order as POST /api/trade: (re)start the scheduler first, + // then fire once now — the scheduler picks up accounts that were + // busy. Starting it first means a batch that throws or runs long + // can never leave it unstarted. It can't double-fire against this + // batch: the tick waits for flat positions, skips while this batch + // holds the trade mutex, and never picks an account in flight. startScheduler(command.action, command.symbol, command.stopAfterAll ?? false); + const results = await runTrade(command.action, command.symbol); const filled = results - .flatMap((r: any) => r.results ?? []) - .filter((r: any) => r.status === 'filled').length; + .flatMap((r) => r.results) + .filter((r) => r.status === 'filled').length; return { filled, scheduler: getSchedulerStatus() }; } diff --git a/lib/outbound-proxy.ts b/lib/outbound-proxy.ts index bbc4cb7..0e1dd2f 100644 --- a/lib/outbound-proxy.ts +++ b/lib/outbound-proxy.ts @@ -12,6 +12,19 @@ * Configured by the `proxy_url` setting: http://, https://, socks5:// or * socks5h:// with optional user:pass. Read once per process — changing it * requires a restart, which the settings UI says out loud. Blank = direct. + * + * Also home of the broker HTTP timeout (applyBrokerHttpDefaults): axios waits + * forever by default, and one broker request that never answers — a + * maintenance window that accepts the connection and goes silent — used to + * wedge a re-login or sync await for the life of the process. Same scoping + * argument as the proxy: every axios call here is broker/quote traffic, so the + * process default is the scoped choice. It covers axios only; websockets and + * the SignalR hub keep their own liveness handling in the clients. + * + * Both defaults are applied when this module loads. Every broker client + * imports it, and that matters: each server bundle (instrumentation.ts, the + * route handlers) can carry its own copy of axios, and a copy that never had + * the defaults applied would dial the brokers direct, with no timeout. */ import axios from 'axios'; @@ -30,7 +43,6 @@ interface ProxyState { dispatcher: Dispatcher | null; httpAgent: Agent | null; httpsAgent: Agent | null; - applied: boolean; } // HMR-safe, and the "read once per process" contract lives here. @@ -39,7 +51,7 @@ const g = globalThis as typeof globalThis & { __outboundProxy?: ProxyState }; function state(): ProxyState { if (g.__outboundProxy) return g.__outboundProxy; - const s: ProxyState = { url: null, dispatcher: null, httpAgent: null, httpsAgent: null, applied: false }; + const s: ProxyState = { url: null, dispatcher: null, httpAgent: null, httpsAgent: null }; g.__outboundProxy = s; const raw = getSetting('proxy_url')?.trim(); @@ -49,7 +61,8 @@ function state(): ProxyState { try { url = new URL(raw); } catch { - console.error(`[proxy] proxy_url is not a URL — trading DIRECT: ${raw}`); + // Not echoed: the value carries the proxy's user:password. + console.error('[proxy] proxy_url is not a URL — trading DIRECT'); return s; } if (!SCHEMES.includes(url.protocol as (typeof SCHEMES)[number])) { @@ -57,22 +70,42 @@ function state(): ProxyState { return s; } - const socks = url.protocol.startsWith('socks'); - if (socks) { - s.dispatcher = socksDispatcher({ - type: 5, - host: url.hostname, - port: Number(url.port || 1080), - userId: url.username ? decodeURIComponent(url.username) : undefined, - password: url.password ? decodeURIComponent(url.password) : undefined, - }); - const agent = new SocksProxyAgent(url); - s.httpAgent = agent; - s.httpsAgent = agent; - } else { - s.dispatcher = new ProxyAgent(url.href); - s.httpAgent = new HttpProxyAgent(url); - s.httpsAgent = new HttpsProxyAgent(url); + // A literal '%' in the user or password (one not starting a %XX escape) + // makes decodeURIComponent throw — here, and inside every proxy library + // below. Escape it so the password is used as typed. This now runs when + // the module loads, where a throw would fail every route that imports it. + const escapeStray = (part: string) => part.replace(/%(?![0-9A-Fa-f]{2})/g, '%25'); + url.username = escapeStray(url.username); + url.password = escapeStray(url.password); + + try { + const socks = url.protocol.startsWith('socks'); + if (socks) { + const dispatcher = socksDispatcher({ + type: 5, + host: url.hostname, + port: Number(url.port || 1080), + userId: url.username ? decodeURIComponent(url.username) : undefined, + password: url.password ? decodeURIComponent(url.password) : undefined, + }); + const agent = new SocksProxyAgent(url); + s.dispatcher = dispatcher; + s.httpAgent = agent; + s.httpsAgent = agent; + } else { + const dispatcher = new ProxyAgent(url.href); + const httpAgent = new HttpProxyAgent(url); + const httpsAgent = new HttpsProxyAgent(url); + s.dispatcher = dispatcher; + s.httpAgent = httpAgent; + s.httpsAgent = httpsAgent; + } + } catch (err) { + // All or nothing — nothing is kept until every piece is built: half a + // proxy (REST proxied, sockets direct) would be worse than the + // documented bad-setting behavior. + console.error(`[proxy] proxy_url could not be set up — trading DIRECT: ${err instanceof Error ? err.message : String(err)}`); + return s; } s.url = url; console.log(`[proxy] broker traffic will use ${url.protocol}//${url.host} (master link stays direct)`); @@ -87,16 +120,52 @@ export function getProxyUrl(): string | null { /** Puts the proxy under every axios call. All axios usage in this app is * broker/quote traffic (TopstepX, Tradovate REST, contract resolver), so the * process-wide default is the scoped choice, not a global one. Idempotent; - * called first in the getClients() bootstrap. */ + * runs when this module loads and again first in the getClients() bootstrap. + * + * "Already applied" is judged on the axios defaults themselves, not a + * process-wide flag: the proxy state is shared through globalThis, but each + * bundle's copy of axios has its own defaults, and a global flag set by the + * first copy would leave every other one dialling direct. */ export function applyBrokerProxy(): void { const s = state(); - if (s.applied || !s.url) return; - s.applied = true; + if (!s.url || axios.defaults.httpsAgent === s.httpsAgent) return; axios.defaults.proxy = false; // agents own the routing; ignore env HTTP(S)_PROXY handling axios.defaults.httpAgent = s.httpAgent!; axios.defaults.httpsAgent = s.httpsAgent!; } +/** Idle limit for one broker HTTP request. axios applies it as a socket + * inactivity timeout, so it bounds a server that has gone silent, not a + * big report that is still streaming. Nothing legitimate comes close: the + * Tradovate report endpoints answer with a p-ticket and are polled (each + * poll its own short request, already capped at 60s per call), logins and + * REST calls answer in seconds. A per-call `timeout` still overrides it. + * + * The clock starts once the socket has connected — a connect that never + * completes is left to the OS connect timeout. Calls that can't wait that + * long (the Tradovate login and renewal) add an AbortSignal of their own. */ +export const BROKER_HTTP_TIMEOUT_MS = 60_000; + +/** Gives axios a timeout unless one is already set (axios's own default, 0, + * means wait forever). Idempotent; runs when this module loads and at the + * top of the getClients() bootstrap. Timeouts surface as ECONNABORTED, which + * the Tradovate report fetch already retries as transient. */ +export function applyBrokerHttpDefaults(): void { + if (!axios.defaults.timeout) axios.defaults.timeout = BROKER_HTTP_TIMEOUT_MS; +} + +// At load, so no broker request from this bundle can go out before them. The +// proxy is skipped during `next build`: the build imports route modules to +// collect page data, and it dials nothing that would need one. Nothing may +// throw out of here: a module that fails to load takes every route that +// imports it (the whole trading side) down with it. +applyBrokerHttpDefaults(); +try { + if (process.env.NEXT_PHASE !== 'phase-production-build') applyBrokerProxy(); +} catch (err) { + console.error(`[proxy] could not apply proxy_url at load: ${err instanceof Error ? err.message : String(err)}`); +} + /** A websocket to a BROKER endpoint: the built-in (undici) WebSocket, dialled * through the proxy when one is set. The master link keeps using plain * `new WebSocket(...)` — do not use this for it. */ diff --git a/lib/platform-client.ts b/lib/platform-client.ts index 48bc40e..4e739e9 100644 --- a/lib/platform-client.ts +++ b/lib/platform-client.ts @@ -43,6 +43,8 @@ export interface PlatformClient { fetchDaysComplete: boolean; syncComplete: boolean; + /** Epoch ms of the last sync that came back good (0 = never). */ + lastSyncAt: number; lastFetchErrors: Record; lastFetchRaw: Record; recentEntityEvents: { entityType: string; eventType: string; entity: unknown; ts: number }[]; @@ -81,6 +83,76 @@ export interface PlatformClient { disconnect(): void; onFill(orderId: number, callback: (fill: Record) => void): void; + health(): ClientHealth; +} + +/** + * Liveness snapshot of one client — what /api/health shows, what the registry + * supervisor judges, and the numbers to read before restarting a broken box. + * Timestamps are epoch ms, 0 = never. Never carries tokens or credentials. + */ +export interface ClientHealth { + platform: 'tradovate' | 'topstepx'; + /** Holding a session token right now. */ + authenticated: boolean; + /** Tradovate: socket readyState name; TopstepX: hub state. 'none' = no transport. */ + transport: string; + syncComplete: boolean; + lastSyncAt: number; + lastLoginAt: number; + lastLoginError: string | null; + consecutiveLoginFailures: number; + /** When the next reconnect/login attempt fires (0 = none scheduled). */ + nextReconnectAt: number; + tokenExpiresAt: number; + /** Last frame from the broker's realtime channel (socket or hub). */ + lastInboundAt: number; + /** Requests still waiting on a broker answer (acks, fills). */ + pendingRequests: number; + /** When the in-flight sync started (0 = none in flight). */ + syncInFlightSince: number; + /** When the in-flight daily P&L report fetch started (0 = none in flight). */ + daysFetchInFlightSince: number; +} + +/** A sync older than this means positions and balances can't be traded on. */ +export const SYNC_STALE_MS = 5 * 60 * 1000; + +/** The one readiness rule shared by the scheduler, the supervisor and /api/health: + * synced, and synced recently. */ +export function isClientReady(client: PlatformClient, now = Date.now()): boolean { + return client.syncComplete && client.lastSyncAt > 0 && now - client.lastSyncAt < SYNC_STALE_MS; +} + +/** + * Thrown when a client KNOWS an order never reached the broker: no live + * connection, the broker refused it, or it failed before anything was sent. + * Anything ambiguous (a timeout waiting for an ack or a fill — it may still + * have filled) must stay a plain Error. The scheduler un-counts a leg only for + * this one, so a wrong "not placed" could let an account trade twice. + */ +export class OrderNotPlacedError extends Error { + readonly orderNotPlaced = true; + constructor(message: string) { + super(message); + this.name = 'OrderNotPlacedError'; + } +} + +/** Duck-typed so it survives module duplication across route bundles. */ +export function isOrderNotPlaced(err: unknown): boolean { + return !!err && typeof err === 'object' && (err as { orderNotPlaced?: unknown }).orderNotPlaced === true; +} + +/** An error's message without the rest of it. AxiosErrors carry the request + * config — Tradovate's login body (the password) and Bearer tokens — so they + * must never be logged whole. */ +export function errorMessage(err: unknown): string { + if (!err || typeof err !== 'object') return String(err); + const e = err as { message?: unknown; code?: unknown; response?: { status?: unknown } }; + const status = typeof e.response?.status === 'number' ? ` (HTTP ${e.response.status})` : ''; + const code = typeof e.code === 'string' && !String(e.message ?? '').includes(e.code) ? ` [${e.code}]` : ''; + return `${typeof e.message === 'string' ? e.message : String(err)}${status}${code}`; } /** lastFetchErrors note for a Tradovate account whose Cash History is diff --git a/lib/topstepx-class.ts b/lib/topstepx-class.ts index 72f3d12..f93868b 100644 --- a/lib/topstepx-class.ts +++ b/lib/topstepx-class.ts @@ -27,11 +27,88 @@ import { getCachedContract } from './contract-resolver'; import { signalrWebSocket } from './outbound-proxy'; import { saveDailyPnL, loadDailyPnL, loadAccountMeta, saveAccountMeta, loadFundTransactions, saveFundTransactions, saveEndedCycleStart } from './db'; import { getAccountConfig } from './account-type-resolver'; -import type { PlatformClient, PlatformAccount, PlatformContract } from './platform-client'; +import type { PlatformClient, PlatformAccount, PlatformContract, ClientHealth } from './platform-client'; +import { OrderNotPlacedError, isOrderNotPlaced, errorMessage } from './platform-client'; const API_BASE = 'https://api.topstepx.com'; const USER_HUB = 'https://rtc.topstepx.com/hubs/user'; +/** + * Self-healing timings. The weekend failure was a chain of awaits with no + * deadline (hub.start, hub.invoke, a re-login) plus a re-login that tore down + * the sync loop first — one wedged await left the firm "waiting for sync" + * until PM2 restarted the process. Every hub await now has a deadline, sync + * and hub setup detect their own wedges, and a watchdog keeps the client from + * ever sitting with no session and nothing scheduled to get one. + */ +const HUB_START_TIMEOUT_MS = 30_000; +const HUB_INVOKE_TIMEOUT_MS = 15_000; +/** Automatic reconnect gives up by itself within ~45s (0/2/10/30s retries); + * a hub still Connecting/Reconnecting after this is stuck, not retrying. */ +const HUB_STALL_MS = 2 * 60_000; +/** A hub setup or REST sync still running after this is wedged — start over. */ +const HUB_SETUP_WEDGED_MS = 3 * 60_000; +const SYNC_WEDGED_MS = 3 * 60_000; +/** Failed logins back off 60s, 120s, … up to 10 min, ±20% so a fleet of + * boxes coming back from the same outage doesn't log in in lockstep. */ +const LOGIN_RETRY_BASE_MS = 60_000; +const LOGIN_RETRY_MAX_MS = 10 * 60_000; +const WATCHDOG_INTERVAL_MS = 60_000; +/** Logged in longer than this with no good sync for as long → the session is + * presumed dead in a way validate didn't catch; log in again. */ +const SESSION_STALE_MS = 10 * 60_000; +/** Auth calls get their own short deadlines: a login that never answers + * would hold loginInFlight — and with it every retry — forever. */ +const LOGIN_TIMEOUT_MS = 30_000; +const VALIDATE_TIMEOUT_MS = 15_000; +const TOKEN_LIFETIME_MS = 24 * 60 * 60 * 1000; +/** Hard cap on every other REST call. axios's own timeout is a socket-idle + * timer that only starts once the socket has connected, so a proxy CONNECT + * that never answers hangs the call — and any single-flight flag it holds, + * like the daily P&L fetch's — for good. Aborting (rather than racing) also + * destroys a request that hasn't been written yet, so an order can't go out + * minutes after its caller gave up on it. Longer than the idle timeout, so + * it only bites where that one can't. */ +const REST_DEADLINE_MS = 90_000; + +/** Rejects if `promise` hasn't settled within `ms`. The underlying operation + * keeps running (callers discard whatever it belonged to); Promise.race + * subscribes to it, so a late rejection never surfaces as unhandled. */ +function withTimeout(promise: Promise, ms: number, label: string): Promise { + let timer: ReturnType | undefined; + const deadline = new Promise((_, reject) => { + timer = setTimeout(() => reject(new Error(`${label} timed out after ${Math.round(ms / 1000)}s`)), ms); + }); + return Promise.race([promise, deadline]).finally(() => clearTimeout(timer)); +} + +/** + * The gateway answered and said no (`success: false`) — as opposed to a + * transport failure, where the request may or may not have been acted on. + * placeOrder turns only this (and pre-handler HTTP refusals) into + * OrderNotPlacedError. + */ +class GatewayRefusedError extends Error { + readonly errorCode: number | undefined; + /** Set when a refusal still names an order — that order exists. */ + readonly orderId: number | undefined; + constructor(message: string, errorCode: number | undefined, orderId: number | undefined) { + super(message); + this.name = 'GatewayRefusedError'; + this.errorCode = errorCode; + this.orderId = orderId; + } +} + +/** PlaceOrderErrorCode 6 (OrderPending) and 7 (UnknownError) don't say the + * order wasn't created, so a refusal carrying them stays ambiguous. */ +const AMBIGUOUS_PLACE_ERRORS = new Set([6, 7]); + +/** HTTP answers given before any order handler runs (by the gateway's + * middleware or the Cloudflare edge in front of it): malformed request, + * missing/expired token, forbidden, rate-limited. Nothing was placed. */ +const REFUSED_BEFORE_HANDLER = new Set([400, 401, 403, 429]); + interface TsxContract { id: string; name: string; @@ -87,6 +164,8 @@ export class TopstepXClient implements PlatformClient { public fetchDaysComplete = false; public syncComplete = false; + /** Epoch ms of the last sync that came back good (0 = never). */ + public lastSyncAt = 0; public lastFetchErrors: Record = {}; public lastFetchRaw: Record = {}; public recentEntityEvents: { entityType: string; eventType: string; entity: unknown; ts: number }[] = []; @@ -96,19 +175,51 @@ export class TopstepXClient implements PlatformClient { private callbackOnSyncRequest: () => Promise; private hub: signalR.HubConnection | null = null; private subscribedAccountIds = new Set(); + /** SubscribeAccounts done on the current hub connection. */ + private accountsFeedSubscribed = false; + + /** ensureHub is single-flight; the generation lets a flight declared + * wedged be superseded without its late finally clearing the new flag. */ + private hubSetupInFlight = false; + private hubSetupSince = 0; + private hubSetupGeneration = 0; + /** ensureHub was asked for while a flight ran (e.g. the first sync's + * accounts landed mid-start) — run once more if the hub came up. */ + private hubRecheckWanted = false; + /** When the hub was first seen Connecting/Reconnecting/Disconnecting (0 = it wasn't). */ + private hubStalledSince = 0; + /** Last frame from the hub (event, invoke answer, handshake). */ + private lastInboundAt = 0; private syncInterval: ReturnType | null = null; private tokenRenewalInterval: ReturnType | null = null; private ocoPollInterval: ReturnType | null = null; private loginRetryTimer: ReturnType | null = null; + private watchdogInterval: ReturnType | null = null; private intentionalDisconnect = false; + + /** requestSync is single-flight too, with the same generation trick. */ private syncInFlight = false; + private syncInFlightSince = 0; + private syncGeneration = 0; + + private loginInFlight = false; + private lastLoginAt = 0; + private lastLoginError: string | null = null; + private consecutiveLoginFailures = 0; + private nextReconnectAt = 0; + /** When the current token was issued or last renewed (24h lifetime). */ + private tokenRefreshedAt = 0; /** Same hourly-with-retry throttle the Tradovate client uses for reports. */ private daysFetchInFlight = false; + private daysFetchInFlightSince = 0; private lastDaysFetchStartedAt = 0; private lastDaysFetchSucceededAt = 0; private nextDaysFetchRetryAt = 0; + /** The first full daily P&L pass has landed (cache fallbacks included). + * No sync counts as good before it — see requestSync. */ + private historyLoaded = false; /** Trades seen for the current CT trade date, per account — realizedPnL is * recomputed from these on every REST sync and every live trade event. */ @@ -134,7 +245,8 @@ export class TopstepXClient implements PlatformClient { this.name = name; this.apiKey = apiKey; this.callbackOnSyncRequest = callbackOnSyncRequest; - this.connectAndAuth(); + this.watchdogInterval = setInterval(() => this.watchdog(), WATCHDOG_INTERVAL_MS); + void this.connectAndAuth(); } // ── auth ───────────────────────────────────────────────────────────────── @@ -144,16 +256,48 @@ export class TopstepXClient implements PlatformClient { } private async post(path: string, body: unknown): Promise { - const res = await axios.post(`${API_BASE}${path}`, body, { headers: this.authHeaders() }); - const data = res.data as T & { success?: boolean; errorMessage?: string | null; errorCode?: number }; - if (data && data.success === false) { - throw new Error(`${path} failed: ${data.errorMessage ?? `errorCode=${data.errorCode}`}`); + const res = await axios.post(`${API_BASE}${path}`, body, { + headers: this.authHeaders(), + signal: AbortSignal.timeout(REST_DEADLINE_MS), + }).catch((err: unknown) => { + // The deadline abort surfaces as a bare "canceled" — say what it + // was. Still a plain Error: for an order it stays ambiguous. + if (axios.isCancel(err)) throw new Error(`${path} timed out after ${REST_DEADLINE_MS / 1000}s`); + throw err; + }); + const data = res.data as T & { success?: boolean; errorMessage?: string | null; errorCode?: number; orderId?: number | null }; + // The gateway answers every call with a JSON object. Anything else — + // an HTML maintenance or proxy page served with a 200, an empty body — + // is a failed call, never an empty result: through the callers' + // `?? []`, a positions answer would read as "flat", a trades answer + // as "nothing traded today", and the sync would certify both. For an + // order it stays a plain Error: it may have gone in. + if (!data || typeof data !== 'object') { + throw new Error(`${path} returned a non-JSON answer (HTTP ${res.status})`); + } + if (data.success === false) { + throw new GatewayRefusedError( + `${path} failed: ${data.errorMessage ?? `errorCode=${data.errorCode}`}`, + typeof data.errorCode === 'number' ? data.errorCode : undefined, + typeof data.orderId === 'number' && data.orderId > 0 ? data.orderId : undefined, + ); } return data; } - private connectAndAuth(): void { - if (this.intentionalDisconnect) return; + /** + * Log in, single-flight. Safe to call from anywhere (constructor, retry + * timer, token renewal, watchdog) — a second caller while one runs is a + * no-op, and it never rejects. + * + * First login starts the hub, the sync loop and its 60s interval. A + * RE-login only swaps the token and nudges a sync: it deliberately leaves + * the sync interval and syncComplete alone. Tearing those down before the + * new session was proven is what left a firm "waiting for sync" forever + * when the re-login chain wedged. + */ + private async connectAndAuth(): Promise { + if (this.intentionalDisconnect || this.loginInFlight) return; if (!this.name) { // loginKey requires the platform username the key was generated for. @@ -161,42 +305,89 @@ export class TopstepXClient implements PlatformClient { // can be traded until the username arrives — say so loudly, once. this.lastFetchErrors['login'] = 'TopstepX username missing — the API key alone cannot log in. Add the username on the dashboard.'; + this.lastLoginError = this.lastFetchErrors['login']; console.error(`[topstepx] ${this.lastFetchErrors['login']}`); return; } - this.login() - .then(async (token) => { - this.token = token; - console.log(`[topstepx] logged in as ${this.name}`); - delete this.lastFetchErrors['login']; + if (this.loginRetryTimer) { clearTimeout(this.loginRetryTimer); this.loginRetryTimer = null; } + this.nextReconnectAt = 0; + this.loginInFlight = true; + try { + // A hard deadline on top of axios's own: that one is a socket + // inactivity timer, which never starts while a proxy CONNECT + // hangs — and a login that never settles holds loginInFlight. + const token = await withTimeout(this.login(), LOGIN_TIMEOUT_MS, 'loginKey'); + if (this.intentionalDisconnect) return; - // Session tokens live 24h; re-validate every hour to stay fresh. - if (this.tokenRenewalInterval) clearInterval(this.tokenRenewalInterval); - this.tokenRenewalInterval = setInterval(() => void this.renewToken(), 60 * 60 * 1000); + this.token = token; + this.lastLoginAt = this.tokenRefreshedAt = Date.now(); + this.consecutiveLoginFailures = 0; + this.lastLoginError = null; + console.log(`[topstepx] logged in as ${this.name}`); + delete this.lastFetchErrors['login']; - if (this.syncInterval) { clearInterval(this.syncInterval); this.syncInterval = null; } - this.syncComplete = false; + // Session tokens live 24h; re-validate every hour to stay fresh. + if (this.tokenRenewalInterval) clearInterval(this.tokenRenewalInterval); + this.tokenRenewalInterval = setInterval(() => void this.renewToken(), 60 * 60 * 1000); - // Hub failure must never read as a login failure: REST polling - // carries every feature (slower), and subscribeAll retries the - // hub on each sync tick. - try { - await this.startHub(); - } catch (err) { - console.error('[topstepx] hub setup failed — running on REST polling only', err instanceof Error ? err.message : err); - } - - await this.requestSync(); + if (!this.syncInterval) { + // First login. The hub is never a hard dependency: REST polling + // carries every feature (slower), and each sync tick calls + // ensureHub again, so a hub that won't come up now gets retried. + void this.ensureHub(); this.syncInterval = setInterval(() => void this.requestSync(), 60_000); - }) - .catch((err) => { - const msg = err instanceof Error ? err.message : String(err); - this.lastFetchErrors['login'] = msg; - console.error(`[topstepx] login failed for ${this.name}: ${msg} — retrying in 60s`); - if (this.loginRetryTimer) clearTimeout(this.loginRetryTimer); - this.loginRetryTimer = setTimeout(() => this.connectAndAuth(), 60_000); - }); + void this.requestSync(); + } else { + // Re-login: the loop keeps running; just use the new session now. + void this.requestSync(); + void this.ensureHub(); + } + } catch (err) { + if (this.intentionalDisconnect) return; + this.consecutiveLoginFailures++; + const msg = errorMessage(err); + this.lastLoginError = msg; + this.lastFetchErrors['login'] = msg; + const backoff = Math.min( + LOGIN_RETRY_BASE_MS * 2 ** Math.min(this.consecutiveLoginFailures - 1, 10), + LOGIN_RETRY_MAX_MS, + ); + const delay = Math.round(backoff * (0.8 + Math.random() * 0.4)); + this.nextReconnectAt = Date.now() + delay; + console.error(`[topstepx] login failed for ${this.name}: ${msg} — attempt ${this.consecutiveLoginFailures}, retrying in ${Math.round(delay / 1000)}s`); + this.loginRetryTimer = setTimeout(() => { + this.loginRetryTimer = null; + this.nextReconnectAt = 0; + void this.connectAndAuth(); + }, delay); + } finally { + this.loginInFlight = false; + } + } + + /** + * Never-dead invariant, checked every minute. Every failure path above is + * supposed to leave a retry scheduled; this catches the ones that don't + * (and sessions that died without anything failing loudly): + * - no token, no login running, no retry pending → log in now; + * - logged in >10 min ago with no good sync for >10 min → the token may + * have died in a way validate didn't catch → log in again. + */ + private watchdog(): void { + if (!this.name || this.intentionalDisconnect) return; + if (this.loginInFlight || this.loginRetryTimer) return; + const now = Date.now(); + if (!this.token) { + console.warn(`[topstepx] watchdog: ${this.name} has no session and no login pending — logging in`); + void this.connectAndAuth(); + return; + } + if (now - this.lastLoginAt > SESSION_STALE_MS && now - this.lastSyncAt > SESSION_STALE_MS) { + const since = this.lastSyncAt > 0 ? `in ${Math.round((now - this.lastSyncAt) / 60_000)} min` : 'since login'; + console.warn(`[topstepx] watchdog: no good sync for ${this.name} ${since} — re-logging in`); + void this.connectAndAuth(); + } } /** LoginErrorCode names from the gateway swagger — the API itself sends @@ -218,6 +409,12 @@ export class TopstepXClient implements PlatformClient { const res = await axios.post(`${API_BASE}/api/Auth/loginKey`, { userName: this.name, apiKey: this.apiKey, + }, { + timeout: LOGIN_TIMEOUT_MS, + // Just after connectAndAuth's own deadline: that one gives the + // clean error; this one kills the request, so a stalled CONNECT + // can't still log in minutes later. + signal: AbortSignal.timeout(LOGIN_TIMEOUT_MS + 5_000), }); const data = res.data as { token?: string; success: boolean; errorCode: number; errorMessage: string | null }; if (!data.success || !data.token) { @@ -228,36 +425,203 @@ export class TopstepXClient implements PlatformClient { } private async renewToken(): Promise { + if (this.intentionalDisconnect || this.loginInFlight) return; + if (!this.token) { void this.connectAndAuth(); return; } + const sent = this.token; try { - const res = await axios.post(`${API_BASE}/api/Auth/validate`, {}, { headers: this.authHeaders() }); + const res = await withTimeout(axios.post(`${API_BASE}/api/Auth/validate`, {}, { + headers: this.authHeaders(), + timeout: VALIDATE_TIMEOUT_MS, + signal: AbortSignal.timeout(VALIDATE_TIMEOUT_MS + 5_000), + }), VALIDATE_TIMEOUT_MS, 'token validate'); + if (this.intentionalDisconnect) return; const data = res.data as { success: boolean; newToken?: string }; if (data.success && data.newToken) { + // A login that landed meanwhile holds a newer session — keep it. + if (this.token !== sent) return; this.token = data.newToken; + this.tokenRefreshedAt = Date.now(); console.log('[topstepx] session token renewed'); } else if (!data.success) { console.warn('[topstepx] token validate failed — re-authenticating'); - this.connectAndAuth(); + void this.connectAndAuth(); } } catch (err) { - console.error('[topstepx] token renewal error — re-authenticating', err); - this.connectAndAuth(); + if (this.intentionalDisconnect) return; + // Never the error object: its request config carries the Bearer token. + console.error(`[topstepx] token renewal error — re-authenticating: ${errorMessage(err)}`); + void this.connectAndAuth(); } } // ── realtime hub ───────────────────────────────────────────────────────── - private async startHub(): Promise { - if (this.hub) { - try { await this.hub.stop(); } catch { /* ignore */ } - this.hub = null; + /** + * Bring the user hub to connected-and-subscribed, as far as one pass can. + * Called on login and every sync tick, and never awaited by anything that + * matters: REST polling carries every feature on its own (slower), so a + * wedged hub must never hold up a sync. + * + * - no hub → build a fresh one and start it; a failed start discards it + * (a HubConnection whose first start failed never auto-reconnects), so + * the next pass builds another; + * - Disconnected (automatic reconnect gave up) → start it again, discard + * on failure; + * - Connecting/Reconnecting/Disconnecting for > 2 min → stuck; discard it + * and build a fresh one; + * - Connected → subscribe whatever isn't subscribed yet. + * + * Single-flight; a flight running past HUB_SETUP_WEDGED_MS is superseded + * (every await in it has a deadline, so that is belt and braces). + */ + private async ensureHub(): Promise { + if (this.intentionalDisconnect || !this.token) return; + if (this.hubSetupInFlight) { + const runningMs = Date.now() - this.hubSetupSince; + if (runningMs < HUB_SETUP_WEDGED_MS) { + this.hubRecheckWanted = true; + return; + } + console.warn(`[topstepx] hub setup running for ${Math.round(runningMs / 1000)}s — looks wedged, starting over`); } - this.subscribedAccountIds.clear(); + const gen = ++this.hubSetupGeneration; + this.hubSetupInFlight = true; + this.hubSetupSince = Date.now(); + this.hubRecheckWanted = false; + let clean = false; + try { + clean = await this.ensureHubPass(gen); + } catch (err) { + console.error(`[topstepx] hub setup failed — running on REST polling only: ${errorMessage(err)}`); + } finally { + // A superseded flight must not clear its successor's flag. + if (gen === this.hubSetupGeneration) { + this.hubSetupInFlight = false; + this.hubSetupSince = 0; + // Asked for again mid-flight (typically: the first sync's + // accounts landed while the hub was starting). Only worth a + // rerun after a clean pass on a live hub — a failed start or + // subscribe waits for the next tick rather than retrying hot. + if (clean && this.hubRecheckWanted && !this.intentionalDisconnect + && this.hub?.state === signalR.HubConnectionState.Connected) { + this.hubRecheckWanted = false; + void this.ensureHub(); + } + } + } + } + /** One ensureHub pass. True when it ended connected with everything + * subscribed; false when it stopped short (the next call carries on). */ + private async ensureHubPass(gen: number): Promise { + const State = signalR.HubConnectionState; + let hub = this.hub; + + if (hub && (hub.state === State.Connecting || hub.state === State.Reconnecting || hub.state === State.Disconnecting)) { + const now = Date.now(); + if (!this.hubStalledSince) this.hubStalledSince = now; + // Automatic reconnect is still working on it — let it. + if (now - this.hubStalledSince < HUB_STALL_MS) return false; + console.warn(`[topstepx] hub stuck ${hub.state} for ${Math.round((now - this.hubStalledSince) / 1000)}s — discarding it`); + this.discardHub(); + hub = null; + } + + if (!hub) { + hub = this.buildHub(); + this.hub = hub; + this.subscribedAccountIds.clear(); + this.accountsFeedSubscribed = false; + if (!(await this.startHub(hub, gen))) return false; + console.log('[topstepx] user hub connected'); + } else if (hub.state === State.Disconnected) { + // Subscriptions died with the old connection. + this.subscribedAccountIds.clear(); + this.accountsFeedSubscribed = false; + if (!(await this.startHub(hub, gen))) return false; + console.log('[topstepx] user hub restarted'); + } + + if (hub.state !== State.Connected) return false; + this.hubStalledSince = 0; + return this.subscribeAll(hub, gen); + } + + /** Start `hub` against a deadline. False when it didn't come up (the hub + * is discarded) or was discarded / superseded while starting. */ + private async startHub(hub: signalR.HubConnection, gen: number): Promise { + try { + await withTimeout(hub.start(), HUB_START_TIMEOUT_MS, 'hub start'); + } catch (err) { + // REST polling still works without the hub; the next sync tick + // builds a fresh one. + console.error(`[topstepx] hub connect failed — falling back to REST polling: ${errorMessage(err)}`); + if (this.hub === hub) this.discardHub(); + return false; + } + if (this.hub !== hub) { + // Discarded (disconnect, or a stall check) while it was starting. + void hub.stop().catch(() => { /* discarded */ }); + return false; + } + if (gen !== this.hubSetupGeneration) return false; + this.hubStalledSince = 0; + this.lastInboundAt = Date.now(); + return true; + } + + /** Subscribe everything the current connection isn't subscribed to yet. + * A failed or timed-out invoke is left for the next pass. True when + * nothing was left over. */ + private async subscribeAll(hub: signalR.HubConnection, gen: number): Promise { + const stillCurrent = () => + this.hub === hub && gen === this.hubSetupGeneration && hub.state === signalR.HubConnectionState.Connected; + const invoke = async (method: string, ...args: unknown[]) => { + await withTimeout(hub.invoke(method, ...args), HUB_INVOKE_TIMEOUT_MS, method); + this.lastInboundAt = Date.now(); + }; + try { + if (!this.accountsFeedSubscribed) { + await invoke('SubscribeAccounts'); + if (!stillCurrent()) return false; + this.accountsFeedSubscribed = true; + } + for (const acc of this.accountList) { + if (this.subscribedAccountIds.has(acc.id)) continue; + await invoke('SubscribeOrders', acc.id); + await invoke('SubscribePositions', acc.id); + await invoke('SubscribeTrades', acc.id); + if (!stillCurrent()) return false; + this.subscribedAccountIds.add(acc.id); + } + return true; + } catch (err) { + console.error(`[topstepx] hub subscribe failed — retrying on the next sync: ${errorMessage(err)}`); + return false; + } + } + + /** Drop the current hub without waiting on it — stop() can hang as long + * as the start it interrupts, so it is fire-and-forget. The hub's + * handlers go quiet the moment this.hub stops pointing at it. */ + private discardHub(): void { + const hub = this.hub; + this.hub = null; + this.subscribedAccountIds.clear(); + this.accountsFeedSubscribed = false; + this.hubStalledSince = 0; + if (hub) void hub.stop().catch(() => { /* discarded */ }); + } + + /** A fresh HubConnection with every handler wired. Each handler first + * checks its hub is still the current one: a discarded connection that + * lingers (or reconnects on its own) must never write state. */ + private buildHub(): signalR.HubConnection { // signalr's Node path does an unconditional dynamic require('ws') / // require('eventsource') when constructing the connection — bundled // builds throw "dynamic usage of require is not supported" on it, so // the package is listed in next.config serverExternalPackages and any - // residual failure here is caught by the caller (REST polling covers + // residual failure here is caught by ensureHub (REST polling covers // everything the hub does, just slower). // With skipNegotiation the hub is a single websocket to a broker // endpoint, so it follows the operator's proxy like the REST calls do. @@ -282,6 +646,8 @@ export class TopstepXClient implements PlatformClient { }; hub.on('GatewayUserAccount', (raw: unknown) => { + if (this.hub !== hub) return; + this.lastInboundAt = Date.now(); const acc = unwrap(raw); if (typeof acc?.id !== 'number') return; this.bufferEvent('account', acc); @@ -299,6 +665,8 @@ export class TopstepXClient implements PlatformClient { }); hub.on('GatewayUserPosition', (raw: unknown) => { + if (this.hub !== hub) return; + this.lastInboundAt = Date.now(); const pos = unwrap(raw); if (typeof pos?.accountId !== 'number') return; this.bufferEvent('position', pos); @@ -306,6 +674,8 @@ export class TopstepXClient implements PlatformClient { }); hub.on('GatewayUserTrade', (raw: unknown) => { + if (this.hub !== hub) return; + this.lastInboundAt = Date.now(); const trade = unwrap(raw) as unknown as TsxTrade; if (typeof trade?.accountId !== 'number') return; this.bufferEvent('trade', trade); @@ -315,67 +685,43 @@ export class TopstepXClient implements PlatformClient { const waiters = this.fillWaiters.filter((w) => w.orderId === trade.orderId); this.fillWaiters = this.fillWaiters.filter((w) => w.orderId !== trade.orderId); for (const w of waiters) { - try { w.callback(trade); } catch (err) { console.error('[topstepx] fill callback error', err); } + try { w.callback(trade); } catch (err) { console.error(`[topstepx] fill callback error: ${errorMessage(err)}`); } } }); hub.on('GatewayUserOrder', (raw: unknown) => { + if (this.hub !== hub) return; + this.lastInboundAt = Date.now(); const order = unwrap(raw); if (typeof order?.id !== 'number') return; this.bufferEvent('order', order); this.handleOcoOrderEvent(order.id, order.status); }); + hub.onreconnecting(() => { + if (this.hub !== hub) return; + // The stall clock ensureHub judges "stuck Reconnecting" by. + if (!this.hubStalledSince) this.hubStalledSince = Date.now(); + }); + hub.onreconnected(() => { + if (this.hub !== hub) return; + this.lastInboundAt = Date.now(); + this.hubStalledSince = 0; console.log('[topstepx] hub reconnected — resubscribing'); this.subscribedAccountIds.clear(); - void this.subscribeAll(); + this.accountsFeedSubscribed = false; + void this.ensureHub(); void this.requestSync(); }); hub.onclose((err) => { - if (this.intentionalDisconnect) return; - console.warn('[topstepx] hub closed', err?.message ?? ''); + if (this.hub !== hub || this.intentionalDisconnect) return; + this.hubStalledSince = 0; + console.warn(`[topstepx] hub closed — the next sync restarts it${err ? `: ${errorMessage(err)}` : ''}`); }); - this.hub = hub; - try { - await hub.start(); - await this.subscribeAll(); - console.log('[topstepx] user hub connected'); - } catch (err) { - // REST polling still works without the hub; automatic reconnect - // only runs after a successful start, so retry via the next sync. - console.error('[topstepx] hub connect failed — falling back to REST polling', err); - } - } - - private async subscribeAll(): Promise { - if (!this.hub) return; - // A hub whose initial start failed never auto-reconnects — each sync - // tick gets another chance to bring it up. - if (this.hub.state === signalR.HubConnectionState.Disconnected) { - try { - await this.hub.start(); - this.subscribedAccountIds.clear(); - console.log('[topstepx] user hub connected'); - } catch { - return; // REST polling carries on; retry next sync - } - } - if (this.hub.state !== signalR.HubConnectionState.Connected) return; - try { - await this.hub.invoke('SubscribeAccounts'); - for (const acc of this.accountList) { - if (this.subscribedAccountIds.has(acc.id)) continue; - await this.hub.invoke('SubscribeOrders', acc.id); - await this.hub.invoke('SubscribePositions', acc.id); - await this.hub.invoke('SubscribeTrades', acc.id); - this.subscribedAccountIds.add(acc.id); - } - } catch (err) { - console.error('[topstepx] hub subscribe failed', err); - } + return hub; } private bufferEvent(entityType: string, entity: unknown): void { @@ -428,12 +774,30 @@ export class TopstepXClient implements PlatformClient { // ── sync ───────────────────────────────────────────────────────────────── + /** + * One REST sync of accounts, positions and today's trades. Single-flight, + * but a sync still running after SYNC_WEDGED_MS is presumed wedged and a + * fresh one starts: nothing may hold syncInFlight forever, because a firm + * that stops syncing stops trading. The superseded run checks its + * generation after every await and bows out without touching state, and + * its finally leaves the new run's flag alone. + */ private async requestSync(): Promise { - if (!this.token || this.syncInFlight) return; + if (!this.token || this.intentionalDisconnect) return; + const startedAt = Date.now(); + if (this.syncInFlight) { + const runningMs = startedAt - this.syncInFlightSince; + if (runningMs < SYNC_WEDGED_MS) return; + console.warn(`[topstepx] sync running for ${Math.round(runningMs / 1000)}s — looks wedged, starting a fresh one`); + } + const gen = ++this.syncGeneration; + const superseded = () => gen !== this.syncGeneration || this.intentionalDisconnect; this.syncInFlight = true; + this.syncInFlightSince = startedAt; try { const accRes = await this.post<{ accounts: { id: number; name: string; balance: number; canTrade: boolean; isVisible: boolean }[] }>( '/api/Account/search', { onlyActiveAccounts: false }); + if (superseded()) return; this.accountList = (accRes.accounts ?? []) .filter((a) => a.isVisible) @@ -444,13 +808,17 @@ export class TopstepXClient implements PlatformClient { const start = new Date(Date.now() - 2 * 24 * 60 * 60 * 1000).toISOString(); const end = new Date(Date.now() + 60 * 60 * 1000).toISOString(); const today = currentTradeDateCT(); + /** Accounts whose positions or trades didn't come back this pass. */ + const staleAccounts: string[] = []; for (const acc of this.accountList) { const balance = (accRes.accounts ?? []).find((a) => a.id === acc.id)?.balance ?? 0; + let accountFresh = true; try { const posRes = await this.post<{ positions: { accountId: number; contractId: string; type: number; size: number; averagePrice: number; creationTimestamp: string }[] }>( '/api/Position/searchOpen', { accountId: acc.id }); + if (superseded()) return; const open = (posRes.positions ?? []).filter((p) => p.size > 0); if (open.length > 0) { this.applyPosition(open[0]); @@ -458,12 +826,15 @@ export class TopstepXClient implements PlatformClient { delete this.positions[acc.id]; } } catch (err) { - console.error(`[topstepx] position sync failed for ${acc.name}`, err instanceof Error ? err.message : err); + accountFresh = false; + console.error(`[topstepx] position sync failed for ${acc.name}: ${errorMessage(err)}`); } + if (superseded()) return; try { const tradeRes = await this.post<{ trades: TsxTrade[] }>( '/api/Trade/search', { accountId: acc.id, startTimestamp: start, endTimestamp: end }); + if (superseded()) return; const map = new Map(); for (const t of tradeRes.trades ?? []) { if (t.voided) continue; @@ -472,38 +843,64 @@ export class TopstepXClient implements PlatformClient { } this.todayTrades[acc.id] = map; } catch (err) { - console.error(`[topstepx] trade sync failed for ${acc.name}`, err instanceof Error ? err.message : err); + accountFresh = false; + console.error(`[topstepx] trade sync failed for ${acc.name}: ${errorMessage(err)}`); } + if (superseded()) return; + if (!accountFresh) staleAccounts.push(acc.name); this.accountCashBalances[acc.id] = { amount: balance, realizedPnL: 0 }; this.recomputeRealized(acc.id); this.autoLiqThresholds[acc.id] = 0; // Withdrawal/deposit detection — the books must explain the balance. + // Never off a failed fetch: a fresh balance against stale trades + // would book the unseen P&L as a (permanent) fund event. try { - this.reconcileFunds(acc); + if (accountFresh) this.reconcileFunds(acc); } catch (err) { - console.error(`[topstepx] fund reconcile failed for ${acc.name}`, err instanceof Error ? err.message : err); + console.error(`[topstepx] fund reconcile failed for ${acc.name}: ${errorMessage(err)}`); } try { this.updateMllFloor(acc); } catch (err) { - console.error(`[topstepx] MLL reconstruction failed for ${acc.name}`, err instanceof Error ? err.message : err); + console.error(`[topstepx] MLL reconstruction failed for ${acc.name}: ${errorMessage(err)}`); } } console.log(`[topstepx] sync — ${this.accountList.length} account(s), ${Object.keys(this.positions).length} open position(s)`); - await this.subscribeAll(); + // Fire-and-forget: the hub subscribes new accounts on its own + // schedule and can never hold this sync up. + void this.ensureHub(); this.maybeFetchDaysTraded(); this.syncComplete = true; - await this.callbackOnSyncRequest(); + // lastSyncAt is what the scheduler trades on (isClientReady), so it + // certifies only a pass where every account's positions and trades + // came back: a stale-flat position or a stale realized P&L walks + // straight through the "already in trade" / "already traded today" + // gates. Nor before the first daily P&L pass has landed — until + // then the plan would run on no history at all. + if (staleAccounts.length > 0) { + console.warn(`[topstepx] sync incomplete — not refreshed: ${staleAccounts.join(', ')}; not counted as a good sync`); + } else if (this.historyLoaded) { + this.lastSyncAt = Date.now(); + } + // The resolver runs off this; its failure is not a sync failure. + try { + await this.callbackOnSyncRequest(); + } catch (err) { + console.error(`[topstepx] post-sync callback failed: ${errorMessage(err)}`); + } } catch (err) { - console.error('[topstepx] sync failed', err instanceof Error ? err.message : err); + console.error(`[topstepx] sync failed: ${errorMessage(err)}`); } finally { - this.syncInFlight = false; + if (gen === this.syncGeneration) { + this.syncInFlight = false; + this.syncInFlightSince = 0; + } } } @@ -679,7 +1076,7 @@ export class TopstepXClient implements PlatformClient { } private maybeFetchDaysTraded(): void { - if (this.daysFetchInFlight) return; + if (this.daysFetchInFlight || this.intentionalDisconnect) return; const now = Date.now(); const initialFetchDue = this.lastDaysFetchStartedAt === 0; @@ -693,11 +1090,18 @@ export class TopstepXClient implements PlatformClient { if (!initialFetchDue && !retryDue && !hourlyRefreshDue) return; this.daysFetchInFlight = true; + this.daysFetchInFlightSince = now; this.lastDaysFetchStartedAt = now; if (retryDue) this.nextDaysFetchRetryAt = 0; this.fetchDaysTraded() .then(({ failedAccounts }) => { + if (!this.historyLoaded && !this.intentionalDisconnect) { + // Readiness waited on this; sync again now rather than + // leaving the firm unready until the next 60s tick. + this.historyLoaded = true; + void this.requestSync(); + } if (failedAccounts > 0) { this.nextDaysFetchRetryAt = Date.now() + 5 * 60 * 1_000; console.warn(`[topstepx] ${failedAccounts} account(s) failed daily P&L fetch — retrying in 5 minutes`); @@ -707,10 +1111,11 @@ export class TopstepXClient implements PlatformClient { }) .catch((err) => { this.nextDaysFetchRetryAt = Date.now() + 5 * 60 * 1_000; - console.error('[topstepx] daily P&L fetch failed — retrying in 5 minutes', err); + console.error(`[topstepx] daily P&L fetch failed — retrying in 5 minutes: ${errorMessage(err)}`); }) .finally(() => { this.daysFetchInFlight = false; + this.daysFetchInFlightSince = 0; }); } @@ -738,6 +1143,7 @@ export class TopstepXClient implements PlatformClient { }; for (const account of this.accountList) { + if (this.intentionalDisconnect) break; // a discarded client fetches (and writes) nothing more if (onlyAccountIds && !onlyAccountIds.includes(account.id)) continue; // Fund transactions never come from the TopstepX API — only what the @@ -788,8 +1194,8 @@ export class TopstepXClient implements PlatformClient { delete this.lastFetchErrors[account.name]; this.updateMllFloor(account); } catch (err) { - const msg = err instanceof Error ? err.message : String(err); - console.error(`[topstepx] trade history failed for ${account.name}:`, msg); + const msg = errorMessage(err); + console.error(`[topstepx] trade history failed for ${account.name}: ${msg}`); failedAccounts++; if (cached.length > 0) { this.fullDailyPnL[account.id] = cached; @@ -841,7 +1247,7 @@ export class TopstepXClient implements PlatformClient { this.contractByName.set(match.name, match); return { id: match.id, name: match.name, tickSize: match.tickSize }; } catch (err) { - console.error(`[topstepx] contract search failed for ${productName}`, err instanceof Error ? err.message : err); + console.error(`[topstepx] contract search failed for ${productName}: ${errorMessage(err)}`); return null; } } @@ -872,6 +1278,29 @@ export class TopstepXClient implements PlatformClient { // ── orders ─────────────────────────────────────────────────────────────── + /** + * Everything an order needs before anything is sent: a session and the + * contract. Failing here means nothing reached the broker, which is the + * one case the scheduler may un-count a leg for — hence OrderNotPlacedError. + */ + private async contractForOrder(contractSymbol: string): Promise { + if (!this.token) throw new OrderNotPlacedError('TopstepX not logged in — order not sent'); + try { + return await this.resolveContractId(contractSymbol); + } catch (err) { + throw new OrderNotPlacedError(`TopstepX contract lookup failed for ${contractSymbol} — order not sent: ${errorMessage(err)}`); + } + } + + /** + * POST /api/Order/place. Only a failure that proves nothing was placed + * becomes OrderNotPlacedError: no session, a gateway `success: false` + * that names no order and no ambiguous code, or an HTTP refusal given + * before any handler ran. Everything else (timeouts, resets, 5xx) may + * have placed the order and stays a plain Error — rewrapped so the + * AxiosError, whose config carries the Bearer token, never travels on + * into someone's log. + */ private async placeOrder( accountId: number, contractId: string, @@ -881,18 +1310,36 @@ export class TopstepXClient implements PlatformClient { limitPrice?: number, stopPrice?: number, ): Promise { - const res = await this.post<{ orderId: number }>('/api/Order/place', { - accountId, - contractId, - type, - side: action === 'Buy' ? SIDE_BUY : SIDE_SELL, - size, - limitPrice: limitPrice ?? null, - stopPrice: stopPrice ?? null, - trailPrice: null, - customTag: null, - }); - return res.orderId; + if (!this.token) throw new OrderNotPlacedError('TopstepX not logged in — order not sent'); + try { + const res = await this.post<{ orderId: number }>('/api/Order/place', { + accountId, + contractId, + type, + side: action === 'Buy' ? SIDE_BUY : SIDE_SELL, + size, + limitPrice: limitPrice ?? null, + stopPrice: stopPrice ?? null, + trailPrice: null, + customTag: null, + }); + return res.orderId; + } catch (err) { + if (err instanceof GatewayRefusedError) { + const ambiguous = err.orderId !== undefined + || (err.errorCode !== undefined && AMBIGUOUS_PLACE_ERRORS.has(err.errorCode)); + if (!ambiguous) throw new OrderNotPlacedError(`TopstepX refused the order: ${err.message}`); + throw new Error(`TopstepX order outcome unknown: ${err.message}`); + } + const response = (err as { response?: { status?: unknown; data?: { orderId?: unknown } } } | null)?.response; + const status = response?.status; + // Same rule as a refusal: an answer that names an order means it exists. + const namesOrder = typeof response?.data?.orderId === 'number' && response.data.orderId > 0; + if (typeof status === 'number' && REFUSED_BEFORE_HANDLER.has(status) && !namesOrder) { + throw new OrderNotPlacedError(`TopstepX rejected the order request: ${errorMessage(err)}`); + } + throw new Error(`TopstepX order request failed (it may still have been placed): ${errorMessage(err)}`); + } } /** Place an order and wait for its fill (trade event, with a REST poll as @@ -905,7 +1352,7 @@ export class TopstepXClient implements PlatformClient { orderType: 'Market' | 'Limit', price?: number ): Promise> { - const contract = await this.resolveContractId(contractSymbol); + const contract = await this.contractForOrder(contractSymbol); const orderId = await this.placeOrder( accountId, contract.id, quantity, action, orderType === 'Market' ? TYPE_MARKET : TYPE_LIMIT, @@ -920,7 +1367,15 @@ export class TopstepXClient implements PlatformClient { settled = true; clearInterval(poll); this.fillWaiters = this.fillWaiters.filter((w) => w.orderId !== orderId); - reject(new Error(`Order ${orderId} placed but no fill within 30s (market may be closed)`)); + // The caller gives up on this order and places no exits, so a + // late fill would leave a position nothing manages — and one + // open position blocks every firm through the open-position + // gate. Cancel it, best effort. The rejection stays a plain + // Error: it may have filled after all. + console.warn(`[topstepx] order ${orderId} not filled within 30s — cancelling it`); + void this.cancelOrder(accountId, orderId).catch((err) => + console.error(`[topstepx] failed to cancel unfilled order ${orderId}: ${errorMessage(err)}`)); + reject(new Error(`Order ${orderId} placed but no fill within 30s (market may be closed) — cancel requested`)); }, 30_000); const settle = (fillPrice: number) => { @@ -966,7 +1421,7 @@ export class TopstepXClient implements PlatformClient { orderType: 'Market' | 'Limit', price?: number ): Promise<{ orderId?: number }> { - const contract = await this.resolveContractId(contractSymbol); + const contract = await this.contractForOrder(contractSymbol); const orderId = await this.placeOrder( accountId, contract.id, quantity, action, orderType === 'Market' ? TYPE_MARKET : TYPE_LIMIT, @@ -979,8 +1434,18 @@ export class TopstepXClient implements PlatformClient { * Emulated OCO exit: the gateway has no linked-order type, so place the * take-profit limit and the stop separately and cancel the survivor when * either fills (order events; a poll and the flat-position hook back it - * up). If the stop can't be placed the limit is cancelled and the error - * rethrown, so the caller's lone-limit fallback starts from a clean book. + * up). + * + * If the stop doesn't go in, what the caller may do next depends on what + * is left at the broker — it falls back to a lone limit only on + * OrderNotPlacedError, and places nothing more on a plain Error: + * - stop refused, limit cancel confirmed → nothing of the pair is working: + * OrderNotPlacedError, and the fallback starts from a clean book; + * - stop refused, limit cancel failed → the limit may still be working: + * plain Error (a fallback on top could double the exit); + * - stop outcome unknown → it may be working, so the limit is kept (the + * position keeps its take-profit, unpaired, as the fallback would have + * left it) and the error is plain. */ public async placeOcoExit( accountId: number, @@ -990,14 +1455,20 @@ export class TopstepXClient implements PlatformClient { limitPrice: number, stopPrice: number, ): Promise<{ orderId?: number }> { - const contract = await this.resolveContractId(contractSymbol); + const contract = await this.contractForOrder(contractSymbol); const limitId = await this.placeOrder(accountId, contract.id, quantity, exitAction, TYPE_LIMIT, limitPrice); let stopId: number; try { stopId = await this.placeOrder(accountId, contract.id, quantity, exitAction, TYPE_STOP, undefined, stopPrice); } catch (err) { - await this.cancelOrder(accountId, limitId).catch(() => { /* best effort */ }); - throw err; + if (!isOrderNotPlaced(err)) { + throw new Error(`OCO stop leg outcome unknown — limit ${limitId} left working, unpaired: ${errorMessage(err)}`); + } + const limitCancelled = await this.cancelOrder(accountId, limitId).then(() => true, () => false); + if (limitCancelled) { + throw new OrderNotPlacedError(`OCO stop leg refused, limit ${limitId} cancelled — no exit working: ${errorMessage(err)}`); + } + throw new Error(`OCO stop leg refused and limit ${limitId} could not be cancelled — it may still be working: ${errorMessage(err)}`); } this.ocoPairs.push({ accountId, limitId, stopId, done: false, placedAt: Date.now() }); @@ -1018,7 +1489,7 @@ export class TopstepXClient implements PlatformClient { const sibling = pair.limitId === orderId ? pair.stopId : pair.limitId; console.log(`[topstepx] OCO leg ${orderId} filled — cancelling sibling ${sibling}`); void this.cancelOrder(pair.accountId, sibling).catch((err) => - console.error(`[topstepx] failed to cancel OCO sibling ${sibling}`, err instanceof Error ? err.message : err)); + console.error(`[topstepx] failed to cancel OCO sibling ${sibling}: ${errorMessage(err)}`)); } } @@ -1072,15 +1543,40 @@ export class TopstepXClient implements PlatformClient { // ── lifecycle ──────────────────────────────────────────────────────────── + public health(): ClientHealth { + return { + platform: 'topstepx', + authenticated: !!this.token, + transport: this.hub ? String(this.hub.state) : 'none', + syncComplete: this.syncComplete, + lastSyncAt: this.lastSyncAt, + lastLoginAt: this.lastLoginAt, + lastLoginError: this.lastLoginError, + consecutiveLoginFailures: this.consecutiveLoginFailures, + nextReconnectAt: this.nextReconnectAt, + // Validate hands out fresh 24h tokens; this is the latest one's expiry. + tokenExpiresAt: this.token && this.tokenRefreshedAt ? this.tokenRefreshedAt + TOKEN_LIFETIME_MS : 0, + lastInboundAt: this.lastInboundAt, + pendingRequests: this.fillWaiters.length, + syncInFlightSince: this.syncInFlight ? this.syncInFlightSince : 0, + daysFetchInFlightSince: this.daysFetchInFlight ? this.daysFetchInFlightSince : 0, + }; + } + + /** Final: nothing this client scheduled fires afterwards, and every + * in-flight login / sync / hub setup checks intentionalDisconnect (or + * that its hub is still current) before touching state again. */ public disconnect(): void { this.intentionalDisconnect = true; if (this.syncInterval) { clearInterval(this.syncInterval); this.syncInterval = null; } if (this.tokenRenewalInterval) { clearInterval(this.tokenRenewalInterval); this.tokenRenewalInterval = null; } if (this.ocoPollInterval) { clearInterval(this.ocoPollInterval); this.ocoPollInterval = null; } if (this.loginRetryTimer) { clearTimeout(this.loginRetryTimer); this.loginRetryTimer = null; } + if (this.watchdogInterval) { clearInterval(this.watchdogInterval); this.watchdogInterval = null; } + this.nextReconnectAt = 0; this.daysFetchInFlight = false; + this.daysFetchInFlightSince = 0; this.nextDaysFetchRetryAt = 0; - void this.hub?.stop().catch(() => { /* ignore */ }); - this.hub = null; + this.discardHub(); } } diff --git a/lib/tradovate-class.ts b/lib/tradovate-class.ts index e2a7531..2de94f3 100644 --- a/lib/tradovate-class.ts +++ b/lib/tradovate-class.ts @@ -7,7 +7,7 @@ import { POINT_VALUES } from './trading-logic'; import { getCachedContract, resolveContracts } from './contract-resolver'; import { saveDailyPnL, loadDailyPnL, saveAccountMeta, loadAccountMeta, saveFundTransactions, loadFundTransactions, saveEndedCycleStart } from './db'; import { brokerWebSocket } from './outbound-proxy'; -import { FILLS_FALLBACK_NOTE } from './platform-client'; +import { FILLS_FALLBACK_NOTE, OrderNotPlacedError, SYNC_STALE_MS, errorMessage, type ClientHealth } from './platform-client'; /** * Entity types Tradovate sends over the user-sync socket. Keep this as the @@ -26,6 +26,111 @@ type TradovateEntityType = | 'position' | 'cashBalance'; +/** + * What auth/accesstokenrequest can answer. HTTP 200 is not success on its + * own: a rate-limited login comes back 200 with no token and a penalty ticket + * instead (p-ticket, plus p-time in seconds to wait before presenting it, and + * p-captcha when Tradovate wants a human), and a refused one with errorText. + */ +type LoginResponseBody = Partial & { + 'p-ticket'?: string; + 'p-time'?: number; + 'p-captcha'?: boolean; + errorText?: string; +}; + +type LoginOutcome = + | { ok: true; info: AuthLoginResponse } + | { ok: false; error: string; retryInMs?: number }; + +/** A listener for pushed entity events. One-shot: it is removed once it fires. + * `cancel` (sendOrder's fill wait) rejects the waiter when the client is + * discarded, so no caller is left hanging on a dead instance. */ +type IndirectCallback = { + entityType: TradovateEntityType; + eventType: 'Created' | 'Updated'; + // Since the entity is not always the same, we need a validator to check if the response is the one we are looking for + validator: (response: any) => boolean; + callback: (response: any) => void; + cancel?: (reason: string) => void; +}; + +/* + * Connection supervision. + * + * This client used to be `login().then(...)` with exactly one retry scheduled + * from ws.onclose. One failed re-login — weekend maintenance, a network blip, + * a rate-limit penalty answered as HTTP 200 without a token — left it dead for + * the life of the process, with syncComplete frozen, and every firm on the + * instance waited on it until someone ran a pm2 restart. A session could also + * die without onclose ever firing (a socket stuck CLOSING, sync answers that + * just stop), and an order sent on a closed socket waited forever. + * + * Now: every failure path ends in scheduleReconnect(); a 15s watchdog notices + * sockets that went quiet, never authorized, or got stuck, and re-arms a + * reconnect if nothing is pending; nothing waits on the broker without a + * deadline. + */ +/** Reconnect after a socket drop on a still-good session. */ +const SOCKET_DROP_RECONNECT_MS = 5_000; +/** Reconnect when the session itself is suspect — straight to a fresh login. */ +const FORCED_RELOGIN_DELAY_MS = 1_000; +/** Login failures back off 5s, 10s, 20s, … capped at 5 min. */ +const BACKOFF_BASE_MS = 5_000; +const BACKOFF_MAX_MS = 5 * 60_000; +/** A captcha demand means we've logged in too often — go quiet for a long while. */ +const CAPTCHA_RETRY_MS = 15 * 60_000; +const PENALTY_MAX_WAIT_MS = 5 * 60_000; +const MAX_PENALTY_ROUNDS = 2; +/** Deadline for login / token renewal / contract lookups over REST. */ +const HTTP_TIMEOUT_MS = 15_000; +/** Report calls answer with a p-ticket to poll when they're slow, so a single + * HTTP call taking this long is a hung connection, not a big report. */ +const REPORT_TIMEOUT_MS = 60_000; +const LIVENESS_CHECK_MS = 15_000; +const HANDSHAKE_TIMEOUT_MS = 30_000; +const AUTHORIZE_TIMEOUT_MS = 30_000; +/** Tradovate heartbeats every few seconds and we sync every 60s — this much + * silence means the session is gone even if the socket says OPEN. */ +const INBOUND_SILENCE_MS = 150_000; +const STUCK_CLOSING_MS = 15_000; +const TOKEN_RENEW_CHECK_MS = 2 * 60_000; +const TOKEN_RENEW_WINDOW_MS = 15 * 60_000; +/** A failed renewal this close to expiry logs in afresh instead of waiting. */ +const TOKEN_RELOGIN_WINDOW_MS = 2 * 60_000; +/** A reconnect reuses the current token only if it has at least this long left. */ +const TOKEN_REUSE_MIN_MS = 5 * 60_000; +/** An order ack later than this is treated as lost (ambiguous — it may have been placed). */ +const ACK_TIMEOUT_MS = 15_000; +const FILL_TIMEOUT_MS = 30_000; +const CONTRACT_FIND_TIMEOUT_MS = 15_000; +/** On-demand front-month resolve (several Yahoo lookups, then Tradovate). */ +const CONTRACT_RESOLVE_DEADLINE_MS = 30_000; +const READY_STATE_NAMES = ['CONNECTING', 'OPEN', 'CLOSING', 'CLOSED']; + +function reconnectBackoffMs(failures: number): number { + return Math.min(BACKOFF_BASE_MS * 2 ** Math.max(0, failures - 1), BACKOFF_MAX_MS); +} + +/** ±20%, so several clients on one IP that failed together don't retry in lockstep. */ +function jitter(ms: number): number { + return Math.round(ms * (0.8 + Math.random() * 0.4)); +} + +function fmtDelay(ms: number): string { + return ms >= 60_000 ? `${(ms / 60_000).toFixed(1)}m` : `${(ms / 1000).toFixed(1)}s`; +} + +/** Settles like `work`, or rejects once `ms` pass first. The work itself + * carries on — only the wait for it is cut short. */ +function withDeadline(work: Promise, ms: number, what: string): Promise { + let timer: ReturnType | undefined; + const deadline = new Promise((_, reject) => { + timer = setTimeout(() => reject(new Error(`${what} took longer than ${ms / 1000}s`)), ms); + }); + return Promise.race([work, deadline]).finally(() => clearTimeout(timer)); +} + export class TradovateClient { private name: string; private password: string; @@ -87,26 +192,64 @@ export class TradovateClient { /** True once the first requestSync has completed and positions/balances are populated. */ public syncComplete = false; + /** Epoch ms of the last sync that came back good (0 = never). */ + public lastSyncAt = 0; + + // ── Connection supervisor state (see the comment above the constants) ── + /** Single-flight guard: true while connectAndAuth() is logging in / opening. */ + private connecting = false; + private reconnectTimer: ReturnType | null = null; + /** When reconnectTimer (or a login penalty wait) fires; 0 = nothing scheduled. */ + private nextReconnectAt = 0; + private consecutiveLoginFailures = 0; + /** Sessions in a row that died before producing a good sync (never + * opened, never authorized, sync refused) — they back off like login + * failures instead of reconnecting (or re-logging in) every few seconds. */ + private consecutiveSocketFailures = 0; + /** Last successful login. */ + private lastLoginAt = 0; + private lastLoginError: string | null = null; + /** The current token is suspect (refused, never authorized a socket, sync + * answers stopped): the next connect logs in instead of reusing it. */ + private needsLogin = true; + /** Socket creation time; reset to the open time once it opens. */ + private socketOpenedAt = 0; + /** The current socket got as far as presenting the token. */ + private authorizeSent = false; + /** When the current socket's authorize was accepted (0 = not yet). */ + private authorizedAt = 0; + private lastInboundAt = 0; + /** First time the watchdog saw the current socket CLOSING/CLOSED. */ + private closingSeenAt = 0; + /** When the oldest still-unanswered syncrequest was sent (0 = none). */ + private syncRequestedAt = 0; + private livenessInterval: ReturnType | null = null; + private renewInFlight = false; + /** The current token's expirationTime was (nearly) past on arrival: this + * machine's clock is off, so the expiry is ignored (noteTokenExpiry()). */ + private tokenExpiryUntrusted = false; + /** A login rate-limit penalty wait in progress — disconnect() cuts it short. */ + private penaltyWait: { timer: ReturnType; resolve: () => void } | null = null; - private ws!: WebSocket; + private ws: WebSocket | null = null; private callbackOnSyncRequest: () => Promise; /** Incrementing ID for outgoing WebSocket messages — ensures concurrent orders don't clobber each other's callbacks. */ private nextMsgId = 100; private getMsgId(): number { return this.nextMsgId++; } - // Events that we sent out, and tradovate gives us a response for the id we sent out + // Events that we sent out, and tradovate gives us a response for the id we sent out. + // One-shot: removed when the answer arrives, and all dropped with the socket. private directEventCallbacks: { - [id: number]: (response: any) => void; + [id: number]: (response: any, status?: number) => void; } = {}; - private indirectEventCallbacks: { - entityType: TradovateEntityType; - eventType: 'Created' | 'Updated'; - // Since the entity is not always the same, we need a validator to check if the response is the one we are looking for - validator: (response: any) => boolean; - callback: (response: any) => void; - }[] = []; + /** Rejecters for awaited direct requests (acks, contract finds), so a socket + * going away fails them instead of leaving the caller waiting forever. */ + private pendingRejecters: { + [id: number]: (reason: string) => void; + } = {}; + private indirectEventCallbacks: IndirectCallback[] = []; constructor( @@ -117,156 +260,538 @@ export class TradovateClient { this.name = name; this.password = password; this.callbackOnSyncRequest = callbackOnSyncRequest; - this.connectAndAuth(); + + // One watchdog and one renewal check for the life of the client — + // they span reconnects, and only disconnect() stops them. + this.livenessInterval = setInterval(() => this.checkLiveness(), LIVENESS_CHECK_MS); + this.tokenRenewalInterval = setInterval(() => this.maybeRenewToken(), TOKEN_RENEW_CHECK_MS); + void this.connectAndAuth(); } - private connectAndAuth(): void { - this.login().then((res) => { - if (!res?.accessToken) { - console.log('Failed to login', res); - return; - } + /** Log prefix — several Tradovate firms can share one instance's log. */ + private get tag(): string { + return `[tradovate ${this.name}]`; + } - console.log('Logged in', res); - this.accessInfo = res; - - // Clear and restart token renewal interval - if (this.tokenRenewalInterval) clearInterval(this.tokenRenewalInterval); - this.tokenRenewalInterval = setInterval(() => { - if ( - new Date(this.accessInfo.expirationTime).getTime() < - new Date().getTime() + 15 * 60 * 1000 - ) { - this.renewAccessToken(); - } - }, 2 * 60 * 1000); - - // Clear stale WS intervals before opening a new connection - if (this.syncInterval) { clearInterval(this.syncInterval); this.syncInterval = null; } - if (this.heartbeatInterval) { clearInterval(this.heartbeatInterval); this.heartbeatInterval = null; } - - // Reset sync state so scheduler waits for the new sync to complete - this.syncComplete = false; - - const randomnumber = Math.random().toString(36).substring(2, 15); - // Broker endpoint — dialled through the operator's proxy when one - // is configured (lib/outbound-proxy.ts). - this.ws = brokerWebSocket(`wss://demo.tradovateapi.com/v1/websocket?r=${randomnumber}`); - - this.ws.onopen = () => { - console.log('Connected to websocket'); - this.ws.send('authorize\n2\n\n' + this.accessInfo.accessToken); - this.directEventCallbacks[2] = (response: any) => { - this.requestSync(); - this.syncInterval = setInterval(() => this.requestSync(), 60000); - - // Every 2.5 seconds send a heartbeat - this.heartbeatInterval = setInterval(() => { - this.ws.send('[]'); - }, 2500); - }; - }; - - this.ws.onmessage = (event) => { - if (event.data[0] === 'a') { - const dataString = event.data.slice(1); - - const data: ( - | { - i?: number; // Id - s?: number; // Status - d?: any; // Data - } - | { - e?: string; - d?: { - entityType: TradovateEntityType; - eventType: 'Created' | 'Updated'; - entity: any; - }; - } - )[] = JSON.parse(dataString); - - for (const response of data) { - if ( - 'i' in response && - response.i && - this.directEventCallbacks[response.i] - ) { - this.directEventCallbacks[response.i](response.d); - } - - if ('e' in response && response.e) { - // Handle callbacks - for (const info of this.indirectEventCallbacks) { - if ( - response.d?.entityType === info?.entityType && - response.d?.eventType === info?.eventType && - info?.validator(response.d.entity) - ) { - info?.callback(response.d.entity); - } - } - - // Buffer recent entity events (last 50) - if (response.d?.entityType) { - this.recentEntityEvents.push({ entityType: response.d.entityType, eventType: response.d.eventType, entity: response.d.entity, ts: Date.now() }); - if (this.recentEntityEvents.length > 50) this.recentEntityEvents.shift(); - } - - // Update positions from WebSocket position events - if (response.d?.entityType === 'position' && response.d?.entity) { - const pos = response.d.entity; - if (pos.netPos !== 0) { - this.positions[pos.accountId] = { - contractId: pos.contractId, - netPos: pos.netPos, - netPrice: pos.netPrice, - timestamp: new Date(pos.timestamp), - }; - } else { - delete this.positions[pos.accountId]; - } - } - - // Update cash balances from WebSocket cashBalance events - if (response.d?.entityType === 'cashBalance' && response.d?.entity) { - const cb = response.d.entity; - if (cb.accountId) { - this.accountCashBalances[cb.accountId] = { - amount: cb.amount, - realizedPnL: cb.realizedPnL, - }; - } - } - } - } - } else if (event.data[0] === 'h') { - // Heartbeat response - } - }; - - this.ws.onerror = (event) => { - console.error('Error on websocket', event); - }; - - this.ws.onclose = (event) => { + /** + * Logs in (unless the current token is still good) and opens a fresh + * socket. Single-flight, never throws, never leaves a rejection unhandled: + * every way it can fail ends in scheduleReconnect(). + */ + private async connectAndAuth(): Promise { + if (this.connecting || this.intentionalDisconnect) return; + this.connecting = true; + try { + // A reconnect after a plain socket drop reuses a token with life + // left in it — fewer logins, fewer rate-limit penalties while + // Tradovate flaps during maintenance. Anything that makes the token + // suspect sets needsLogin. + if (this.needsLogin || !this.tokenValidFor(TOKEN_REUSE_MIN_MS)) { + const outcome = await this.attemptLogin(); if (this.intentionalDisconnect) return; - console.warn(`[tradovate] WebSocket closed (code=${event.code}) — reconnecting in 5s`); - if (this.syncInterval) { clearInterval(this.syncInterval); this.syncInterval = null; } - if (this.heartbeatInterval) { clearInterval(this.heartbeatInterval); this.heartbeatInterval = null; } - setTimeout(() => this.connectAndAuth(), 5_000); - }; + if (!outcome.ok) { + this.consecutiveLoginFailures++; + this.lastLoginError = outcome.error; + this.scheduleReconnect( + `login failed: ${outcome.error} (${this.consecutiveLoginFailures} in a row)`, + outcome.retryInMs ?? reconnectBackoffMs(this.consecutiveLoginFailures), + ); + return; + } + this.accessInfo = outcome.info; + this.noteTokenExpiry(); + this.lastLoginAt = Date.now(); + this.lastLoginError = null; + this.consecutiveLoginFailures = 0; + this.needsLogin = false; + console.log(`${this.tag} logged in — token expires ${outcome.info.expirationTime}`); + } + this.openSocket(); + } catch (err) { + // Nothing above is expected to throw (brokerWebSocket can, on a bad + // proxy setting) — but a throw must never end the reconnect loop. + this.consecutiveSocketFailures++; + this.scheduleReconnect(`connect failed: ${errorMessage(err)}`, reconnectBackoffMs(this.consecutiveSocketFailures)); + } finally { + this.connecting = false; + } + } + + /** + * One login attempt, riding out up to MAX_PENALTY_ROUNDS rate-limit + * penalties: Tradovate answers a throttled login with a p-ticket to present + * again after p-time seconds. p-captcha means it wants a human — never try + * to solve it, back off for a long while instead. + */ + private async attemptLogin(): Promise { + let pTicket: string | undefined; + for (let round = 0; ; round++) { + let body: LoginResponseBody | undefined; + try { + body = await this.login(pTicket); + } catch (err) { + // axios rejects every non-2xx (and timeouts); a penalty ticket + // could ride on one of those too. + const data = (err as { response?: { data?: unknown } })?.response?.data as LoginResponseBody | undefined; + if (data && typeof data === 'object' && data['p-ticket']) { + body = data; + } else { + const text = data && typeof data === 'object' && typeof data.errorText === 'string' ? `: ${data.errorText}` : ''; + return { ok: false, error: `${errorMessage(err)}${text}` }; + } + } + if (this.intentionalDisconnect) return { ok: false, error: 'client disconnected' }; + + if (body?.accessToken) return { ok: true, info: body as AuthLoginResponse }; + if (body?.['p-captcha']) { + return { ok: false, error: 'Tradovate asked for a captcha (too many logins)', retryInMs: CAPTCHA_RETRY_MS }; + } + const ticket = body?.['p-ticket']; + const pTime = Number(body?.['p-time']); + if (ticket && pTime > 0) { + if (round >= MAX_PENALTY_ROUNDS) { + return { ok: false, error: `still rate-limited after ${round} penalty waits` }; + } + const waitMs = Math.min(pTime * 1000, PENALTY_MAX_WAIT_MS); + console.warn(`${this.tag} login rate-limited — waiting ${fmtDelay(waitMs)} to retry with the penalty ticket`); + pTicket = ticket; + await this.penaltySleep(waitMs); + if (this.intentionalDisconnect) return { ok: false, error: 'client disconnected' }; + continue; + } + return { ok: false, error: body?.errorText ? String(body.errorText) : 'login answered without an access token' }; + } + } + + private penaltySleep(ms: number): Promise { + return new Promise((resolve) => { + this.nextReconnectAt = Date.now() + ms; + const timer = setTimeout(() => { + this.penaltyWait = null; + this.nextReconnectAt = 0; + resolve(); + }, ms); + this.penaltyWait = { timer, resolve }; }); } - private requestSync(): void { - this.directEventCallbacks[3] = (response: any) => { - try { - if (!response) { - console.error('[requestSync] Received null/undefined response — auth may have failed'); - return; + /** + * Arms the one reconnect timer. Never stacks: if one is already pending it + * stays as it is (a login backoff isn't cut short by a later socket event). + */ + private scheduleReconnect(reason: string, delayMs = SOCKET_DROP_RECONNECT_MS): void { + if (this.intentionalDisconnect) return; + if (this.reconnectTimer) { + console.warn(`${this.tag} ${reason} — reconnect already due in ${fmtDelay(Math.max(0, this.nextReconnectAt - Date.now()))}`); + return; + } + const wait = jitter(delayMs); + this.nextReconnectAt = Date.now() + wait; + console.warn(`${this.tag} ${reason} — reconnecting in ${fmtDelay(wait)}`); + this.reconnectTimer = setTimeout(() => { + this.reconnectTimer = null; + this.nextReconnectAt = 0; + void this.connectAndAuth(); + }, wait); + } + + private openSocket(): void { + // Only ever one live socket per client — a leftover would keep feeding + // callbacks and heartbeats alongside the new one. + if (this.ws) this.dropSocket('replaced by a new connection'); + + const randomnumber = Math.random().toString(36).substring(2, 15); + // Broker endpoint — dialled through the operator's proxy when one + // is configured (lib/outbound-proxy.ts). + const ws = brokerWebSocket(`wss://demo.tradovateapi.com/v1/websocket?r=${randomnumber}`); + this.ws = ws; + this.socketOpenedAt = Date.now(); + this.authorizeSent = false; + this.authorizedAt = 0; + this.closingSeenAt = 0; + // Reset sync state so scheduler waits for the new sync to complete + this.syncComplete = false; + + // Every handler first checks it still belongs to the current socket: + // a replaced socket's late events must not touch the new session. + ws.onopen = () => { + if (this.ws !== ws) return; + this.socketOpenedAt = Date.now(); // the authorize deadline runs from here + console.log(`${this.tag} websocket connected — authorizing`); + this.directEventCallbacks[2] = (_response: unknown, status?: number) => this.onAuthorized(ws, status); + if (this.sendFrame('authorize\n2\n\n' + this.accessInfo.accessToken)) { + this.authorizeSent = true; + } else { + this.abandonSocket('authorize could not be sent'); + } + }; + + ws.onmessage = (event) => { + if (this.ws !== ws) return; + // Any frame ('o', 'h', 'a', …) proves the session is alive. + this.lastInboundAt = Date.now(); + this.handleFrame(event.data); + }; + + ws.onerror = (event) => { + if (this.ws !== ws) return; + // onclose follows and does the recovery; just note it (no event dump). + const msg = 'message' in event ? (event as ErrorEvent).message : ''; + console.warn(`${this.tag} websocket error${typeof msg === 'string' && msg ? `: ${msg}` : ''}`); + }; + + ws.onclose = (event) => { + if (this.ws !== ws) return; + this.abandonSocket(`websocket closed (code=${event.code})`); + }; + } + + private onAuthorized(ws: WebSocket, status?: number): void { + if (this.ws !== ws) return; + if (typeof status === 'number' && status !== 200) { + // The token was refused — another socket with it would be refused too. + this.abandonSocket(`websocket authorize refused (status ${status})`, true); + return; + } + this.authorizedAt = Date.now(); + console.log(`${this.tag} websocket authorized — syncing`); + + this.clearSessionIntervals(); + this.requestSync(); + this.syncInterval = setInterval(() => { + if (this.ws === ws) this.requestSync(); + }, 60000); + + // Every 2.5 seconds send a heartbeat + this.heartbeatInterval = setInterval(() => { + if (this.ws === ws) this.sendFrame('[]'); + }, 2500); + } + + private handleFrame(raw: unknown): void { + // 'o' open, 'h' heartbeat, 'c' close — nothing to do beyond liveness. + if (typeof raw !== 'string' || raw[0] !== 'a') return; + + let data: ( + | { + i?: number; // Id + s?: number; // Status + d?: any; // Data + } + | { + e?: string; + d?: { + entityType: TradovateEntityType; + eventType: 'Created' | 'Updated'; + entity: any; + }; + } + )[]; + try { + data = JSON.parse(raw.slice(1)); + } catch { + console.warn(`${this.tag} unparseable frame (${raw.length} bytes) ignored`); + return; + } + if (!Array.isArray(data)) return; + + for (const response of data) { + if (!response || typeof response !== 'object') continue; + + if ( + 'i' in response && + response.i && + this.directEventCallbacks[response.i] + ) { + // One-shot: removed before it runs, so a handler that re-registers + // its id (the fixed ids 2/3/10/30 are per request) isn't undone. + const id = response.i; + const callback = this.directEventCallbacks[id]; + delete this.directEventCallbacks[id]; + delete this.pendingRejecters[id]; + try { + callback(response.d, response.s); + } catch (err) { + console.error(`${this.tag} handler for request ${id} failed: ${errorMessage(err)}`); } + } + + if ('e' in response && response.e) { + // Handle callbacks — all one-shot (fill waits), so take them out + // of the list before running them. + const matched = this.indirectEventCallbacks.filter((info) => + response.d?.entityType === info.entityType && + response.d?.eventType === info.eventType && + info.validator(response.d.entity) + ); + if (matched.length > 0) { + this.indirectEventCallbacks = this.indirectEventCallbacks.filter((info) => !matched.includes(info)); + for (const info of matched) { + try { + info.callback(response.d?.entity); + } catch (err) { + console.error(`${this.tag} ${info.entityType} handler failed: ${errorMessage(err)}`); + } + } + } + + // Buffer recent entity events (last 50) + if (response.d?.entityType) { + this.recentEntityEvents.push({ entityType: response.d.entityType, eventType: response.d.eventType, entity: response.d.entity, ts: Date.now() }); + if (this.recentEntityEvents.length > 50) this.recentEntityEvents.shift(); + } + + // Update positions from WebSocket position events + if (response.d?.entityType === 'position' && response.d?.entity) { + const pos = response.d.entity; + if (pos.netPos !== 0) { + this.positions[pos.accountId] = { + contractId: pos.contractId, + netPos: pos.netPos, + netPrice: pos.netPrice, + timestamp: new Date(pos.timestamp), + }; + } else { + delete this.positions[pos.accountId]; + } + } + + // Update cash balances from WebSocket cashBalance events + if (response.d?.entityType === 'cashBalance' && response.d?.entity) { + const cb = response.d.entity; + if (cb.accountId) { + this.accountCashBalances[cb.accountId] = { + amount: cb.amount, + realizedPnL: cb.realizedPnL, + }; + } + } + } + } + } + + /** + * Gives up on the current socket (if any) and schedules the way back. + * Used for every way a session ends — onclose, and the watchdog for the + * ways that never fire onclose. `relogin` means the token itself is + * suspect: log in afresh, after a short delay. + */ + private abandonSocket(reason: string, relogin = false): void { + if (this.intentionalDisconnect) return; + if (relogin) this.needsLogin = true; + // A connect in flight opens a fresh socket anyway (and logs in first + // if needsLogin was just set before its login) — don't race it. + if (this.connecting) return; + + const hadSocket = this.ws !== null; + const wasAuthorized = this.authorizedAt > 0; + const tokenPresented = this.authorizeSent; + // lastSyncAt only moves on a good sync, and one on this socket can't + // predate its open. + const synced = wasAuthorized && this.lastSyncAt >= this.socketOpenedAt; + this.dropSocket(reason); + + let delay = relogin ? FORCED_RELOGIN_DELAY_MS : SOCKET_DROP_RECONNECT_MS; + if (hadSocket && !synced) { + // A session that died before it produced a single good sync backs + // off if it keeps happening, rather than reconnecting every few + // seconds — and, when it forces a login (a sync refused with 401 + // on every fresh session), rather than logging in every second. + // If it presented the token and got no yes, the token is suspect + // too: log in afresh. A handshake that never completed says + // nothing about the token — keep it, so a websocket-host outage + // doesn't turn into a login every few seconds (and a rate-limit + // penalty on top). + if (tokenPresented && !wasAuthorized) this.needsLogin = true; + this.consecutiveSocketFailures++; + if (this.consecutiveSocketFailures > 1) delay = reconnectBackoffMs(this.consecutiveSocketFailures); + } + this.scheduleReconnect(reason, delay); + } + + /** Detaches and closes the current socket and clears everything that + * belonged to its session. Pending requests fail with `reason`. */ + private dropSocket(reason: string): void { + const ws = this.ws; + this.ws = null; + if (ws) { + ws.onopen = null; + ws.onmessage = null; + ws.onerror = null; + ws.onclose = null; + try { ws.close(); } catch { /* already closing */ } + } + this.clearSessionIntervals(); + this.syncComplete = false; + this.authorizeSent = false; + this.authorizedAt = 0; + this.closingSeenAt = 0; + this.syncRequestedAt = 0; + this.failPendingRequests(reason); + } + + private clearSessionIntervals(): void { + if (this.syncInterval) { clearInterval(this.syncInterval); this.syncInterval = null; } + if (this.heartbeatInterval) { clearInterval(this.heartbeatInterval); this.heartbeatInterval = null; } + } + + /** Answers to requests sent on a socket can't arrive on the next one: + * reject the awaited ones, drop the internal ones (authorize, sync, + * product list — re-sent on the new socket). Fill watchers stay: fills + * pushed after the reconnect still arrive. */ + private failPendingRequests(reason: string): void { + const rejecters = Object.values(this.pendingRejecters); + this.pendingRejecters = {}; + this.directEventCallbacks = {}; + for (const reject of rejecters) reject(reason); + } + + /** Every 15s: catch the ways a session dies without onclose, and make sure + * something is always pending while the client is meant to be connected. */ + private checkLiveness(): void { + // A connect in flight is bounded (HTTP deadlines, capped penalty waits). + if (this.intentionalDisconnect || this.connecting) return; + try { + const problem = this.livenessProblem(Date.now()); + if (problem) { + this.abandonSocket(problem.reason, problem.relogin); + return; + } + if (!this.ws && !this.reconnectTimer) this.scheduleReconnect('nothing scheduled'); + } catch (err) { + console.error(`${this.tag} liveness check failed: ${errorMessage(err)}`); + } + } + + private livenessProblem(now: number): { reason: string; relogin?: boolean } | null { + const ws = this.ws; + if (!ws) return null; + + const state = ws.readyState; + if (state === WebSocket.CLOSING || state === WebSocket.CLOSED) { + // onclose normally takes it from here — unless it never fires. + if (!this.closingSeenAt) this.closingSeenAt = now; + else if (now - this.closingSeenAt >= STUCK_CLOSING_MS) return { reason: 'websocket stuck closing' }; + return null; + } + this.closingSeenAt = 0; + + // Renewal normally keeps this from happening; a session on an expired + // token is dead whatever the socket says. + const expiresAt = this.tokenExpiresAt(); + if (expiresAt > 0 && expiresAt <= now) return { reason: 'access token expired', relogin: true }; + + if (state === WebSocket.CONNECTING) { + return now - this.socketOpenedAt > HANDSHAKE_TIMEOUT_MS ? { reason: 'websocket handshake timed out' } : null; + } + + if (!this.authorizedAt) { + return now - this.socketOpenedAt > AUTHORIZE_TIMEOUT_MS + ? { reason: 'websocket authorize never answered', relogin: true } + : null; + } + if (now - Math.max(this.lastInboundAt, this.authorizedAt) > INBOUND_SILENCE_MS) { + return { reason: `no frames for ${INBOUND_SILENCE_MS / 1000}s` }; + } + if (now - this.authorizedAt > SYNC_STALE_MS && now - this.lastSyncAt > SYNC_STALE_MS) { + return { reason: 'sync responses stopped', relogin: true }; + } + return null; + } + + /** Sends a frame on the current socket if it is open. Never throws. */ + private sendFrame(frame: string): boolean { + const ws = this.ws; + if (!ws || ws.readyState !== WebSocket.OPEN) return false; + try { + ws.send(frame); + return true; + } catch (err) { + console.warn(`${this.tag} websocket send failed: ${errorMessage(err)}`); + return false; + } + } + + /** Why `ws` can't carry a request right now, or null if it can. */ + private socketProblem(ws: WebSocket | null): string | null { + if (!ws) return 'Tradovate socket not connected'; + if (ws.readyState !== WebSocket.OPEN) return `Tradovate socket ${READY_STATE_NAMES[ws.readyState] ?? 'not open'}`; + if (!this.authorizedAt) return 'Tradovate socket not authorized yet'; + return null; + } + + /** + * Sends one request on the socket and settles on its answer; it can't + * hang. No answer within `timeoutMs`, or the socket going away first, + * rejects with a plain Error — ambiguous, the broker may have acted on it. + * Failing before anything was sent rejects with OrderNotPlacedError. + * `onAnswer` runs synchronously inside the frame handler, so a fill + * watcher it registers still sees events later in the same frame. + */ + private sendRequest( + endpoint: string, + rest: string, + timeoutMs: number, + onAnswer: (data: unknown, status?: number) => T | PromiseLike, + ): Promise { + const ws = this.ws; + const problem = this.socketProblem(ws); + if (!ws || problem) return Promise.reject(new OrderNotPlacedError(`${endpoint} not sent — ${problem}`)); + + const id = this.getMsgId(); + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { + delete this.directEventCallbacks[id]; + delete this.pendingRejecters[id]; + reject(new Error(`${endpoint}: no answer from Tradovate within ${timeoutMs / 1000}s — outcome unknown`)); + }, timeoutMs); + this.directEventCallbacks[id] = (data: unknown, status?: number) => { + clearTimeout(timer); + try { + resolve(onAnswer(data, status)); + } catch (err) { + reject(err); + } + }; + this.pendingRejecters[id] = (reason: string) => { + clearTimeout(timer); + reject(new Error(`${endpoint}: ${reason} before Tradovate answered — outcome unknown`)); + }; + try { + ws.send(`${endpoint}\n${id}\n${rest}`); + } catch (err) { + clearTimeout(timer); + delete this.directEventCallbacks[id]; + delete this.pendingRejecters[id]; + reject(new OrderNotPlacedError(`${endpoint} not sent — ${errorMessage(err)}`)); + } + }); + } + + /** Runs the sync-complete handler without letting it reject unhandled. */ + private notifySyncComplete(): void { + try { + this.callbackOnSyncRequest().catch((err) => { + console.error(`${this.tag} sync-complete handler failed: ${errorMessage(err)}`); + }); + } catch (err) { + console.error(`${this.tag} sync-complete handler failed: ${errorMessage(err)}`); + } + } + + private requestSync(): void { + if (this.socketProblem(this.ws)) return; + this.directEventCallbacks[3] = (response: any, status?: number) => { + this.syncRequestedAt = 0; + // A refused or empty answer must not wipe the accounts, positions + // and balances we have — nor pass for a fresh sync. + if (!response || typeof response === 'string' || (typeof status === 'number' && status !== 200)) { + console.error(`${this.tag} sync request failed (status ${status ?? '?'}): ${typeof response === 'string' && response ? response : 'empty response'}`); + // 401: the session is gone — no point waiting for the watchdog. + if (status === 401) this.abandonSocket('sync refused (status 401)', true); + return; + } + try { // liquidateOnly flag lives in accountRiskStatuses const riskStatusById: { [accountId: number]: { liquidateOnly?: string } } = ( response.accountRiskStatuses || [] @@ -324,31 +849,41 @@ export class TradovateClient { {} as { [accountId: number]: { contractId: number; netPos: number; netPrice: number; timestamp: Date } } ); - console.log(`[requestSync] ${this.accountList.length} account(s), ${Object.keys(this.accountCashBalances).length} balance(s)`); + this.lastSyncAt = Date.now(); + // A session counts as working once it has synced (abandonSocket). + this.consecutiveSocketFailures = 0; + console.log(`${this.tag} sync:${this.accountList.length} account(s), ${Object.keys(this.accountCashBalances).length} balance(s)`); this.maybeFetchDaysTraded(); - if (this.products.length > 0) { + // The product list only feeds fetchInstrumentFees. Wait for it + // on a session's first sync, but not for ever: still unanswered + // at the next sync, it is lost — sync without it (a late answer + // still loads it) rather than hold syncComplete false, and every + // firm on the instance with it, while the syncs themselves are good. + if (this.products.length > 0 || this.directEventCallbacks[30]) { this.syncComplete = true; - this.callbackOnSyncRequest(); + this.notifySyncComplete(); return; } this.directEventCallbacks[30] = (products: any) => { if (Array.isArray(products) && products.length > 0) { this.products = products.map((p: any) => ({ id: p.id, name: p.name })); - console.log(`Loaded ${this.products.length} products`); + console.log(`${this.tag} loaded ${this.products.length} products`); } this.syncComplete = true; - this.callbackOnSyncRequest(); + this.notifySyncComplete(); }; - this.ws.send('product/list\n30\n\n'); + this.sendFrame('product/list\n30\n\n'); } catch (err) { - console.error('[requestSync] Error processing sync response:', err); + console.error(`${this.tag} error processing sync response: ${errorMessage(err)}`); } }; - this.ws.send('user/syncrequest\n3\n\n{"splitResponses":false}'); + if (this.sendFrame('user/syncrequest\n3\n\n{"splitResponses":false}') && !this.syncRequestedAt) { + this.syncRequestedAt = Date.now(); + } } private maybeFetchDaysTraded(): void { @@ -379,7 +914,7 @@ export class TradovateClient { .then(({ failedAccounts }) => { if (failedAccounts > 0) { this.nextDaysFetchRetryAt = Date.now() + 5 * 60 * 1_000; - console.warn(`[fetchDaysTraded] ${failedAccounts} account(s) failed during ${reason} — retrying in 5 minutes`); + console.warn(`${this.tag} fetchDaysTraded: ${failedAccounts} account(s) failed during ${reason} — retrying in 5 minutes`); return; } @@ -387,7 +922,8 @@ export class TradovateClient { }) .catch((err) => { this.nextDaysFetchRetryAt = Date.now() + 5 * 60 * 1_000; - console.error(`[fetchDaysTraded] ${reason} failed — retrying in 5 minutes`, err); + // errorMessage(): an AxiosError logged whole dumps the Bearer token. + console.error(`${this.tag} fetchDaysTraded ${reason} failed — retrying in 5 minutes: ${errorMessage(err)}`); }) .finally(() => { this.daysFetchInFlight = false; @@ -477,6 +1013,9 @@ export class TradovateClient { const maxAttempts = 3; let lastErr: unknown = null; for (let attempt = 1; attempt <= maxAttempts; attempt++) { + // A discarded client stops polling rather than keep a report + // conversation going on a session nobody uses any more. + if (this.intentionalDisconnect) throw new Error('client disconnected'); try { let reportData = (await axios.post( 'https://rpt-demo.tradovateapi.com/v1/reports/requestreport', @@ -490,16 +1029,17 @@ export class TradovateClient { { name: 'account', value: accountName }, ], }, - { headers: authHeaders } + { headers: authHeaders, timeout: REPORT_TIMEOUT_MS } )).data; let pollAttempts = 0; while (reportData?.['p-ticket'] && pollAttempts < 30) { const pTicket: string = reportData['p-ticket']; const pTime: number = Math.max(1, reportData['p-time'] ?? 1); await new Promise((r) => setTimeout(r, pTime * 1000)); + if (this.intentionalDisconnect) throw new Error('client disconnected'); reportData = (await axios.get( 'https://rpt-demo.tradovateapi.com/v1/reports/getreport', - { params: { 'p-ticket': pTicket }, headers: authHeaders } + { params: { 'p-ticket': pTicket }, headers: authHeaders, timeout: REPORT_TIMEOUT_MS } )).data; pollAttempts++; } @@ -508,7 +1048,7 @@ export class TradovateClient { lastErr = err; if (!isTransient(err) || attempt === maxAttempts) throw err; const backoffMs = 500 * Math.pow(2, attempt - 1); // 500ms, 1s, 2s - console.warn(`[requestReport] ${accountName} ${name}: transient error (attempt ${attempt}/${maxAttempts}), retrying in ${backoffMs}ms`); + console.warn(`${this.tag} requestReport ${accountName} ${name}: transient error (${errorMessage(err)}, attempt ${attempt}/${maxAttempts}), retrying in ${backoffMs}ms`); await new Promise((r) => setTimeout(r, backoffMs)); } } @@ -529,6 +1069,9 @@ export class TradovateClient { }; for (const account of this.accountList) { + // A discarded client (credentials changed, firm removed) stops here + // rather than keep writing rows next to its replacement's. + if (this.intentionalDisconnect) break; if (onlyAccountIds && !onlyAccountIds.includes(account.id)) continue; // Load last known fund date and fund transactions from DB const storedFundDate = loadAccountMeta(account.id, 'last_fund_date'); @@ -704,13 +1247,14 @@ export class TradovateClient { saveDailyPnL(account.id, account.name, fresh); // upsert only fresh entries — preserves older cached rows this.lastFetchErrors[account.name] = FILLS_FALLBACK_NOTE; } catch (err) { + if (this.intentionalDisconnect) break; const msg = err instanceof Error ? err.message : String(err); - console.error(`[fetchDaysTraded] both reports failed for ${account.name}:`, msg); + console.error(`${this.tag} fetchDaysTraded: both reports failed for ${account.name}: ${msg}`); failedAccounts++; // Cache already loaded and filtered at top of loop — just log the error if (cached.length > 0) { this.lastFetchErrors[account.name] = `${msg} (using ${cached.length} cached entries from database)`; - console.log(`[fetchDaysTraded] using ${cached.length} cached entries from DB for ${account.name}`); + console.log(`${this.tag} fetchDaysTraded: using ${cached.length} cached entries from DB for ${account.name}`); } else { this.dailyPnL[account.id] ??= []; this.daysTraded[account.id] ??= 0; @@ -722,7 +1266,7 @@ export class TradovateClient { return { failedAccounts }; } - private async login(): Promise { + private async login(pTicket?: string): Promise { const chl = '' + (Date.now() - 1581e9); const sec = computeSec(this.name, this.password, this.deviceId, chl); @@ -733,6 +1277,9 @@ export class TradovateClient { return btoa(r); }; + // Non-2xx and timeouts reject (callers log errorMessage(), never the + // AxiosError itself — its config carries this body). The abort signal + // backs the timeout up for a connect that never gets a socket. const res = await axios.post('https://live.tradovateapi.com/v1/auth/accesstokenrequest', { name: this.name, password: generateEncodedPassword({ name: this.name, password: this.password }), @@ -744,33 +1291,100 @@ export class TradovateClient { chl: chl, sec: sec, enc: true, - }); + ...(pTicket ? { 'p-ticket': pTicket } : {}), + }, { timeout: HTTP_TIMEOUT_MS, signal: AbortSignal.timeout(HTTP_TIMEOUT_MS + 5_000) }); - if (res.status !== 200) { - console.error('Failed to login', res.data); + return res.data ?? {}; + } + + /** Access token expiry as epoch ms (0 = none, unparseable, or not + * comparable with this machine's clock — see noteTokenExpiry()). */ + private tokenExpiresAt(): number { + if (this.tokenExpiryUntrusted) return 0; + const raw = this.accessInfo?.expirationTime; + if (!raw) return 0; + const t = new Date(raw).getTime(); + return Number.isFinite(t) ? t : 0; + } + + /** + * Called whenever a token arrives (login or renewal). Tradovate hands out + * tokens with well over an hour to live, so one that is already expired — + * or nearly — by our clock means this machine's clock is ahead of + * Tradovate's, not that the token is dead. Trusting that expiry would have + * the watchdog "expire" every fresh token and log in again every 15s (a + * rate-limit penalty, then a captcha, for every client on this IP). + * Treat the expiry as unknown instead: renew every 2 minutes and log in on + * reconnect, as the code did before it read the expiry at all. + */ + private noteTokenExpiry(): void { + const raw = this.accessInfo?.expirationTime; + const t = raw ? new Date(raw).getTime() : NaN; + const untrusted = Number.isFinite(t) && t - Date.now() < TOKEN_RENEW_WINDOW_MS; + if (untrusted && !this.tokenExpiryUntrusted) { + console.warn(`${this.tag} new token expires ${raw}, which is (nearly) past by this machine's clock — check the system clock; ignoring the expiry`); } + this.tokenExpiryUntrusted = untrusted; + } - return res.data; + /** Holding a token known to stay valid for at least `ms` more. */ + private tokenValidFor(ms: number): boolean { + return !!this.accessInfo?.accessToken && this.tokenExpiresAt() > Date.now() + ms; + } + + /** The 2-minute renewal check. Never rejects, never overlaps itself. */ + private maybeRenewToken(): void { + if (this.intentionalDisconnect || this.connecting || this.renewInFlight || !this.accessInfo?.accessToken) return; + const now = Date.now(); + const expiresAt = this.tokenExpiresAt(); + // An unparseable expiry renews anyway — an extra renewal beats a silent expiry. + if (expiresAt > 0 && expiresAt - now > TOKEN_RENEW_WINDOW_MS) return; + // Already expired: renewal can't save it; the watchdog / next connect logs in. + if (expiresAt > 0 && expiresAt <= now) return; + + this.renewInFlight = true; + this.renewAccessToken() + .catch((err) => console.error(`${this.tag} token renewal failed: ${errorMessage(err)}`)) + .finally(() => { this.renewInFlight = false; }); } private async renewAccessToken(): Promise { - const res = await axios.get('https://live.tradovateapi.com/v1/auth/renewAccessToken', { - headers: { - Authorization: `Bearer ${this.accessInfo.accessToken}`, - }, - }); + const token = this.accessInfo?.accessToken; + if (!token) return; + try { + const res = await axios.get('https://live.tradovateapi.com/v1/auth/renewAccessToken', { + headers: { + Authorization: `Bearer ${token}`, + }, + timeout: HTTP_TIMEOUT_MS, + signal: AbortSignal.timeout(HTTP_TIMEOUT_MS + 5_000), + }); + if (this.intentionalDisconnect) return; + // A re-login while this was in flight already replaced the token. + if (this.accessInfo?.accessToken !== token) return; - if (res.status !== 200) { - console.error('Failed to renew access token', res.data); + const body = res.data as LoginResponseBody | undefined; + if (!body?.accessToken) { + throw new Error(body?.errorText ? String(body.errorText) : 'renewal answered without an access token'); + } + // Merge, so a renewal answer without expirationTime can't leave it undefined. + this.accessInfo = { ...this.accessInfo, ...body } as AuthLoginResponse; + this.noteTokenExpiry(); + console.log(`${this.tag} renewed access token — expires ${this.accessInfo.expirationTime}`); + } catch (err) { + if (this.intentionalDisconnect || this.accessInfo?.accessToken !== token) return; + console.warn(`${this.tag} token renewal failed: ${errorMessage(err)}`); + const status = (err as { response?: { status?: unknown } })?.response?.status; + const expiresAt = this.tokenExpiresAt(); + if (status === 401 || (expiresAt > 0 && expiresAt - Date.now() < TOKEN_RELOGIN_WINDOW_MS)) { + this.abandonSocket('token renewal failed', true); + } } - - console.log('Renewed access token'); - this.accessInfo = res.data; } async getAccountList(): Promise { if (!this.accessInfo?.accessToken) { - console.log('Not authenticated'); + console.log(`${this.tag} not authenticated`); return []; } @@ -778,6 +1392,7 @@ export class TradovateClient { headers: { Authorization: `Bearer ${this.accessInfo.accessToken}`, }, + timeout: HTTP_TIMEOUT_MS, }); return res.data; @@ -790,22 +1405,39 @@ export class TradovateClient { const cached = getCachedContract(productName); if (cached) return { id: cached.id, name: cached.name, tickSize: cached.tickSize }; - // If the cache is cold, resolve on demand with the same logic the settings page uses. - const resolved = await resolveContracts([productName], this.accessInfo.accessToken); - const contract = resolved[productName]; - if (contract) { - return { id: contract.id, name: contract.name, tickSize: contract.tickSize }; + // Lookups fail (a 401 on an expiring token, a network blip) — that must + // come back as "no contract", not a throw that aborts every firm's trade. + try { + // If the cache is cold, resolve on demand with the same logic the settings page uses. + // Under a deadline: its lookups (Yahoo, then Tradovate) must not + // hold every firm's trade. A late answer still fills the cache. + const resolved = await withDeadline( + resolveContracts([productName], this.accessInfo.accessToken), + CONTRACT_RESOLVE_DEADLINE_MS, + 'contract resolve', + ); + const contract = resolved[productName]; + if (contract) { + return { id: contract.id, name: contract.name, tickSize: contract.tickSize }; + } + } catch (err) { + console.warn(`${this.tag} contract resolve for ${productName} failed: ${errorMessage(err)} — trying contract/suggest`); } - // Last resort: fall back to Tradovate's ordered suggestions. - const res = await axios.get( - `https://demo.tradovateapi.com/v1/contract/suggest?t=${encodeURIComponent(productName)}&l=20`, - { headers: { Authorization: `Bearer ${this.accessInfo.accessToken}` } } - ); - const contracts: Array<{ id: number; name: string; status: string; providerTickSize: number }> = res.data ?? []; - const match = contracts.find((c) => c.name.startsWith(productName)); - if (!match) return null; - return { id: match.id, name: match.name, tickSize: match.providerTickSize ?? 0.25 }; + try { + // Last resort: fall back to Tradovate's ordered suggestions. + const res = await axios.get( + `https://demo.tradovateapi.com/v1/contract/suggest?t=${encodeURIComponent(productName)}&l=20`, + { headers: { Authorization: `Bearer ${this.accessInfo.accessToken}` }, timeout: HTTP_TIMEOUT_MS } + ); + const contracts: Array<{ id: number; name: string; status: string; providerTickSize: number }> = res.data ?? []; + const match = contracts.find((c) => c.name.startsWith(productName)); + if (!match) return null; + return { id: match.id, name: match.name, tickSize: match.providerTickSize ?? 0.25 }; + } catch (err) { + console.error(`${this.tag} contract lookup for ${productName} failed: ${errorMessage(err)}`); + return null; + } } async fetchInstrumentFees(symbols: string[]): Promise<{ [symbol: string]: number }> { @@ -818,7 +1450,7 @@ export class TradovateClient { const res = await axios.post( 'https://demo.tradovateapi.com/v1/contract/getproductfeeparams', { productIds }, - { headers: { Authorization: `Bearer ${this.accessInfo.accessToken}` } } + { headers: { Authorization: `Bearer ${this.accessInfo.accessToken}` }, timeout: HTTP_TIMEOUT_MS } ); const result: { [symbol: string]: number } = {}; @@ -835,7 +1467,7 @@ export class TradovateClient { (param.orderRoutingFee ?? 0); result[product.name] = parseFloat(raw.toFixed(4)); console.log( - `[fees] ${product.name}: clearing=${param.clearingFee ?? 0}` + + `${this.tag} fees ${product.name}: clearing=${param.clearingFee ?? 0}` + ` exchange=${param.exchangeFee ?? 0}` + ` nfa=${param.nfaFee ?? 0}` + ` brokerage=${param.brokerageFee ?? 0}` + @@ -852,20 +1484,72 @@ export class TradovateClient { async requestContractsFromSocket(names: string[]): Promise<{ [name: string]: Contract; }> { - if (!this.ws) { - console.log('Websocket not connected'); + // Callers expect an object — a dead socket or a lost answer is {} with + // a warning, never a promise left pending. + try { + return await this.sendRequest('contract/finds', `names=${names.join(',')}`, CONTRACT_FIND_TIMEOUT_MS, (response) => { + const items: Contract[] = Array.isArray(response) ? response : []; + return items.reduce((acc: { [name: string]: Contract }, item: Contract) => { + return { ...acc, [item.name]: item }; + }, {} as { [name: string]: Contract }); + }); + } catch (err) { + console.warn(`${this.tag} contract/finds failed: ${errorMessage(err)}`); return {}; } + } - return new Promise((resolve) => { - this.directEventCallbacks[10] = (response: any) => { - resolve( - (response || []).reduce((acc: { [name: string]: Contract }, item: Contract) => { - return { ...acc, [item.name]: item }; - }, {} as { [name: string]: Contract }) - ); + /** A string or empty ack is the broker refusing the order — nothing was placed. */ + private static acceptedAck(response: unknown, endpoint: string): { orderId?: number; id?: number; commandStatus?: string } { + if (typeof response === 'string') throw new OrderNotPlacedError(response || `${endpoint} refused`); + if (!response) throw new OrderNotPlacedError(`Empty response from ${endpoint}`); + if (typeof response !== 'object') return {}; + const ack = response as { orderId?: number; id?: number; commandStatus?: string; failureReason?: string; failureText?: string }; + // PlaceOrderResult / PlaceOCOResult: a refusal (SessionClosed, + // LiquidationOnly, …) carries failureReason and no order id — no order + // exists. Silently taking it as placed left a position with no exit + // (placeoco) or waited 30s for a fill that could never come. + if (ack.orderId == null && ack.id == null + && ((ack.failureReason && ack.failureReason !== 'Success') || ack.failureText)) { + throw new OrderNotPlacedError(`${endpoint} refused: ${ack.failureText || ack.failureReason}`); + } + return ack; + } + + /** Resolves with the fill event for `orderId` (it carries the real + * execution price). Registered synchronously from the ack handler, so a + * fill later in the same frame as the ack is not missed. */ + private waitForFill(orderId: number | undefined): Promise> { + // Fills are pushed on the event stream, independently of the ack: one + // that got ahead of it (an earlier frame, or earlier in the same one) + // is already in the event buffer. Missed, a filled entry came back + // "outcome unknown" and the caller placed no exit on an open position. + const early = orderId == null ? undefined : this.recentEntityEvents.find((ev) => + ev.entityType === 'fill' && ev.eventType === 'Created' && ev.entity?.orderId === orderId); + if (early) { + console.log(`${this.tag} fill (ahead of its ack):`, JSON.stringify(early.entity)); + return Promise.resolve(early.entity); + } + return new Promise((resolve, reject) => { + const watcher: IndirectCallback = { + entityType: 'fill', + eventType: 'Created', + validator: (item: any) => item?.orderId === orderId, + callback: (fill: any) => { + clearTimeout(timeout); + console.log(`${this.tag} fill:`, JSON.stringify(fill)); + resolve(fill); + }, + cancel: (reason: string) => { + clearTimeout(timeout); + reject(new Error(`Order ${orderId} acknowledged but ${reason} before its fill arrived — outcome unknown`)); + }, }; - this.ws.send(`contract/finds\n10\nnames=${names.join(',')}`); + const timeout = setTimeout(() => { + this.indirectEventCallbacks = this.indirectEventCallbacks.filter((info) => info !== watcher); + reject(new Error(`Order ${orderId} acknowledged but no fill within 30s (market may be closed)`)); + }, FILL_TIMEOUT_MS); + this.indirectEventCallbacks.push(watcher); }); } @@ -877,58 +1561,45 @@ export class TradovateClient { orderType: 'Market' | 'Limit', price?: number ): Promise> { - if (!this.ws) throw new Error('WebSocket not connected'); + // Checked before the audit frame so a dead socket is a clean + // "not placed" rather than a send into nothing. + const problem = this.socketProblem(this.ws); + if (problem) throw new OrderNotPlacedError(`order/placeorder not sent — ${problem}`); - this.ws.send( + if (!this.sendFrame( `user/registeraudituseraction\n25\n\n${JSON.stringify({ accountId, actionType: action + orderType, details: `${action} ${quantity} ${contractSymbol} ${orderType}${price ? ` @ ${price}` : ''}, TIF Day`, })}` - ); + )) { + throw new OrderNotPlacedError('order/placeorder not sent — Tradovate socket rejected the audit frame'); + } - return new Promise((resolve, reject) => { - const msgId = this.getMsgId(); - this.directEventCallbacks[msgId] = (response: any) => { - console.log(`[sendOrder] raw ack:`, JSON.stringify(response)); - if (typeof response === 'string' || !response) { - reject(new Error(typeof response === 'string' ? response : 'Empty response from order/placeorder')); - return; - } + return this.sendRequest( + 'order/placeorder', + `\n${JSON.stringify({ + accountId, + action, + symbol: contractSymbol, + orderQty: quantity, + orderType, + price, + timeInForce: 'Day', + text: 'DOM', + })}`, + ACK_TIMEOUT_MS, + (response) => { + console.log(`${this.tag} sendOrder raw ack:`, JSON.stringify(response)); + const ack = TradovateClient.acceptedAck(response, 'order/placeorder'); // Tradovate returns the command object: { id, commandStatus, orderId, ... } - const orderId = response.orderId ?? response.id; - console.log(`[sendOrder] orderId=${orderId} status=${response.commandStatus}`); + const orderId = ack.orderId ?? ack.id; + console.log(`${this.tag} sendOrder orderId=${orderId} status=${ack.commandStatus}`); // Wait for the fill event — it carries the real execution price - const timeout = setTimeout(() => { - reject(new Error(`Order ${orderId} acknowledged but no fill within 30s (market may be closed)`)); - }, 30000); - - this.indirectEventCallbacks.push({ - entityType: 'fill', - eventType: 'Created', - validator: (item: any) => item?.orderId === orderId, - callback: (fill: any) => { - clearTimeout(timeout); - console.log(`[sendOrder] fill:`, JSON.stringify(fill)); - resolve(fill); - }, - }); - }; - - this.ws.send( - `order/placeorder\n${msgId}\n\n${JSON.stringify({ - accountId, - action, - symbol: contractSymbol, - orderQty: quantity, - orderType, - price, - timeInForce: 'Day', - text: 'DOM', - })}` - ); - }); + return this.waitForFill(orderId); + }, + ); } /** Place an order and resolve as soon as the command is acknowledged (does not wait for fill). */ @@ -940,31 +1611,24 @@ export class TradovateClient { orderType: 'Market' | 'Limit', price?: number ): Promise<{ orderId?: number }> { - if (!this.ws) throw new Error('WebSocket not connected'); - - const msgId = this.getMsgId(); - return new Promise((resolve, reject) => { - this.directEventCallbacks[msgId] = (response: any) => { - if (typeof response === 'string') { - reject(new Error(response)); - return; - } - resolve({ orderId: response?.orderId ?? response?.id }); - }; - - this.ws.send( - `order/placeorder\n${msgId}\n\n${JSON.stringify({ - accountId, - action, - symbol: contractSymbol, - orderQty: quantity, - orderType, - price, - timeInForce: 'Day', - text: 'DOM', - })}` - ); - }); + return this.sendRequest( + 'order/placeorder', + `\n${JSON.stringify({ + accountId, + action, + symbol: contractSymbol, + orderQty: quantity, + orderType, + price, + timeInForce: 'Day', + text: 'DOM', + })}`, + ACK_TIMEOUT_MS, + (response) => { + const ack = TradovateClient.acceptedAck(response, 'order/placeorder'); + return { orderId: ack.orderId ?? ack.id }; + }, + ); } /** Place an OCO exit pair after an entry fill: a limit take-profit linked @@ -979,49 +1643,78 @@ export class TradovateClient { limitPrice: number, stopPrice: number, ): Promise<{ orderId?: number }> { - if (!this.ws) throw new Error('WebSocket not connected'); - - const msgId = this.getMsgId(); - return new Promise((resolve, reject) => { - this.directEventCallbacks[msgId] = (response: any) => { - if (typeof response === 'string') { - reject(new Error(response)); - return; - } - resolve({ orderId: response?.orderId ?? response?.id }); - }; - - this.ws.send( - `order/placeoco\n${msgId}\n\n${JSON.stringify({ - accountId, + return this.sendRequest( + 'order/placeoco', + `\n${JSON.stringify({ + accountId, + action: exitAction, + symbol: contractSymbol, + orderQty: quantity, + orderType: 'Limit', + price: limitPrice, + timeInForce: 'Day', + text: 'DOM', + other: { action: exitAction, - symbol: contractSymbol, - orderQty: quantity, - orderType: 'Limit', - price: limitPrice, + orderType: 'Stop', + stopPrice, timeInForce: 'Day', text: 'DOM', - other: { - action: exitAction, - orderType: 'Stop', - stopPrice, - timeInForce: 'Day', - text: 'DOM', - }, - })}` - ); - }); + }, + })}`, + ACK_TIMEOUT_MS, + (response) => { + const ack = TradovateClient.acceptedAck(response, 'order/placeoco'); + return { orderId: ack.orderId ?? ack.id }; + }, + ); } - /** Close the WebSocket connection and stop all intervals. Call before discarding the instance. */ + /** Liveness snapshot for /api/health and the registry supervisor. No tokens. */ + public health(): ClientHealth { + const now = Date.now(); + const expiresAt = this.tokenExpiresAt(); + return { + platform: 'tradovate', + authenticated: !!this.accessInfo?.accessToken && (expiresAt === 0 || expiresAt > now), + transport: this.ws ? (READY_STATE_NAMES[this.ws.readyState] ?? String(this.ws.readyState)) : 'none', + syncComplete: this.syncComplete, + lastSyncAt: this.lastSyncAt, + lastLoginAt: this.lastLoginAt, + lastLoginError: this.lastLoginError, + consecutiveLoginFailures: this.consecutiveLoginFailures, + nextReconnectAt: this.nextReconnectAt, + tokenExpiresAt: expiresAt, + lastInboundAt: this.lastInboundAt, + pendingRequests: Object.keys(this.directEventCallbacks).length + this.indirectEventCallbacks.length, + syncInFlightSince: this.syncRequestedAt, + daysFetchInFlightSince: this.daysFetchInFlight ? this.lastDaysFetchStartedAt : 0, + }; + } + + /** Close the WebSocket connection and stop all timers. Call before discarding + * the instance — nothing it scheduled fires afterwards. */ public disconnect(): void { this.intentionalDisconnect = true; - if (this.syncInterval) { clearInterval(this.syncInterval); this.syncInterval = null; } - if (this.heartbeatInterval) { clearInterval(this.heartbeatInterval); this.heartbeatInterval = null; } + if (this.reconnectTimer) { clearTimeout(this.reconnectTimer); this.reconnectTimer = null; } + this.nextReconnectAt = 0; + if (this.livenessInterval) { clearInterval(this.livenessInterval); this.livenessInterval = null; } if (this.tokenRenewalInterval) { clearInterval(this.tokenRenewalInterval); this.tokenRenewalInterval = null; } + if (this.penaltyWait) { + // Cut the wait short; the login loop sees intentionalDisconnect and stops. + const { timer, resolve } = this.penaltyWait; + this.penaltyWait = null; + clearTimeout(timer); + resolve(); + } + // Closes the socket with its handlers detached, clears sync/heartbeat, + // rejects pending acks (ambiguous: they may have been placed). + this.dropSocket('client disconnected'); + const watchers = this.indirectEventCallbacks; + this.indirectEventCallbacks = []; + for (const info of watchers) info.cancel?.('client disconnected'); this.daysFetchInFlight = false; this.nextDaysFetchRetryAt = 0; - try { this.ws?.close(); } catch { /* ignore */ } } /** Register a one-time callback for when a fill arrives for a given orderId. */ diff --git a/lib/version.ts b/lib/version.ts index d265b81..29e490b 100644 --- a/lib/version.ts +++ b/lib/version.ts @@ -22,12 +22,16 @@ export interface BuildVersion { time: string; } -let cached: BuildVersion | null | undefined; +// Per process, not per bundle: instrumentation.ts and the route handlers load +// their own copies of this module, and each would otherwise read HEAD at its own +// first call — after an update's pull, a late first call reports code that isn't +// running yet. +const _g = globalThis as typeof globalThis & { __buildVersion?: BuildVersion | null }; /** null when this isn't a git checkout or git isn't installed — both mean * "no version to report", and the master shows the machine as an old build. */ export function getBuildVersion(): BuildVersion | null { - if (cached !== undefined) return cached; + if (_g.__buildVersion !== undefined) return _g.__buildVersion; try { const out = execFileSync('git', ['log', '-1', '--format=%h %cI'], { cwd: process.cwd(), @@ -35,9 +39,9 @@ export function getBuildVersion(): BuildVersion | null { stdio: ['ignore', 'pipe', 'ignore'], }).trim(); const [commit, time] = out.split(' '); - cached = commit && time ? { commit, time } : null; + _g.__buildVersion = commit && time ? { commit, time } : null; } catch { - cached = null; + _g.__buildVersion = null; } - return cached; + return _g.__buildVersion; } diff --git a/scripts/start-all.mjs b/scripts/start-all.mjs index 70bba17..f295bc2 100644 --- a/scripts/start-all.mjs +++ b/scripts/start-all.mjs @@ -4,10 +4,12 @@ * * Run by the "at log on" scheduled task. Safe to run by hand at any time. * - * The warm-up is not cosmetic. getClients() in lib/clients.ts is lazily - * bootstrapped — Tradovate clients, the contract resolver, the reporter and the - * persisted-schedule resume all start inside it — so until something makes an - * HTTP request the process sits idle and none of that happens. + * The warm-up used to be what started trading: getClients() in lib/clients.ts + * was only bootstrapped by the first HTTP request. Builds with + * instrumentation.ts boot it when the server starts, so the warm-up is now a + * readiness check — it waits for each instance to answer, loads the dashboard + * state once, and logs the scheduler and /api/health status. (On an older + * build it still does the bootstrapping, so it stays.) */ import { spawnSync } from 'node:child_process'; import { ensurePythonDeps } from './ensure-python-deps.mjs'; @@ -58,7 +60,7 @@ for (const instance of loadInstances()) { continue; } - log(`warming up ${instance.name} (this is what triggers the client bootstrap)`); + log(`warming up ${instance.name}`); try { await fetch(`${base}/api/state`, { signal: AbortSignal.timeout(60_000) }); } catch (err) { @@ -71,6 +73,27 @@ for (const instance of loadInstances()) { ? `${instance.name}: scheduler resumed — ${status.action} ${status.symbol}` : `${instance.name}: scheduler is stopped`); } catch { /* non-critical */ } + + // One line of client health. Right after boot a client or two may still + // be logging in, so this reports; it doesn't fail the run. Builds from + // before /api/health answer 404 — nothing to say then. + try { + const res = await fetch(`${base}/api/health`, { signal: AbortSignal.timeout(10_000) }); + if (res.status !== 404) { + const health = await res.json(); + if (!res.ok) { + log(`${instance.name}: /api/health failed — ${health.error ?? `HTTP ${res.status}`}`); + } else { + // Firms without a client yet aren't failing to sync — same + // rule as the route's own `ok` (and the updater's line). + const clients = (Array.isArray(health.clients) ? health.clients : []) + .filter((c) => c?.hasClient !== false); + const waiting = clients.filter((c) => !c.ready).map((c) => c.firm); + log(`${instance.name}: ${clients.length - waiting.length}/${clients.length} firm(s) ready` + + (waiting.length > 0 ? ` — not ready yet: ${waiting.join(', ')}` : '')); + } + } + } catch { /* non-critical */ } } if (failed) process.exit(1);