From b932452e985b5f88d53a99aef8b7681699f08bde Mon Sep 17 00:00:00 2001 From: Brandon Li Date: Wed, 23 Sep 2026 20:18:33 -0500 Subject: [PATCH] Self-healing broker clients, per-firm scheduler isolation, eager boot MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Why the autotrader sometimes broke after a weekend until `pm2 restart`: `next start` only logs uncaught errors, so nothing ever crashed, and a few paths could leave a client dead for the life of the process. Tradovate: a closed socket scheduled exactly one re-login, with no catch — one failure during weekend maintenance (or a rate-limit penalty returned as a 200 without a token) and the client never came back. connectAndAuth is now a single-flight loop that never throws: jittered backoff capped at 5 min, penalty tickets waited out, captcha demands backed off. A 15s watchdog abandons silent, never-authorized or stuck-closing sockets and stale syncs; renewal falls back to a fresh login; orders fail fast (OrderNotPlacedError) on a dead socket and time out instead of hanging; a bad sync answer no longer wipes accounts and positions. TopstepX: hub start/stop/invoke and every REST call are bounded; a wedged hub is discarded and rebuilt; a stuck sync is superseded after 3 min; re-login no longer tears down the sync loop; a watchdog re-logs when syncs go stale; a no-fill timeout cancels the order. Scheduler: one unready client used to hold every firm forever. It now holds the tick for a 10-minute grace, then trades the other firms without it. Per-firm try/catch in batch collection, one batch at a time, accounts marked in flight, legs un-counted only when the entry never reached the broker, and the scheduler starts before the first batch fires. Boot and supervision: instrumentation.ts boots the process at server start (it used to wait for the first dashboard request); a supervisor rebuilds a client that has been unhealthy for 20 min (not a TopstepX one holding a position); broker HTTP calls default to a 60s timeout; GET /api/health shows each client's liveness and the scheduler's last decision. Shared state the eager boot would otherwise split across bundles lives on globalThis. Logs no longer carry access tokens or whole AxiosErrors (which include the encoded Tradovate password). Co-Authored-By: Claude Opus 5.5 --- app/api/health/route.ts | 72 +++ app/api/trade/route.ts | 17 +- instrumentation.ts | 40 ++ lib/auto-trade.ts | 704 ++++++++++++++++---- lib/clients.ts | 268 +++++++- lib/contract-resolver.ts | 12 +- lib/master-link.ts | 14 +- lib/outbound-proxy.ts | 113 +++- lib/platform-client.ts | 72 +++ lib/topstepx-class.ts | 748 ++++++++++++++++++---- lib/tradovate-class.ts | 1313 +++++++++++++++++++++++++++++--------- lib/version.ts | 14 +- scripts/start-all.mjs | 33 +- 13 files changed, 2793 insertions(+), 627 deletions(-) create mode 100644 app/api/health/route.ts create mode 100644 instrumentation.ts diff --git a/app/api/health/route.ts b/app/api/health/route.ts new file mode 100644 index 0000000..64f0718 --- /dev/null +++ b/app/api/health/route.ts @@ -0,0 +1,72 @@ +import { readFileSync } from 'node:fs'; +import path from 'node:path'; +import { NextResponse } from 'next/server'; +import { getClients } from '@/lib/clients'; +import { getTradableFirms } from '@/lib/firm-catalog'; +import { getSchedulerHealth, getSchedulerStatus } from '@/lib/auto-trade'; +import { getBuildVersion } from '@/lib/version'; +import { errorMessage, isClientReady } from '@/lib/platform-client'; + +// Liveness has to be computed now, never served from a cache. +export const dynamic = 'force-dynamic'; + +/** A one-line marker file, or null when it isn't there. */ +function readMarker(...segments: string[]): string | null { + try { + return readFileSync(path.join(process.cwd(), ...segments), 'utf8').trim() || null; + } catch { + return null; + } +} + +/** + * GET /api/health — is this instance able to trade, and if not, why. + * + * The numbers to read before restarting a broken box, and what start-all + * prints after boot. Always 200 when the handler runs: `ok` in the body is the + * verdict (every tradable firm that has a client is synced, and synced + * recently), so a caller can tell "unhealthy" from "not answering". + * + * Calls getClients(), so hitting it also boots an instance that somehow + * hasn't been. + * + * Built field by field on purpose — firm rows carry passwords and API keys, + * and nothing here may echo a credential or a token. + */ +export async function GET() { + try { + const now = Date.now(); + const clients = getClients(); + + const entries = getTradableFirms().map((firm) => { + const base = { firmId: firm.id, firm: firm.name, platform: firm.platform }; + const client = clients.get(firm.id); + if (!client) return { ...base, hasClient: false, ready: false, accounts: 0 }; + + const ready = isClientReady(client, now); + const accounts = client.accountList.length; + try { + return { ...base, hasClient: true, ready, accounts, ...client.health() }; + } catch (err) { + // One client's broken health() must not hide the rest. + return { ...base, hasClient: true, ready, accounts, healthError: errorMessage(err) }; + } + }); + + return NextResponse.json({ + ok: entries.every((e) => !e.hasClient || e.ready), + now, + uptimeSec: Math.round(process.uptime()), + version: getBuildVersion(), + // Read from disk on every call: the build .next holds now (the + // running one, unless a build landed since this process started), + // and the commit the updater last finished building. + buildId: readMarker('.next', 'BUILD_ID'), + builtCommit: readMarker('scripts', '.built-commit'), + scheduler: { ...getSchedulerStatus(), ...getSchedulerHealth() }, + clients: entries, + }); + } catch (err) { + return NextResponse.json({ ok: false, error: errorMessage(err) }, { status: 500 }); + } +} diff --git a/app/api/trade/route.ts b/app/api/trade/route.ts index f08d6cd..f5e90cc 100644 --- a/app/api/trade/route.ts +++ b/app/api/trade/route.ts @@ -1,5 +1,6 @@ import { NextRequest, NextResponse } from 'next/server'; import { runTrade, startScheduler } from '@/lib/auto-trade'; +import { errorMessage } from '@/lib/platform-client'; import { POINT_VALUES } from '@/lib/trading-logic'; export async function POST(req: NextRequest) { @@ -15,15 +16,19 @@ export async function POST(req: NextRequest) { return NextResponse.json({ error: `Unknown symbol: ${symbol}` }, { status: 400 }); } + // (Re)start the scheduler with this action + symbol first, so a batch + // that throws or runs long can never leave it unstarted. It can't + // double-fire against the batch below: the tick waits for flat + // positions, skips while this batch holds the trade mutex, and never + // picks an account whose order is still in flight. + startScheduler(action, symbol, stopAfterAll ?? false); + // Execute trade immediately for all eligible accounts const results = await runTrade(action, symbol); - // (Re)start the scheduler with this action + symbol - startScheduler(action, symbol, stopAfterAll ?? false); - return NextResponse.json(results); - } catch (err: any) { - console.error('[POST /api/trade]', err); - return NextResponse.json({ error: err?.message ?? 'Trade failed' }, { status: 500 }); + } catch (err: unknown) { + console.error(`[POST /api/trade] ${errorMessage(err)}`); + return NextResponse.json({ error: errorMessage(err) || 'Trade failed' }, { status: 500 }); } } diff --git a/instrumentation.ts b/instrumentation.ts new file mode 100644 index 0000000..cc4cd20 --- /dev/null +++ b/instrumentation.ts @@ -0,0 +1,40 @@ +/** + * Server start hook — Next runs register() once per server process, before + * it serves the first request (a root instrumentation.ts is picked up with no + * config in Next 15+). + * + * It boots the trading background work eagerly. getClients() in lib/clients.ts + * starts the broker clients, the contract resolver, the reporter, the master + * link, the client supervisor and the persisted-schedule resume — and it used + * to run only on the first HTTP request. So a restarted instance (a crash, an + * update, pm2 after a reboot) sat idle, not trading and invisible to the + * master, until someone opened the dashboard or start-all's warm-up curl + * happened to reach it. + * + * The boot must never take the server down with it: a failure is logged, and + * if the boot never got going the first request still starts it through the + * same lazy getClients() path as before. + * + * This module is its own server bundle, so it can hold its own instances of + * lib/ modules, separate from the route handlers'. Anything the two must share + * (the client pool, the scheduler, timers) lives on globalThis for that reason. + */ + +export async function register(): Promise { + // The nodejs check is the documented shape for a Node-only import, kept + // exactly: the edge compile (Turbopack always builds one) inlines + // NEXT_RUNTIME and drops the block, so lib/clients — sqlite, sockets — + // never reaches the edge bundle, where it would fail the build. The build + // check is defense in depth (Next itself doesn't call register() during + // `next build`) — a build must never dial brokers or resume the scheduler. + if (process.env.NEXT_RUNTIME === 'nodejs') { + if (process.env.NEXT_PHASE === 'phase-production-build') return; + try { + const { getClients } = await import('./lib/clients'); + getClients(); + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + console.error(`[instrumentation] eager boot failed — clients will start on the first request: ${message}`); + } + } +} diff --git a/lib/auto-trade.ts b/lib/auto-trade.ts index ac32a71..eda0935 100644 --- a/lib/auto-trade.ts +++ b/lib/auto-trade.ts @@ -18,39 +18,302 @@ import { type DayPlan, } from './trading-logic'; import { getSetting, setSetting } from './db'; +import { errorMessage, isClientReady, isOrderNotPlaced, type PlatformClient } from './platform-client'; + +// ── process-wide state ─────────────────────────────────────────────────────── + +/** + * Everything the batches share, held on globalThis like the scheduler state + * further down. Next can bundle this module more than once — the scheduler + * tick runs from whichever bundle booted the clients, POST /api/trade and + * /api/copy-trade from their own route bundles — and plain module-level state + * would split into one private copy per bundle: a leg recorded by the tick + * would be invisible to a copy-trade, the trade mutex would lock nothing, and + * the session-open jitter could be rolled twice in one day. + */ +interface TradeRuntime { + /** In-memory runaway guard: legs fired per account per trade date, plus + * the sum of their targets — realized P&L falling short of that sum is + * how a stop-out is detected. Resets on restart, which is fine — day + * completion is derived from realized PnL, not from this counter. */ + legCounts: Map; + /** Today's session-open delay, keyed by Central calendar day (see openJitterMinutes). */ + sessionOpenJitter: Map; + /** Accounts whose orders are being placed right now. Their position only + * shows once the fill comes back over the socket, so until then this is + * what keeps them out of every other batch. */ + accountsInFlight: Set; + /** Firm id → when the tick first saw its client not ready (see UNREADY_GRACE_MS). */ + unreadySince: Map; + /** Firm id → when the tick last logged it as quarantined (rate limit). */ + quarantineLoggedAt: Map; + /** When the batch holding the trade mutex took it (0 = no batch in flight). */ + tradeInFlightSince: number; + // What getSchedulerHealth reports. + lastTickAt: number; + lastTickOutcome: TickOutcome; + lastFireAt: number; + holding: string[]; + quarantined: string[]; +} + +/** What the scheduler's last tick decided (getSchedulerHealth). */ +export type TickOutcome = + | 'not-running' | 'holding-for-sync' | 'positions-open' | 'closed' + | 'no-eligible' | 'fired' | 'skipped-in-flight' | `error: ${string}`; + +const _tradeGlobal = globalThis as typeof globalThis & { __autoTradeRuntime?: TradeRuntime }; + +function runtime(): TradeRuntime { + if (!_tradeGlobal.__autoTradeRuntime) { + _tradeGlobal.__autoTradeRuntime = { + legCounts: new Map(), + sessionOpenJitter: new Map(), + accountsInFlight: new Set(), + unreadySince: new Map(), + quarantineLoggedAt: new Map(), + tradeInFlightSince: 0, + lastTickAt: 0, + lastTickOutcome: 'not-running', + lastFireAt: 0, + holding: [], + quarantined: [], + }; + } + return _tradeGlobal.__autoTradeRuntime; +} + +/** How long the tick holds every firm for one whose client isn't ready. Long + * enough for a restart or a reconnect to finish syncing; past it the client + * is presumed dead and its firm is quarantined (see assessReadiness). */ +const UNREADY_GRACE_MS = 10 * 60 * 1000; +/** How often a quarantined firm is re-announced in the log. */ +const QUARANTINE_LOG_EVERY_MS = 5 * 60 * 1000; +/** A batch that has held the trade mutex this long is presumed wedged. */ +const TRADE_WEDGED_MS = 10 * 60 * 1000; // ── helpers ────────────────────────────────────────────────────────────────── +/** How many accounts one batch may fire, never below 1. The settings PATCH + * stores whatever the form sends, and a cleared field arrives as '' — + * parseInt('') is NaN, which slice() takes as 0: every batch would fire + * nobody while the tick reported 'fired'. Non-numeric falls back to the + * default 5, like a missing setting; the form's own floor is 1. */ +function maxConcurrentAccounts(): number { + const configured = parseInt(getSetting('max_concurrent_accounts') ?? '5', 10); + return Math.max(1, Number.isFinite(configured) ? configured : 5); +} + function isAccountDead(amount: number, autoLiqThreshold: number): boolean { // Tradovate floors are positive balances; reconstructed TopstepX MLL // floors are negative (or +$0.01 once locked at breakeven). 0 = no floor. return autoLiqThreshold !== 0 && amount <= autoLiqThreshold; } -// In-memory runaway guard: legs fired per account per trade date, plus the sum -// of their targets — realized P&L falling short of that sum is how a stop-out -// is detected. Resets on restart, which is fine — day completion is derived -// from realized PnL, not from this counter. -const _legCounts = new Map(); - function legsToday(accountId: number, tradeDate: string): number { - const entry = _legCounts.get(accountId); + const entry = runtime().legCounts.get(accountId); return entry && entry.date === tradeDate ? entry.count : 0; } function legSumToday(accountId: number, tradeDate: string): number { - const entry = _legCounts.get(accountId); + const entry = runtime().legCounts.get(accountId); return entry && entry.date === tradeDate ? entry.sum : 0; } function recordLeg(accountId: number, tradeDate: string, legAmount: number): void { - _legCounts.set(accountId, { + runtime().legCounts.set(accountId, { date: tradeDate, count: legsToday(accountId, tradeDate) + 1, sum: Math.round((legSumToday(accountId, tradeDate) + legAmount) * 100) / 100, }); } +/** Take back a leg recordLeg counted for an entry that never reached the + * broker. Left standing, it would bench a stop-type account for the day — + * realized P&L "falls short" of a leg that never traded — and burn one of + * the day's MAX_LEGS_PER_DAY. */ +function unrecordLeg(accountId: number, tradeDate: string, legAmount: number): void { + const legCounts = runtime().legCounts; + const entry = legCounts.get(accountId); + if (!entry || entry.date !== tradeDate || entry.count <= 0) return; + if (entry.count === 1) { + legCounts.delete(accountId); + return; + } + legCounts.set(accountId, { + date: tradeDate, + count: entry.count - 1, + sum: Math.round((entry.sum - legAmount) * 100) / 100, + }); +} + +/** + * Send a leg's market entry. The leg was recorded before this — the runaway + * guard has to see it while the batch is still in flight — so it is taken + * back here only when the client KNOWS the order never reached the broker + * (OrderNotPlacedError). Anything ambiguous, like an ack or fill timeout, + * keeps the leg: the order may well have filled, and a leg counted that never + * happened only costs a leg, while one forgotten that did can trade the + * account twice. + */ +async function sendEntry( + tag: string, + // eslint-disable-next-line @typescript-eslint/no-explicit-any + client: any, + acc: { id: number; name: string }, + contractName: string, + contracts: number, + action: 'Buy' | 'Sell', + tradeDate: string, + legAmount: number, +) { + try { + return await client.sendOrder(acc.id, contractName, contracts, action, 'Market'); + } catch (err) { + if (isOrderNotPlaced(err)) { + unrecordLeg(acc.id, tradeDate, legAmount); + console.warn(`${tag} ${acc.name}: entry never reached the broker — leg un-counted`); + } else { + console.warn(`${tag} ${acc.name}: entry outcome unknown — leg kept (it may have filled)`); + } + throw err; + } +} + +/** Run one account's part of a batch with the account marked in flight, so + * no other batch picks it while its orders are still being placed. */ +async function withAccountInFlight(accountId: number, fire: () => Promise): Promise { + const inFlight = runtime().accountsInFlight; + inFlight.add(accountId); + try { + return await fire(); + } finally { + inFlight.delete(accountId); + } +} + +// ── readiness ──────────────────────────────────────────────────────────────── + +type TradableFirm = ReturnType[number]; +interface FirmClient { firm: TradableFirm; client: PlatformClient } + +/** + * Tradable firms that have a client, split by isClientReady. Everything that + * trades or gates works from `ready` only: a client whose sync is missing or + * stale holds a frozen snapshot, whose positions may be long gone and which + * may be missing ones opened since — it must neither trade nor gate. + */ +function firmClients(now = Date.now()): { ready: FirmClient[]; unready: FirmClient[] } { + const clients = getClients(); + const ready: FirmClient[] = []; + const unready: FirmClient[] = []; + for (const firm of getTradableFirms()) { + const client = clients.get(firm.id); + if (!client) continue; + (isClientReady(client, now) ? ready : unready).push({ firm, client }); + } + return { ready, unready }; +} + +function countOpenPositions(ready: FirmClient[]): number { + return ready.reduce( + (count, { client }) => count + client.accountList.filter((acc) => !!client.positions[acc.id]).length, + 0, + ); +} + +/** + * Which firms hold the tick, and which it trades without. + * + * A firm whose client isn't ready might hold positions nobody can see, so for + * the first UNREADY_GRACE_MS the whole tick waits for it, as it always has — + * right for a restart or a reconnect, which settle within a minute or two. + * Past the grace period the client is presumed dead (broker maintenance, a + * login that keeps failing) and the firm is quarantined: the other firms + * trade without it, rather than one broken session stopping every firm on the + * machine. A quarantined firm trades nothing — runTrade and copyTrade only + * ever see ready clients — and rejoins on its first good sync. + * + * A client that has synced and found no accounts is ignored outright: it + * cannot hold positions or trade, so there is nothing to wait for. + */ +function assessReadiness( + ready: FirmClient[], + unready: FirmClient[], + now: number, +): { holding: TradableFirm[]; quarantined: { firm: TradableFirm; unreadyMs: number }[] } { + const rt = runtime(); + const holding: TradableFirm[] = []; + const quarantined: { firm: TradableFirm; unreadyMs: number }[] = []; + const tracked = new Set(); + + for (const { firm, client } of unready) { + if (client.lastSyncAt > 0 && client.accountList.length === 0) continue; + tracked.add(firm.id); + const since = rt.unreadySince.get(firm.id) ?? now; + rt.unreadySince.set(firm.id, since); + const unreadyMs = now - since; + if (unreadyMs < UNREADY_GRACE_MS) holding.push(firm); + else quarantined.push({ firm, unreadyMs }); + } + + // Forget firms that are ready again (or gone), and say so for any that + // had been traded around. + for (const id of [...rt.unreadySince.keys()]) { + if (tracked.has(id)) continue; + const back = ready.find((r) => r.firm.id === id); + if (back && rt.quarantineLoggedAt.has(id)) { + console.log(`[scheduler] ${back.firm.name} synced again — trading it with the others`); + } + rt.unreadySince.delete(id); + rt.quarantineLoggedAt.delete(id); + } + + return { holding, quarantined }; +} + +// ── trade mutex ────────────────────────────────────────────────────────────── + +/** + * One batch at a time, instance-wide. The tick, a master "trade" command, + * POST /api/trade and /api/copy-trade can all fire a batch, and two batches + * collecting at once would each see the other's accounts as still flat — the + * position gate only trips once a fill has come back over the socket. + * + * A batch that has held the mutex past TRADE_WEDGED_MS is presumed stuck on a + * broker call that will never answer, and the next one goes ahead without it + * rather than nothing trading for the rest of the process's life. The stuck + * batch's accounts stay in accountsInFlight, so they are never traded twice, + * and if the stuck batch does wake up it finds it no longer holds the mutex + * and abandons its now-stale selection (holdsTradeMutex). + * + * Returns the lease to hand to releaseTradeMutex, or null to skip. + */ +function acquireTradeMutex(tag: string): number | null { + const rt = runtime(); + const now = Date.now(); + if (rt.tradeInFlightSince > 0) { + const heldMs = now - rt.tradeInFlightSince; + if (heldMs < TRADE_WEDGED_MS) { + console.log(`${tag} another batch in flight for ${Math.round(heldMs / 1000)}s — skipping`); + return null; + } + console.warn(`${tag} batch in flight for ${Math.round(heldMs / 60_000)}m looks wedged — proceeding without it`); + } + rt.tradeInFlightSince = now; + return now; +} + +function holdsTradeMutex(lease: number): boolean { + return runtime().tradeInFlightSince === lease; +} + +/** Only the holder unlocks: a wedged batch that finally returns must not + * release the batch that took over from it. */ +function releaseTradeMutex(lease: number): void { + if (holdsTradeMutex(lease)) runtime().tradeInFlightSince = 0; +} + /** Obfuscation stop for the current leg, in dollars. Infinity = no stop. * Day 1 of a cycle carries the $1,000 account stop (firstDayStop), min-day * (small) trades a 2×-target stop (minDayStop) — and the day after a losing @@ -134,6 +397,9 @@ function evaluateAccountForTrade(client: any, acc: { id: number; name: string; a if (isAccountDead(cash.amount, autoLiqThreshold)) return { ...base, eligible: false, reason: 'dead' }; if (!acc.active) return { ...base, eligible: false, reason: 'inactive (DLL)' }; if (client.positions[acc.id]) return { ...base, eligible: false, reason: 'already in trade' }; + // Another batch is placing this account's order right now; its position + // won't show until the fill comes back, so the check above can't see it. + if (runtime().accountsInFlight.has(acc.id)) return { ...base, eligible: false, reason: 'order in flight' }; // Obfuscation stops: a stop-out shows as realized P&L falling short of the // legs fired today — the day is booked; the make-back ride is tomorrow. @@ -210,9 +476,9 @@ function evaluateAccountForTrade(client: any, acc: { id: number; name: string; a // Random 1–20 minute delay added to the session open, picked once per Central // calendar day. Re-rolling on every check would move the open boundary between // polls, letting a trade slip through a moment the next check would reject. -const sessionOpenJitter = new Map(); - +// Held in runtime() so every bundle of this module agrees on the day's roll. function openJitterMinutes(centralDay: string): number { + const sessionOpenJitter = runtime().sessionOpenJitter; let jitter = sessionOpenJitter.get(centralDay); if (jitter === undefined) { jitter = 1 + Math.floor(Math.random() * 20); // 1–20 inclusive @@ -290,23 +556,46 @@ function isInNoTradeWindow(): boolean { // ── core trade logic ────────────────────────────────────────────────────────── -export async function runTrade(action: 'Buy' | 'Sell' | 'Auto', symbol: string) { +/** One account's outcome in a batch: its fill summary, or { status: 'error', reason }. */ +type LegResult = { status: string; [key: string]: unknown }; +type FirmTradeResults = { firm: string; results: LegResult[] }; +type TradeOutcome = Extract; + +export async function runTrade(action: 'Buy' | 'Sell' | 'Auto', symbol: string): Promise { + return (await runTradeBatch(action, symbol)).results; +} + +/** runTrade plus what it decided, which the tick records for getSchedulerHealth. */ +async function runTradeBatch( + action: 'Buy' | 'Sell' | 'Auto', + symbol: string, +): Promise<{ outcome: TradeOutcome; results: FirmTradeResults[] }> { + const lease = acquireTradeMutex('[auto-trade]'); + if (lease === null) return { outcome: 'skipped-in-flight', results: [] }; + try { + return await runTradeLocked(action, symbol, lease); + } finally { + releaseTradeMutex(lease); + } +} + +async function runTradeLocked( + action: 'Buy' | 'Sell' | 'Auto', + symbol: string, + lease: number, +): Promise<{ outcome: TradeOutcome; results: FirmTradeResults[] }> { if (isInNoTradeWindow()) { console.log('[auto-trade] CME market closed — skipping'); - return []; + return { outcome: 'closed', results: [] }; } - // Block if any account still has an open position (same gate as the scheduler tick) - const allClients = getClients(); - const allFirms = getTradableFirms(); - const openPositions = allFirms.reduce((count, firm) => { - const c = allClients.get(firm.id); - if (!c) return count; - return count + c.accountList.filter(acc => !!c.positions[acc.id]).length; - }, 0); + // Block if any account still has an open position (same gate as the scheduler tick). + // Ready clients only, like everything below: a stale snapshot must neither trade nor gate. + const { ready } = firmClients(); + const openPositions = countOpenPositions(ready); if (openPositions > 0) { console.log(`[auto-trade] ${openPositions} position(s) still open — skipping`); - return []; + return { outcome: 'positions-open', results: [] }; } // Resolve 'Auto' symbol once per batch so all accounts trade the same symbol @@ -324,9 +613,7 @@ export async function runTrade(action: 'Buy' | 'Sell' | 'Auto', symbol: string) if (!pointValue) throw new Error(`Unknown symbol: ${resolvedSymbol}`); const tradeDate = currentTradeDateCT(); - const maxConcurrent = parseInt(getSetting('max_concurrent_accounts') ?? '5', 10); - const firms = getTradableFirms(); - const clients = getClients(); + const maxConcurrent = maxConcurrentAccounts(); // ── Phase 1: collect ALL eligible accounts across ALL firms in parallel ── type EligibleItem = { @@ -346,9 +633,8 @@ export async function runTrade(action: 'Buy' | 'Sell' | 'Auto', symbol: string) const allEligible: EligibleItem[] = []; - await Promise.all(firms.map(async (firm) => { - const client = clients.get(firm.id); - if (!client || client.accountList.length === 0) return; + await Promise.all(ready.map(async ({ firm, client }) => { + if (client.accountList.length === 0) return; // Skip this firm entirely if the symbol is banned for it if (firm.bannedSymbols.includes(resolvedSymbol)) { @@ -356,26 +642,42 @@ export async function runTrade(action: 'Buy' | 'Sell' | 'Auto', symbol: string) return; } + // One firm's trouble (a contract lookup that throws, a malformed + // snapshot) skips that firm, never the batch — a rejection here would + // fail the Promise.all and every other firm would sit the tick out. + try { + const contract = await client.findFrontMonthContract(resolvedSymbol); + if (!contract) return; - const contract = await client.findFrontMonthContract(resolvedSymbol); - if (!contract) return; + const found: EligibleItem[] = []; + for (const acc of client.accountList) { + const ev = evaluateAccountForTrade(client, acc, tradeDate); + if (!ev || !ev.eligible || !ev.plan) continue; - for (const acc of client.accountList) { - const ev = evaluateAccountForTrade(client, acc, tradeDate); - if (!ev || !ev.eligible || !ev.plan) continue; - - allEligible.push({ - firmName: firm.name, client, acc, contract, - cash: ev.cash, dailyPnL: ev.dailyPnL, daysTraded: ev.daysTraded, - plan: ev.plan, isExtraDay: ev.isExtraDay, drain: ev.drain, - prefix: getAccountConfig(acc.id).accountType?.prefix ?? '', - }); + found.push({ + firmName: firm.name, client, acc, contract, + cash: ev.cash, dailyPnL: ev.dailyPnL, daysTraded: ev.daysTraded, + plan: ev.plan, isExtraDay: ev.isExtraDay, drain: ev.drain, + prefix: getAccountConfig(acc.id).accountType?.prefix ?? '', + }); + } + allEligible.push(...found); + } catch (err) { + console.error(`[auto-trade] ${firm.name}: skipped — ${errorMessage(err)}`); } })); if (allEligible.length === 0) { console.log('[auto-trade] no eligible accounts found'); - return []; + return { outcome: 'no-eligible', results: [] }; + } + + // Collecting took so long that another batch presumed this one wedged and + // took the mutex. That batch traded on fresher numbers than these, so + // firing now could trade an account twice. + if (!holdsTradeMutex(lease)) { + console.warn('[auto-trade] superseded while collecting accounts — abandoning this batch'); + return { outcome: 'skipped-in-flight', results: [] }; } // Take only the first batch — all fired simultaneously, no rolling pool. @@ -395,9 +697,12 @@ export async function runTrade(action: 'Buy' | 'Sell' | 'Auto', symbol: string) .sort((a, b) => b[1].length - a[1].length || a[0].localeCompare(b[0]))[0]; const batch = batchGroup.slice(0, maxConcurrent); console.log(`[auto-trade] ${allEligible.length} eligible account(s) across ${groups.size} prefix group(s) — firing batch of ${batch.length} ${batchPrefix || '(no prefix)'}`); + runtime().lastFireAt = Date.now(); // ── Phase 2: fire the batch simultaneously ── - const tradeResults = await Promise.allSettled(batch.map(async (item) => { + // Each account is marked in flight before anything is placed and released + // once its orders are done, so no overlapping batch can pick it meanwhile. + const tradeResults = await Promise.allSettled(batch.map((item) => withAccountInFlight(item.acc.id, async () => { const { client, acc, contract } = item; const cfg = getAccountConfig(acc.id).accountType!; @@ -443,15 +748,19 @@ export async function runTrade(action: 'Buy' | 'Sell' | 'Auto', symbol: string) minLegs - legsToday(acc.id, tradeDate), ) : item.plan.remainingToday; + // Counted before anything is sent so the runaway guard sees it while + // the batch is in flight; taken back only if the entry never reached + // the broker (here, or in sendEntry). recordLeg(acc.id, tradeDate, legAmount); - const inst = await legInstrument(client, item.drain, contract, pointValue); + const inst = await legInstrument(client, item.drain, contract, pointValue) + .catch((err: unknown) => { unrecordLeg(acc.id, tradeDate, legAmount); throw err; }); const legContract = inst.contract; const pv = inst.pointValue; const rawContracts = Math.max(1, Math.ceil(legAmount / 1000)); const contracts = cfg.maxPositionSize > 0 ? Math.min(rawContracts, cfg.maxPositionSize) : rawContracts; - const fill = await client.sendOrder(acc.id, legContract.name, contracts, resolvedAction, 'Market'); + const fill = await sendEntry('[auto-trade]', client, acc, legContract.name, contracts, resolvedAction, tradeDate, legAmount); // Wait briefly for the cash balance WebSocket update to reflect entry commission await new Promise(r => setTimeout(r, 1000)); @@ -481,9 +790,18 @@ export async function runTrade(action: 'Buy' | 'Sell' | 'Auto', symbol: string) try { exitOrder = await client.placeOcoExit(acc.id, legContract.name, contracts, exitAction, exitPrice, stopPrice); } catch (err) { - // The position is already open — never leave it exit-less. Fall - // back to the classic lone limit and shout about it. - console.error(`[auto-trade] ${acc.name}: placeoco REJECTED (${err instanceof Error ? err.message : err}) — falling back to lone limit exit, NO STOP`); + // The position is already open — never leave it exit-less when + // the pair never went in (refused, or not sent): fall back to + // the classic lone limit and shout about it. An unknown outcome + // (the ack timed out, the connection went after the send) is + // different: the pair may be working, and a lone limit on top + // doubles the exit — both filling flips the account into an + // opposite position nothing manages. Place nothing more then. + if (!isOrderNotPlaced(err)) { + console.error(`[auto-trade] ${acc.name}: placeoco outcome UNKNOWN (${errorMessage(err)}) — NOT placing a second exit; check this position's exits at the broker`); + throw new Error(`entry filled @ ${fill.price} but the exit's outcome is unknown — ${errorMessage(err)}`); + } + console.error(`[auto-trade] ${acc.name}: placeoco REJECTED (${errorMessage(err)}) — falling back to lone limit exit, NO STOP`); stopPrice = null; exitOrder = await client.placeOrderNoWait(acc.id, legContract.name, contracts, exitAction, 'Limit', exitPrice); } @@ -508,22 +826,28 @@ export async function runTrade(action: 'Buy' | 'Sell' | 'Auto', symbol: string) exitPrice, commission: entryCommission, }; - })); + }))); // Group results by firm for the response - const firmResultsMap = new Map(); + const firmResultsMap = new Map(); for (let i = 0; i < batch.length; i++) { const firmName = batch[i].firmName; if (!firmResultsMap.has(firmName)) firmResultsMap.set(firmName, []); const r = tradeResults[i]; + if (r.status === 'rejected') { + console.error(`[auto-trade] ${batch[i].acc.name} (${firmName}): failed — ${errorMessage(r.reason)}`); + } firmResultsMap.get(firmName)!.push( r.status === 'fulfilled' ? r.value - : { status: 'error', reason: (r.reason as any)?.message ?? String(r.reason) } + : { status: 'error', reason: errorMessage(r.reason) } ); } - return Array.from(firmResultsMap.entries()).map(([firm, results]) => ({ firm, results })); + return { + outcome: 'fired', + results: Array.from(firmResultsMap.entries()).map(([firm, results]) => ({ firm, results })), + }; } // ── copy trade ─────────────────────────────────────────────────────────────── @@ -535,14 +859,30 @@ export async function runTrade(action: 'Buy' | 'Sell' | 'Auto', symbol: string) */ export async function copyTrade() { console.log('[copy-trade] called'); + // Shares runTrade's mutex: a copy landing mid-batch would see the batch's + // accounts as still flat and could fill past max_concurrent_accounts. + const lease = acquireTradeMutex('[copy-trade]'); + if (lease === null) return []; + try { + return await copyTradeLocked(lease); + } finally { + releaseTradeMutex(lease); + } +} + +async function copyTradeLocked(lease: number): Promise { if (isInNoTradeWindow()) { console.log('[copy-trade] outside trading hours — skipping'); return []; } - const firms = getTradableFirms(); - const clients = getClients(); - const maxConcurrent = Math.max(1, parseInt(getSetting('max_concurrent_accounts') ?? '5', 10)); + // Ready clients only, as in runTrade: a stale snapshot must neither be + // copied from, counted against the slots, nor traded. + const { ready, unready } = firmClients(); + if (unready.length > 0) { + console.log(`[copy-trade] not synced — leaving out: ${unready.map((u) => u.firm.name).join(', ')}`); + } + const maxConcurrent = maxConcurrentAccounts(); // Find all accounts with open positions to determine direction + symbol let resolvedAction: 'Buy' | 'Sell' | null = null; @@ -554,9 +894,7 @@ export async function copyTrade() { let positionedClient: any = null; const positionedPrefixes = new Set(); - for (const firm of firms) { - const client = clients.get(firm.id); - if (!client) continue; + for (const { client } of ready) { for (const acc of client.accountList) { const pos = client.positions[acc.id]; if (!pos) continue; @@ -585,10 +923,15 @@ export async function copyTrade() { if (positionedClient) { for (const sym of enabledSymbols) { - const contract = await positionedClient.findFrontMonthContract(sym); - if (contract && contract.id === positionContractId) { - resolvedSymbol = sym; - break; + // A lookup that throws rules out that one symbol, not the copy. + try { + const contract = await positionedClient.findFrontMonthContract(sym); + if (contract && contract.id === positionContractId) { + resolvedSymbol = sym; + break; + } + } catch (err) { + console.error(`[copy-trade] ${sym}: contract lookup failed — ${errorMessage(err)}`); } } } @@ -629,10 +972,9 @@ export async function copyTrade() { }; const eligible: CopyItem[] = []; - await Promise.all(firms.map(async (firm) => { - const client = clients.get(firm.id); - if (!client || client.accountList.length === 0) { - console.log(`[copy-trade] ${firm.name}: no client or empty account list — skip firm`); + await Promise.all(ready.map(async ({ firm, client }) => { + if (client.accountList.length === 0) { + console.log(`[copy-trade] ${firm.name}: empty account list — skip firm`); return; } if (firm.bannedSymbols.includes(resolvedSymbol!)) { @@ -640,24 +982,31 @@ export async function copyTrade() { return; } - const contract = await client.findFrontMonthContract(resolvedSymbol!); - if (!contract) { - console.log(`[copy-trade] ${firm.name}: cannot resolve front-month contract for ${resolvedSymbol} — skip firm`); - return; - } + // Same isolation as runTrade: one firm's throw skips that firm only. + try { + const contract = await client.findFrontMonthContract(resolvedSymbol!); + if (!contract) { + console.log(`[copy-trade] ${firm.name}: cannot resolve front-month contract for ${resolvedSymbol} — skip firm`); + return; + } - for (const acc of client.accountList) { - const ev = evaluateAccountForTrade(client, acc, tradeDate); - if (!ev) { console.log(`[copy-trade] ${acc.name}: no matching config prefix — skip`); continue; } - if (!ev.eligible || !ev.plan) { console.log(`[copy-trade] ${acc.name}: ${ev.reason} — skip`); continue; } - // copyTrade has no extra-day branch — those accounts wait for the scheduler - if (ev.isExtraDay) { console.log(`[copy-trade] ${acc.name}: extra-day account — skip`); continue; } + const found: CopyItem[] = []; + for (const acc of client.accountList) { + const ev = evaluateAccountForTrade(client, acc, tradeDate); + if (!ev) { console.log(`[copy-trade] ${acc.name}: no matching config prefix — skip`); continue; } + if (!ev.eligible || !ev.plan) { console.log(`[copy-trade] ${acc.name}: ${ev.reason} — skip`); continue; } + // copyTrade has no extra-day branch — those accounts wait for the scheduler + if (ev.isExtraDay) { console.log(`[copy-trade] ${acc.name}: extra-day account — skip`); continue; } - eligible.push({ - firmName: firm.name, client, acc, contract, cash: ev.cash, dailyPnL: ev.dailyPnL, plan: ev.plan, - drain: ev.drain, - prefix: getAccountConfig(acc.id).accountType?.prefix ?? '', - }); + found.push({ + firmName: firm.name, client, acc, contract, cash: ev.cash, dailyPnL: ev.dailyPnL, plan: ev.plan, + drain: ev.drain, + prefix: getAccountConfig(acc.id).accountType?.prefix ?? '', + }); + } + eligible.push(...found); + } catch (err) { + console.error(`[copy-trade] ${firm.name}: skipped — ${errorMessage(err)}`); } })); @@ -671,10 +1020,17 @@ export async function copyTrade() { return []; } - console.log(`[copy-trade] copying ${resolvedAction} ${resolvedSymbol} to ${batch.length} account(s)`); + // Superseded while collecting (see runTrade) — this selection is stale. + if (!holdsTradeMutex(lease)) { + console.warn('[copy-trade] superseded while collecting accounts — abandoning this copy'); + return []; + } - // Fire orders (same as Phase 2 of runTrade) - const tradeResults = await Promise.allSettled(batch.map(async (item) => { + console.log(`[copy-trade] copying ${resolvedAction} ${resolvedSymbol} to ${batch.length} account(s)`); + runtime().lastFireAt = Date.now(); + + // Fire orders (same as Phase 2 of runTrade, accounts marked in flight the same way) + const tradeResults = await Promise.allSettled(batch.map((item) => withAccountInFlight(item.acc.id, async () => { const { client, acc, contract } = item; const cfg = getAccountConfig(acc.id).accountType!; @@ -686,15 +1042,17 @@ export async function copyTrade() { minLegs - legsToday(acc.id, tradeDate), ) : item.plan.remainingToday; + // Counted before sending, taken back only if the entry never reached the broker (see runTrade). recordLeg(acc.id, tradeDate, legAmount); - const inst = await legInstrument(client, item.drain, contract, pointValue); + const inst = await legInstrument(client, item.drain, contract, pointValue) + .catch((err: unknown) => { unrecordLeg(acc.id, tradeDate, legAmount); throw err; }); const legContract = inst.contract; const pv = inst.pointValue; const rawContracts = Math.max(1, Math.ceil(legAmount / 1000)); const contracts = cfg.maxPositionSize > 0 ? Math.min(rawContracts, cfg.maxPositionSize) : rawContracts; - const fill = await client.sendOrder(acc.id, legContract.name, contracts, resolvedAction!, 'Market'); + const fill = await sendEntry('[copy-trade]', client, acc, legContract.name, contracts, resolvedAction!, tradeDate, legAmount); await new Promise(r => setTimeout(r, 1000)); const updatedCash = client.accountCashBalances[acc.id] ?? { amount: 0, realizedPnL: 0 }; @@ -720,7 +1078,12 @@ export async function copyTrade() { try { exitOrder = await client.placeOcoExit(acc.id, legContract.name, contracts, exitAction, exitPrice, stopPrice); } catch (err) { - console.error(`[copy-trade] ${acc.name}: placeoco REJECTED (${err instanceof Error ? err.message : err}) — falling back to lone limit exit, NO STOP`); + // Lone-limit fallback only for a pair that never went in (see runTrade). + if (!isOrderNotPlaced(err)) { + console.error(`[copy-trade] ${acc.name}: placeoco outcome UNKNOWN (${errorMessage(err)}) — NOT placing a second exit; check this position's exits at the broker`); + throw new Error(`entry filled @ ${fill.price} but the exit's outcome is unknown — ${errorMessage(err)}`); + } + console.error(`[copy-trade] ${acc.name}: placeoco REJECTED (${errorMessage(err)}) — falling back to lone limit exit, NO STOP`); stopPrice = null; exitOrder = await client.placeOrderNoWait(acc.id, legContract.name, contracts, exitAction, 'Limit', exitPrice); } @@ -745,15 +1108,18 @@ export async function copyTrade() { exitPrice, commission: entryCommission, }; - })); + }))); const results: unknown[] = []; for (let i = 0; i < batch.length; i++) { const r = tradeResults[i]; + if (r.status === 'rejected') { + console.error(`[copy-trade] ${batch[i].acc.name} (${batch[i].firmName}): failed — ${errorMessage(r.reason)}`); + } results.push( r.status === 'fulfilled' ? r.value - : { status: 'error', reason: (r.reason as any)?.message ?? String(r.reason) } + : { status: 'error', reason: errorMessage(r.reason) } ); } return results; @@ -763,13 +1129,31 @@ export async function copyTrade() { /** Returns true if any configured account could still trade today (eligible for a next leg / extra-day, or has an open position). */ function hasRemainingConfiguredAccounts(): boolean { - const firms = getTradableFirms(); - const clients = getClients(); + const { ready, unready } = firmClients(); const tradeDate = currentTradeDateCT(); - for (const firm of firms) { - const client = clients.get(firm.id); - if (!client || client.accountList.length === 0) continue; + // A client that isn't ready can't say whether its accounts are done — its + // snapshot is missing or stale — so while one might hold any of ours, the + // scheduler stays up for it to rejoin. Only a client that never synced + // and knows no accounts is an unknown in full; one whose (stale) list + // shows none of ours can be ruled out. + // + // Only while the session is open, though. Outside it the unready firm + // couldn't trade anyway, and after the close its day is over. Unbounded, + // one firm that never comes back (a wrong password, a TopstepX key with + // no username) would keep a stop-after-all scheduler up for good, trading + // the ready firms every session after — exactly what the operator opted + // out of. (Before an open with every ready account already done, this + // stops rather than wait on the unready firm: the safe side.) + if (!isInNoTradeWindow()) { + for (const { client } of unready) { + if (client.lastSyncAt === 0 && client.accountList.length === 0) return true; + if (client.accountList.some((acc) => getAccountConfig(acc.id).accountType)) return true; + } + } + + for (const { client } of ready) { + if (client.accountList.length === 0) continue; for (const acc of client.accountList) { const cfg = getAccountConfig(acc.id).accountType; @@ -777,6 +1161,8 @@ function hasRemainingConfiguredAccounts(): boolean { // Account with open position = still in play if (client.positions[acc.id]) return true; + // Order being placed right now = still in play (evaluation calls it ineligible) + if (runtime().accountsInFlight.has(acc.id)) return true; const ev = evaluateAccountForTrade(client, acc, tradeDate); if (ev?.eligible) return true; @@ -810,6 +1196,21 @@ function getState(): SchedulerState { export function startScheduler(action: 'Buy' | 'Sell' | 'Auto', symbol: string, stopAfterAll: boolean = false) { const state = getState(); + // The settings reads and writes are the part that can throw, so they go + // before anything in memory changes: a failure then leaves whatever + // schedule was running as it was, never one marked running with its + // interval cleared and nothing scheduled. A non-numeric setting falls back + // to 60s — setInterval takes NaN as 1ms. + const configuredSecs = parseInt(getSetting('tick_interval_seconds') ?? '60', 10); + const intervalSecs = Math.max(5, Number.isFinite(configuredSecs) ? configuredSecs : 60); + + // Mirror to the settings table so a restart can pick the schedule back up. + // The interval itself is in-memory only; resumeSchedulerIfPersisted() recreates it. + setSetting('scheduler_running', '1'); + setSetting('scheduler_action', action); + setSetting('scheduler_symbol', symbol); + setSetting('scheduler_stop_after_all', stopAfterAll ? '1' : '0'); + // Clear any existing interval if (state.intervalId !== null) { clearInterval(state.intervalId); @@ -820,43 +1221,44 @@ export function startScheduler(action: 'Buy' | 'Sell' | 'Auto', symbol: string, state.running = true; state.stopAfterAll = stopAfterAll; - // Mirror to the settings table so a restart can pick the schedule back up. - // The interval itself is in-memory only; resumeSchedulerIfPersisted() recreates it. - setSetting('scheduler_running', '1'); - setSetting('scheduler_action', action); - setSetting('scheduler_symbol', symbol); - setSetting('scheduler_stop_after_all', stopAfterAll ? '1' : '0'); - const tick = async () => { if (!state.running) return; state.lastRun = new Date(); - - // Skip this tick until every client has completed its initial sync (positions are populated) - const clients = getClients(); - const firms = getTradableFirms(); - const notReady = firms.filter(f => { - const c = clients.get(f.id); - return c && !c.syncComplete; - }); - if (notReady.length > 0) { - console.log(`[scheduler] waiting for sync: ${notReady.map(f => f.name).join(', ')}`); - return; - } - - // Skip this tick if any account still has an open position from the previous batch - const openPositions = firms.reduce((count, firm) => { - const client = clients.get(firm.id); - if (!client) return count; - return count + client.accountList.filter(acc => !!client.positions[acc.id]).length; - }, 0); - if (openPositions > 0) { - console.log(`[scheduler] ${openPositions} position(s) still open — skipping tick`); - return; - } + const rt = runtime(); + const now = Date.now(); + rt.lastTickAt = now; try { - const results = await runTrade(state.action, state.symbol); - const filled = results.flatMap((r: any) => r.results ?? []).filter((r: any) => r.status === 'filled').length; + // Hold the tick while any firm's client is still inside its sync + // grace period (positions may not be populated yet); trade around + // firms that have been out past it (see assessReadiness). + const { ready, unready } = firmClients(now); + const { holding, quarantined } = assessReadiness(ready, unready, now); + rt.holding = holding.map((f) => f.name); + rt.quarantined = quarantined.map((q) => q.firm.name); + if (holding.length > 0) { + rt.lastTickOutcome = 'holding-for-sync'; + console.log(`[scheduler] waiting for sync: ${rt.holding.join(', ')}`); + return; + } + for (const { firm, unreadyMs } of quarantined) { + if (now - (rt.quarantineLoggedAt.get(firm.id) ?? 0) < QUARANTINE_LOG_EVERY_MS) continue; + rt.quarantineLoggedAt.set(firm.id, now); + console.warn(`[scheduler] ${firm.name} not synced for ${Math.round(unreadyMs / 60_000)}m — trading the other firms without it`); + } + + // Skip this tick if any account still has an open position from the + // previous batch — one batch fires only when every (ready) account is flat. + const openPositions = countOpenPositions(ready); + if (openPositions > 0) { + rt.lastTickOutcome = 'positions-open'; + console.log(`[scheduler] ${openPositions} position(s) still open — skipping tick`); + return; + } + + const { outcome, results } = await runTradeBatch(state.action, state.symbol); + rt.lastTickOutcome = outcome; + const filled = results.flatMap((r) => r.results).filter((r) => r.status === 'filled').length; if (filled > 0) console.log(`[scheduler] tick: ${filled} account(s) filled`); // Auto-stop if user opted in and no configured accounts can trade anymore @@ -865,11 +1267,11 @@ export function startScheduler(action: 'Buy' | 'Sell' | 'Auto', symbol: string, stopScheduler(); } } catch (err) { - console.error('[scheduler] tick error:', err); + rt.lastTickOutcome = `error: ${errorMessage(err)}`; + console.error(`[scheduler] tick error: ${errorMessage(err)}`); } }; - const intervalSecs = Math.max(5, parseInt(getSetting('tick_interval_seconds') ?? '60', 10)); state.intervalId = setInterval(tick, intervalSecs * 1_000); console.log(`[scheduler] started — ${action} ${symbol} every ${intervalSecs}s`); } @@ -881,6 +1283,17 @@ export function stopScheduler() { state.intervalId = null; } state.running = false; + + // Nothing watches readiness while stopped, so the grace clocks would be + // stale by the next start — that start gives every firm a fresh grace. + // Reset before the settings write, which can throw. + const rt = runtime(); + rt.unreadySince.clear(); + rt.quarantineLoggedAt.clear(); + rt.holding = []; + rt.quarantined = []; + rt.lastTickOutcome = 'not-running'; + setSetting('scheduler_running', '0'); console.log('[scheduler] stopped'); } @@ -888,9 +1301,10 @@ export function stopScheduler() { /** * Restart the schedule that was running before the process went down. * - * No sync-wait here on purpose: tick() already skips while any client reports - * !syncComplete, and again while any account holds an open position. So the - * worst case is a few logged no-op ticks until the clients finish syncing. + * No sync-wait here on purpose: tick() already holds while any client isn't + * ready (for up to UNREADY_GRACE_MS each), and again while any account holds + * an open position. So the worst case is a few logged no-op ticks until the + * clients finish syncing. */ export function resumeSchedulerIfPersisted(): void { if (getSetting('scheduler_running') !== '1') return; @@ -915,3 +1329,31 @@ export function getSchedulerStatus() { sessionOpenDelayMin: todaysSessionOpenDelay(), }; } + +/** + * Why the scheduler is or isn't trading, for health checks — separate from + * getSchedulerStatus, whose shape is what the master receives. `holding` are + * the firms the last tick waited on (inside their sync grace period); + * `quarantined` the ones past it, traded around. Timestamps are epoch ms, + * 0 = never; lastFireAt counts batches from any caller, not just the tick. + * Until the first tick after a start, the outcome still reads 'not-running'. + */ +export function getSchedulerHealth(): { + lastTickAt: number; + lastTickOutcome: TickOutcome; + lastFireAt: number; + holding: string[]; + quarantined: string[]; + tradeInFlightSince: number; +} { + const running = getState().running; + const rt = runtime(); + return { + lastTickAt: rt.lastTickAt, + lastTickOutcome: running ? rt.lastTickOutcome : 'not-running', + lastFireAt: rt.lastFireAt, + holding: running ? [...rt.holding] : [], + quarantined: running ? [...rt.quarantined] : [], + tradeInFlightSince: rt.tradeInFlightSince, + }; +} diff --git a/lib/clients.ts b/lib/clients.ts index 8c3211b..f4d3de3 100644 --- a/lib/clients.ts +++ b/lib/clients.ts @@ -1,23 +1,32 @@ import { TradovateClient } from './tradovate-class'; import { TopstepXClient } from './topstepx-class'; -import type { PlatformClient } from './platform-client'; +import { errorMessage, isClientReady, type PlatformClient } from './platform-client'; import { getInstruments } from './db'; import { getTradableFirms, startCatalogSync } from './firm-catalog'; import { resolveFirmAssignments } from './account-type-resolver'; import { resolveContracts } from './contract-resolver'; import { startReporter, sendReport } from './reporter'; import { startBackupSync } from './backup-sync'; -import { applyBrokerProxy } from './outbound-proxy'; +import { applyBrokerHttpDefaults, applyBrokerProxy } from './outbound-proxy'; import { startMasterLink } from './master-link'; import { resumeSchedulerIfPersisted } from './auto-trade'; // Use global to persist the client pool across HMR reloads in dev mode. // Keyed by the master's firm id, which is a uuid string. +// +// It is also what keeps the pool single in production: instrumentation.ts +// boots it from its own server bundle, and the route handlers can each hold a +// separate instance of this module. Registry state therefore lives here, never +// in module-level variables. const g = global as typeof globalThis & { __tradovateClients?: Map; __tradovateClientsInitialized?: boolean; __tradovateClientCreds?: Map; __contractResolverTimer?: ReturnType; + __contractResolveRetryTimer?: ReturnType; + __contractResolveRetries?: number; + __clientSupervisorTimer?: ReturnType; + __clientSupervision?: Map; }; function ensureMap(): Map { @@ -74,7 +83,14 @@ export function initClient( export function removeClient(id: string): void { const client = ensureMap().get(id); - client?.disconnect(); + // A throwing disconnect() must still take the client out of the pool: + // left in, half torn down, it would never reconnect — and the supervisor + // rebuilding it would fail the same way every time. + try { + client?.disconnect(); + } catch (err) { + console.error(`[clients] disconnect failed for firm ${id}: ${errorMessage(err)}`); + } ensureMap().delete(id); credsMap().delete(id); } @@ -127,11 +143,21 @@ export function reconcileClients(): void { export function getClients(): Map { const map = ensureMap(); + + // `next build` imports route modules to collect page data. Nothing there + // should call this, but a build that did must never dial the brokers or + // resume the scheduler — so hand back the empty pool and leave the flag + // unset, as if nothing had happened. + if (process.env.NEXT_PHASE === 'phase-production-build') return map; + if (!g.__tradovateClientsInitialized) { g.__tradovateClientsInitialized = true; try { - // Before anything dials out: put the operator's proxy (if any) - // under the broker clients. Master traffic stays direct by design. + // Before anything dials out: a timeout on every broker HTTP call, + // and the operator's proxy (if any) under the broker clients. + // Master traffic stays direct by design. (outbound-proxy applies + // both when it loads, too; these are idempotent.) + applyBrokerHttpDefaults(); applyBrokerProxy(); // Pull the catalog before anything else — without it there are no @@ -158,6 +184,9 @@ export function getClients(): Map { // Schedule daily resolve at midnight scheduleDailyResolve(); + // Rebuild any client that has been wedged for too long. + startClientSupervisor(); + // Start master dashboard reporter startReporter(); @@ -180,30 +209,67 @@ export function getClients(): Map { // ── Contract auto-resolve ──────────────────────────────────────────────────── +// With no authenticated Tradovate client yet — a cold catalog, a slow login, +// broker maintenance at boot — keep checking once a minute for half an hour +// instead of giving up until midnight. Missing it leaves the contract cache +// cold: the settings page shows nothing, TopstepX loses the resolver's +// front-month names, and the first trade pays for the lookups itself. +const CONTRACT_RESOLVE_RETRY_MS = 60_000; +const CONTRACT_RESOLVE_MAX_RETRIES = 30; + function triggerContractResolve(): void { + // A midnight run supersedes a pending retry; either way only one chain. + if (g.__contractResolveRetryTimer) clearTimeout(g.__contractResolveRetryTimer); + g.__contractResolveRetryTimer = undefined; + const map = ensureMap(); // Find first client with a valid access token (the Yahoo-driven resolver // confirms contract ids against Tradovate; TopstepX clients resolve their // own contracts but still reuse the resolver's front-month names). + // A token alone isn't enough: a client whose session died keeps its + // expired one, and resolving with that fails with nothing left to retry. let accessToken: string | null = null; for (const [, c] of map) { // eslint-disable-next-line @typescript-eslint/no-explicit-any const token = (c as any).accessInfo?.accessToken; - if (token) { accessToken = token; break; } + if (!token) continue; + let authenticated = true; + try { authenticated = c.health().authenticated; } catch { /* no health: trust the token */ } + if (authenticated) { accessToken = token; break; } } if (!accessToken) { - console.log('[contract-resolver] No authenticated Tradovate client yet — skipping resolve'); + const retries = (g.__contractResolveRetries ?? 0) + 1; + if (retries > CONTRACT_RESOLVE_MAX_RETRIES) { + console.log( + `[contract-resolver] No authenticated Tradovate client after ${CONTRACT_RESOLVE_MAX_RETRIES} retries` + + ' — giving up until the midnight resolve', + ); + g.__contractResolveRetries = 0; + return; + } + if (retries === 1) { + console.log('[contract-resolver] No authenticated Tradovate client yet — retrying every 60s'); + } + g.__contractResolveRetries = retries; + g.__contractResolveRetryTimer = setTimeout(() => triggerContractResolve(), CONTRACT_RESOLVE_RETRY_MS); return; } + g.__contractResolveRetries = 0; - const symbols = getInstruments().filter((i) => i.enabled).map((i) => i.symbol); - console.log(`[contract-resolver] Resolving ${symbols.length} instruments...`); - resolveContracts(symbols, accessToken) - .then((results) => { - const rolled = Object.entries(results).filter(([, v]) => v?.alternative); - console.log(`[contract-resolver] Done — ${rolled.length} contract(s) rolled via price match`); - }) - .catch((err) => console.error('[contract-resolver] Resolve failed:', err)); + // Runs from timers: a throw here (the instruments read) would escape as + // an uncaught exception. + try { + const symbols = getInstruments().filter((i) => i.enabled).map((i) => i.symbol); + console.log(`[contract-resolver] Resolving ${symbols.length} instruments...`); + resolveContracts(symbols, accessToken) + .then((results) => { + const rolled = Object.entries(results).filter(([, v]) => v?.alternative); + console.log(`[contract-resolver] Done — ${rolled.length} contract(s) rolled via price match`); + }) + .catch((err) => console.error(`[contract-resolver] Resolve failed: ${errorMessage(err)}`)); + } catch (err) { + console.error(`[contract-resolver] Resolve failed: ${errorMessage(err)}`); + } } function scheduleDailyResolve(): void { @@ -215,7 +281,179 @@ function scheduleDailyResolve(): void { const now = new Date(); if (now.getHours() === 0 && now.getMinutes() === 0) { console.log('[contract-resolver] Midnight resolve triggered'); + // Its own half hour of retries, not what's left of a chain it cuts off. + g.__contractResolveRetries = 0; triggerContractResolve(); } }, 60_000); } + +// ── Client supervision ─────────────────────────────────────────────────────── + +/** + * Rebuilds a client that has been unable to trade for a long time. + * + * The clients reconnect themselves, but only along the paths their authors + * foresaw. A re-login that fails in an unexpected way, an await that never + * settles, a socket that dies without a close event — any of those used to + * leave a client wedged for the life of the process, and because the + * scheduler waits on every firm, one wedged firm stopped them all. After a + * weekend of broker maintenance that meant a pm2 restart. A fresh client is + * exactly what the restart gave it, so the registry hands one out itself: + * same firm, same credentials, new instance. + * + * "Unready" is isClientReady() saying no — the same rule the scheduler and + * /api/health use: not synced, or no good sync in the last 5 minutes. + * + * Deliberately slow — 20 minutes unready before a rebuild, and at most one + * rebuild per firm per 20 minutes. The clients' own retries get the first go, + * and a broker that is really down, or rate-limiting logins, must not be + * hammered with new sessions. Some clients are never replaced while unready + * (see rebuildHold): one whose own login retry is pending, and a TopstepX + * one holding a position. + */ +const SUPERVISOR_INTERVAL_MS = 60_000; +const REBUILD_AFTER_UNREADY_MS = 20 * 60_000; +const REBUILD_COOLDOWN_MS = 20 * 60_000; + +/** The message TopstepXClient records when a firm has an API key but no + * username. That client never even tries to log in, so a rebuild changes + * nothing — adding the username changes the credentials, and + * reconcileClients rebuilds it then. */ +const TOPSTEPX_USERNAME_MISSING = /username missing/i; + +interface SupervisionEntry { + /** The instance being judged. A different one (rebuilt here, or replaced + * by reconcileClients after a credentials change) starts a fresh clock. */ + client: PlatformClient; + /** When it was first seen unready (0 = ready at the last check). */ + unhealthySince: number; + lastRebuildAt: number; + /** Last time a due rebuild was held back and said so (log rate limit). */ + holdLoggedAt: number; +} + +function supervisionMap(): Map { + if (!g.__clientSupervision) g.__clientSupervision = new Map(); + return g.__clientSupervision; +} + +function startClientSupervisor(): void { + // HMR-safe, and safe when resetClients() re-runs the bootstrap. + if (g.__clientSupervisorTimer) clearInterval(g.__clientSupervisorTimer); + g.__clientSupervisorTimer = setInterval(superviseClients, SUPERVISOR_INTERVAL_MS); +} + +/** Why a client isn't ready, short enough for one log line. Never the token. */ +function describeUnready(client: PlatformClient, now: number): string { + const parts: string[] = []; + try { + const health = client.health(); + parts.push(`transport ${health.transport}`); + if (health.lastLoginError) parts.push(`login error: ${health.lastLoginError.slice(0, 160)}`); + } catch { + parts.push('health unavailable'); + } + parts.push(client.lastSyncAt > 0 + ? `last good sync ${Math.round((now - client.lastSyncAt) / 60_000)}m ago` + : 'never synced'); + return parts.join(', '); +} + +/** + * Why replacing this client now would do more harm than good, or null. + * + * - Its login keeps failing and its own retry is already scheduled. Nothing + * in the instance is broken — the broker is down, refusing the + * credentials, or rate-limiting — and a fresh instance would fail the same + * way while starting its backoff over at seconds: a burst of logins after + * every rebuild instead of one every few minutes, which is how a login + * rate-limit turns into a captcha lockout. + * - TopstepX with a position open. The gateway has no OCO, so the client + * itself cancels the surviving exit leg when the other fills; a fresh + * instance knows nothing of those pairs, and the survivor would be left + * working at the broker after the position closes — an order that can open + * a position nobody asked for. It waits until the account is flat. + */ +function rebuildHold(platform: string, client: PlatformClient, now: number): string | null { + try { + const health = client.health(); + if (health.consecutiveLoginFailures > 0 && health.nextReconnectAt > now) { + return `its login retry is due in ${Math.max(1, Math.round((health.nextReconnectAt - now) / 60_000))}m` + + ` (${health.consecutiveLoginFailures} failure(s) in a row)`; + } + } catch { /* no health: judge it on the rest */ } + if (platform === 'topstepx' && Object.keys(client.positions ?? {}).length > 0) { + return 'it holds an open position (TopstepX exit brackets are tracked only inside the client)'; + } + return null; +} + +function superviseClients(): void { + // Runs inside setInterval: nothing may escape, or the process logs an + // uncaught exception and the supervisor keeps firing into the same fault. + try { + const map = ensureMap(); + const book = supervisionMap(); + const now = Date.now(); + + // A firm that left the registry takes its bookkeeping with it. + for (const id of [...book.keys()]) { + if (!map.has(id)) book.delete(id); + } + + let firms: ReturnType | null = null; + for (const [id, client] of [...map]) { + try { + let entry = book.get(id); + if (!entry) { + entry = { client, unhealthySince: 0, lastRebuildAt: 0, holdLoggedAt: 0 }; + book.set(id, entry); + } else if (entry.client !== client) { + entry.client = client; + entry.unhealthySince = 0; + } + + if (isClientReady(client, now)) { + entry.unhealthySince = 0; + continue; + } + if (!entry.unhealthySince) { + entry.unhealthySince = now; + continue; + } + if (now - entry.unhealthySince < REBUILD_AFTER_UNREADY_MS) continue; + if (now - entry.lastRebuildAt < REBUILD_COOLDOWN_MS) continue; + + // Only read when a rebuild is actually due — it hits the DB. + firms ??= getTradableFirms(); + const firm = firms.find((f) => f.id === id); + if (!firm) continue; // no longer tradable: reconcileClients drops it + if (firm.platform === 'topstepx' + && TOPSTEPX_USERNAME_MISSING.test(client.lastFetchErrors?.['login'] ?? '')) continue; + + const minutes = Math.round((now - entry.unhealthySince) / 60_000); + const hold = rebuildHold(firm.platform, client, now); + if (hold) { + if (now - entry.holdLoggedAt >= REBUILD_COOLDOWN_MS) { + entry.holdLoggedAt = now; + console.log(`[clients] ${firm.name} unhealthy for ${minutes}m — not rebuilding while ${hold}`); + } + continue; + } + console.log( + `[clients] ${firm.name} unhealthy for ${minutes}m (${describeUnready(client, now)})` + + ' — rebuilding client', + ); + entry.lastRebuildAt = now; + removeClient(id); + entry.client = initClient(firm.id, firm.name, firm.platform, firm); + entry.unhealthySince = now; + } catch (err) { + console.error(`[clients] supervisor check failed for firm ${id}: ${errorMessage(err)}`); + } + } + } catch (err) { + console.error(`[clients] supervisor tick failed: ${errorMessage(err)}`); + } +} diff --git a/lib/contract-resolver.ts b/lib/contract-resolver.ts index 3071f34..df6986d 100644 --- a/lib/contract-resolver.ts +++ b/lib/contract-resolver.ts @@ -13,6 +13,7 @@ */ import axios from 'axios'; +import { errorMessage } from './platform-client'; interface ContractInfo { id: number; @@ -31,7 +32,13 @@ const PROBE_WINDOW = 6; // ── Cache ──────────────────────────────────────────────────────────────────── -const cache = new Map(); +// On globalThis, not module scope: instrumentation.ts boots the process from its +// own bundle, so the boot and midnight resolves would otherwise fill a cache the +// route handlers (settings page, manual trades) never see. +const _g = globalThis as typeof globalThis & { + __contractCache?: Map; +}; +const cache = (_g.__contractCache ??= new Map()); const CACHE_TTL_MS = 30 * 60 * 1000; // 30 minutes export function getCachedContract(symbol: string): ResolvedContract | null { @@ -205,7 +212,8 @@ export async function resolveContracts( console.log(`[contract-resolver] ${sym}: ${match.name} resolved`); } } catch (err) { - console.error(`[contract-resolver] ${sym}: Tradovate lookup failed`, err); + // Never the whole error: an AxiosError carries the Bearer token in its config. + console.error(`[contract-resolver] ${sym}: Tradovate lookup failed — ${errorMessage(err)}`); results[sym] = null; } })); diff --git a/lib/master-link.ts b/lib/master-link.ts index b8f92c7..7f379f1 100644 --- a/lib/master-link.ts +++ b/lib/master-link.ts @@ -99,14 +99,18 @@ async function execute(command: Command): Promise { if (command.symbol !== 'Auto' && !POINT_VALUES[command.symbol]) { throw new Error(`Unknown symbol: ${command.symbol}`); } - // Same order as POST /api/trade: fire once now, then leave the - // scheduler running to pick up accounts that were busy. - const results = await runTrade(command.action, command.symbol); + // Same order as POST /api/trade: (re)start the scheduler first, + // then fire once now — the scheduler picks up accounts that were + // busy. Starting it first means a batch that throws or runs long + // can never leave it unstarted. It can't double-fire against this + // batch: the tick waits for flat positions, skips while this batch + // holds the trade mutex, and never picks an account in flight. startScheduler(command.action, command.symbol, command.stopAfterAll ?? false); + const results = await runTrade(command.action, command.symbol); const filled = results - .flatMap((r: any) => r.results ?? []) - .filter((r: any) => r.status === 'filled').length; + .flatMap((r) => r.results) + .filter((r) => r.status === 'filled').length; return { filled, scheduler: getSchedulerStatus() }; } diff --git a/lib/outbound-proxy.ts b/lib/outbound-proxy.ts index bbc4cb7..0e1dd2f 100644 --- a/lib/outbound-proxy.ts +++ b/lib/outbound-proxy.ts @@ -12,6 +12,19 @@ * Configured by the `proxy_url` setting: http://, https://, socks5:// or * socks5h:// with optional user:pass. Read once per process — changing it * requires a restart, which the settings UI says out loud. Blank = direct. + * + * Also home of the broker HTTP timeout (applyBrokerHttpDefaults): axios waits + * forever by default, and one broker request that never answers — a + * maintenance window that accepts the connection and goes silent — used to + * wedge a re-login or sync await for the life of the process. Same scoping + * argument as the proxy: every axios call here is broker/quote traffic, so the + * process default is the scoped choice. It covers axios only; websockets and + * the SignalR hub keep their own liveness handling in the clients. + * + * Both defaults are applied when this module loads. Every broker client + * imports it, and that matters: each server bundle (instrumentation.ts, the + * route handlers) can carry its own copy of axios, and a copy that never had + * the defaults applied would dial the brokers direct, with no timeout. */ import axios from 'axios'; @@ -30,7 +43,6 @@ interface ProxyState { dispatcher: Dispatcher | null; httpAgent: Agent | null; httpsAgent: Agent | null; - applied: boolean; } // HMR-safe, and the "read once per process" contract lives here. @@ -39,7 +51,7 @@ const g = globalThis as typeof globalThis & { __outboundProxy?: ProxyState }; function state(): ProxyState { if (g.__outboundProxy) return g.__outboundProxy; - const s: ProxyState = { url: null, dispatcher: null, httpAgent: null, httpsAgent: null, applied: false }; + const s: ProxyState = { url: null, dispatcher: null, httpAgent: null, httpsAgent: null }; g.__outboundProxy = s; const raw = getSetting('proxy_url')?.trim(); @@ -49,7 +61,8 @@ function state(): ProxyState { try { url = new URL(raw); } catch { - console.error(`[proxy] proxy_url is not a URL — trading DIRECT: ${raw}`); + // Not echoed: the value carries the proxy's user:password. + console.error('[proxy] proxy_url is not a URL — trading DIRECT'); return s; } if (!SCHEMES.includes(url.protocol as (typeof SCHEMES)[number])) { @@ -57,22 +70,42 @@ function state(): ProxyState { return s; } - const socks = url.protocol.startsWith('socks'); - if (socks) { - s.dispatcher = socksDispatcher({ - type: 5, - host: url.hostname, - port: Number(url.port || 1080), - userId: url.username ? decodeURIComponent(url.username) : undefined, - password: url.password ? decodeURIComponent(url.password) : undefined, - }); - const agent = new SocksProxyAgent(url); - s.httpAgent = agent; - s.httpsAgent = agent; - } else { - s.dispatcher = new ProxyAgent(url.href); - s.httpAgent = new HttpProxyAgent(url); - s.httpsAgent = new HttpsProxyAgent(url); + // A literal '%' in the user or password (one not starting a %XX escape) + // makes decodeURIComponent throw — here, and inside every proxy library + // below. Escape it so the password is used as typed. This now runs when + // the module loads, where a throw would fail every route that imports it. + const escapeStray = (part: string) => part.replace(/%(?![0-9A-Fa-f]{2})/g, '%25'); + url.username = escapeStray(url.username); + url.password = escapeStray(url.password); + + try { + const socks = url.protocol.startsWith('socks'); + if (socks) { + const dispatcher = socksDispatcher({ + type: 5, + host: url.hostname, + port: Number(url.port || 1080), + userId: url.username ? decodeURIComponent(url.username) : undefined, + password: url.password ? decodeURIComponent(url.password) : undefined, + }); + const agent = new SocksProxyAgent(url); + s.dispatcher = dispatcher; + s.httpAgent = agent; + s.httpsAgent = agent; + } else { + const dispatcher = new ProxyAgent(url.href); + const httpAgent = new HttpProxyAgent(url); + const httpsAgent = new HttpsProxyAgent(url); + s.dispatcher = dispatcher; + s.httpAgent = httpAgent; + s.httpsAgent = httpsAgent; + } + } catch (err) { + // All or nothing — nothing is kept until every piece is built: half a + // proxy (REST proxied, sockets direct) would be worse than the + // documented bad-setting behavior. + console.error(`[proxy] proxy_url could not be set up — trading DIRECT: ${err instanceof Error ? err.message : String(err)}`); + return s; } s.url = url; console.log(`[proxy] broker traffic will use ${url.protocol}//${url.host} (master link stays direct)`); @@ -87,16 +120,52 @@ export function getProxyUrl(): string | null { /** Puts the proxy under every axios call. All axios usage in this app is * broker/quote traffic (TopstepX, Tradovate REST, contract resolver), so the * process-wide default is the scoped choice, not a global one. Idempotent; - * called first in the getClients() bootstrap. */ + * runs when this module loads and again first in the getClients() bootstrap. + * + * "Already applied" is judged on the axios defaults themselves, not a + * process-wide flag: the proxy state is shared through globalThis, but each + * bundle's copy of axios has its own defaults, and a global flag set by the + * first copy would leave every other one dialling direct. */ export function applyBrokerProxy(): void { const s = state(); - if (s.applied || !s.url) return; - s.applied = true; + if (!s.url || axios.defaults.httpsAgent === s.httpsAgent) return; axios.defaults.proxy = false; // agents own the routing; ignore env HTTP(S)_PROXY handling axios.defaults.httpAgent = s.httpAgent!; axios.defaults.httpsAgent = s.httpsAgent!; } +/** Idle limit for one broker HTTP request. axios applies it as a socket + * inactivity timeout, so it bounds a server that has gone silent, not a + * big report that is still streaming. Nothing legitimate comes close: the + * Tradovate report endpoints answer with a p-ticket and are polled (each + * poll its own short request, already capped at 60s per call), logins and + * REST calls answer in seconds. A per-call `timeout` still overrides it. + * + * The clock starts once the socket has connected — a connect that never + * completes is left to the OS connect timeout. Calls that can't wait that + * long (the Tradovate login and renewal) add an AbortSignal of their own. */ +export const BROKER_HTTP_TIMEOUT_MS = 60_000; + +/** Gives axios a timeout unless one is already set (axios's own default, 0, + * means wait forever). Idempotent; runs when this module loads and at the + * top of the getClients() bootstrap. Timeouts surface as ECONNABORTED, which + * the Tradovate report fetch already retries as transient. */ +export function applyBrokerHttpDefaults(): void { + if (!axios.defaults.timeout) axios.defaults.timeout = BROKER_HTTP_TIMEOUT_MS; +} + +// At load, so no broker request from this bundle can go out before them. The +// proxy is skipped during `next build`: the build imports route modules to +// collect page data, and it dials nothing that would need one. Nothing may +// throw out of here: a module that fails to load takes every route that +// imports it (the whole trading side) down with it. +applyBrokerHttpDefaults(); +try { + if (process.env.NEXT_PHASE !== 'phase-production-build') applyBrokerProxy(); +} catch (err) { + console.error(`[proxy] could not apply proxy_url at load: ${err instanceof Error ? err.message : String(err)}`); +} + /** A websocket to a BROKER endpoint: the built-in (undici) WebSocket, dialled * through the proxy when one is set. The master link keeps using plain * `new WebSocket(...)` — do not use this for it. */ diff --git a/lib/platform-client.ts b/lib/platform-client.ts index 48bc40e..4e739e9 100644 --- a/lib/platform-client.ts +++ b/lib/platform-client.ts @@ -43,6 +43,8 @@ export interface PlatformClient { fetchDaysComplete: boolean; syncComplete: boolean; + /** Epoch ms of the last sync that came back good (0 = never). */ + lastSyncAt: number; lastFetchErrors: Record; lastFetchRaw: Record; recentEntityEvents: { entityType: string; eventType: string; entity: unknown; ts: number }[]; @@ -81,6 +83,76 @@ export interface PlatformClient { disconnect(): void; onFill(orderId: number, callback: (fill: Record) => void): void; + health(): ClientHealth; +} + +/** + * Liveness snapshot of one client — what /api/health shows, what the registry + * supervisor judges, and the numbers to read before restarting a broken box. + * Timestamps are epoch ms, 0 = never. Never carries tokens or credentials. + */ +export interface ClientHealth { + platform: 'tradovate' | 'topstepx'; + /** Holding a session token right now. */ + authenticated: boolean; + /** Tradovate: socket readyState name; TopstepX: hub state. 'none' = no transport. */ + transport: string; + syncComplete: boolean; + lastSyncAt: number; + lastLoginAt: number; + lastLoginError: string | null; + consecutiveLoginFailures: number; + /** When the next reconnect/login attempt fires (0 = none scheduled). */ + nextReconnectAt: number; + tokenExpiresAt: number; + /** Last frame from the broker's realtime channel (socket or hub). */ + lastInboundAt: number; + /** Requests still waiting on a broker answer (acks, fills). */ + pendingRequests: number; + /** When the in-flight sync started (0 = none in flight). */ + syncInFlightSince: number; + /** When the in-flight daily P&L report fetch started (0 = none in flight). */ + daysFetchInFlightSince: number; +} + +/** A sync older than this means positions and balances can't be traded on. */ +export const SYNC_STALE_MS = 5 * 60 * 1000; + +/** The one readiness rule shared by the scheduler, the supervisor and /api/health: + * synced, and synced recently. */ +export function isClientReady(client: PlatformClient, now = Date.now()): boolean { + return client.syncComplete && client.lastSyncAt > 0 && now - client.lastSyncAt < SYNC_STALE_MS; +} + +/** + * Thrown when a client KNOWS an order never reached the broker: no live + * connection, the broker refused it, or it failed before anything was sent. + * Anything ambiguous (a timeout waiting for an ack or a fill — it may still + * have filled) must stay a plain Error. The scheduler un-counts a leg only for + * this one, so a wrong "not placed" could let an account trade twice. + */ +export class OrderNotPlacedError extends Error { + readonly orderNotPlaced = true; + constructor(message: string) { + super(message); + this.name = 'OrderNotPlacedError'; + } +} + +/** Duck-typed so it survives module duplication across route bundles. */ +export function isOrderNotPlaced(err: unknown): boolean { + return !!err && typeof err === 'object' && (err as { orderNotPlaced?: unknown }).orderNotPlaced === true; +} + +/** An error's message without the rest of it. AxiosErrors carry the request + * config — Tradovate's login body (the password) and Bearer tokens — so they + * must never be logged whole. */ +export function errorMessage(err: unknown): string { + if (!err || typeof err !== 'object') return String(err); + const e = err as { message?: unknown; code?: unknown; response?: { status?: unknown } }; + const status = typeof e.response?.status === 'number' ? ` (HTTP ${e.response.status})` : ''; + const code = typeof e.code === 'string' && !String(e.message ?? '').includes(e.code) ? ` [${e.code}]` : ''; + return `${typeof e.message === 'string' ? e.message : String(err)}${status}${code}`; } /** lastFetchErrors note for a Tradovate account whose Cash History is diff --git a/lib/topstepx-class.ts b/lib/topstepx-class.ts index 72f3d12..f93868b 100644 --- a/lib/topstepx-class.ts +++ b/lib/topstepx-class.ts @@ -27,11 +27,88 @@ import { getCachedContract } from './contract-resolver'; import { signalrWebSocket } from './outbound-proxy'; import { saveDailyPnL, loadDailyPnL, loadAccountMeta, saveAccountMeta, loadFundTransactions, saveFundTransactions, saveEndedCycleStart } from './db'; import { getAccountConfig } from './account-type-resolver'; -import type { PlatformClient, PlatformAccount, PlatformContract } from './platform-client'; +import type { PlatformClient, PlatformAccount, PlatformContract, ClientHealth } from './platform-client'; +import { OrderNotPlacedError, isOrderNotPlaced, errorMessage } from './platform-client'; const API_BASE = 'https://api.topstepx.com'; const USER_HUB = 'https://rtc.topstepx.com/hubs/user'; +/** + * Self-healing timings. The weekend failure was a chain of awaits with no + * deadline (hub.start, hub.invoke, a re-login) plus a re-login that tore down + * the sync loop first — one wedged await left the firm "waiting for sync" + * until PM2 restarted the process. Every hub await now has a deadline, sync + * and hub setup detect their own wedges, and a watchdog keeps the client from + * ever sitting with no session and nothing scheduled to get one. + */ +const HUB_START_TIMEOUT_MS = 30_000; +const HUB_INVOKE_TIMEOUT_MS = 15_000; +/** Automatic reconnect gives up by itself within ~45s (0/2/10/30s retries); + * a hub still Connecting/Reconnecting after this is stuck, not retrying. */ +const HUB_STALL_MS = 2 * 60_000; +/** A hub setup or REST sync still running after this is wedged — start over. */ +const HUB_SETUP_WEDGED_MS = 3 * 60_000; +const SYNC_WEDGED_MS = 3 * 60_000; +/** Failed logins back off 60s, 120s, … up to 10 min, ±20% so a fleet of + * boxes coming back from the same outage doesn't log in in lockstep. */ +const LOGIN_RETRY_BASE_MS = 60_000; +const LOGIN_RETRY_MAX_MS = 10 * 60_000; +const WATCHDOG_INTERVAL_MS = 60_000; +/** Logged in longer than this with no good sync for as long → the session is + * presumed dead in a way validate didn't catch; log in again. */ +const SESSION_STALE_MS = 10 * 60_000; +/** Auth calls get their own short deadlines: a login that never answers + * would hold loginInFlight — and with it every retry — forever. */ +const LOGIN_TIMEOUT_MS = 30_000; +const VALIDATE_TIMEOUT_MS = 15_000; +const TOKEN_LIFETIME_MS = 24 * 60 * 60 * 1000; +/** Hard cap on every other REST call. axios's own timeout is a socket-idle + * timer that only starts once the socket has connected, so a proxy CONNECT + * that never answers hangs the call — and any single-flight flag it holds, + * like the daily P&L fetch's — for good. Aborting (rather than racing) also + * destroys a request that hasn't been written yet, so an order can't go out + * minutes after its caller gave up on it. Longer than the idle timeout, so + * it only bites where that one can't. */ +const REST_DEADLINE_MS = 90_000; + +/** Rejects if `promise` hasn't settled within `ms`. The underlying operation + * keeps running (callers discard whatever it belonged to); Promise.race + * subscribes to it, so a late rejection never surfaces as unhandled. */ +function withTimeout(promise: Promise, ms: number, label: string): Promise { + let timer: ReturnType | undefined; + const deadline = new Promise((_, reject) => { + timer = setTimeout(() => reject(new Error(`${label} timed out after ${Math.round(ms / 1000)}s`)), ms); + }); + return Promise.race([promise, deadline]).finally(() => clearTimeout(timer)); +} + +/** + * The gateway answered and said no (`success: false`) — as opposed to a + * transport failure, where the request may or may not have been acted on. + * placeOrder turns only this (and pre-handler HTTP refusals) into + * OrderNotPlacedError. + */ +class GatewayRefusedError extends Error { + readonly errorCode: number | undefined; + /** Set when a refusal still names an order — that order exists. */ + readonly orderId: number | undefined; + constructor(message: string, errorCode: number | undefined, orderId: number | undefined) { + super(message); + this.name = 'GatewayRefusedError'; + this.errorCode = errorCode; + this.orderId = orderId; + } +} + +/** PlaceOrderErrorCode 6 (OrderPending) and 7 (UnknownError) don't say the + * order wasn't created, so a refusal carrying them stays ambiguous. */ +const AMBIGUOUS_PLACE_ERRORS = new Set([6, 7]); + +/** HTTP answers given before any order handler runs (by the gateway's + * middleware or the Cloudflare edge in front of it): malformed request, + * missing/expired token, forbidden, rate-limited. Nothing was placed. */ +const REFUSED_BEFORE_HANDLER = new Set([400, 401, 403, 429]); + interface TsxContract { id: string; name: string; @@ -87,6 +164,8 @@ export class TopstepXClient implements PlatformClient { public fetchDaysComplete = false; public syncComplete = false; + /** Epoch ms of the last sync that came back good (0 = never). */ + public lastSyncAt = 0; public lastFetchErrors: Record = {}; public lastFetchRaw: Record = {}; public recentEntityEvents: { entityType: string; eventType: string; entity: unknown; ts: number }[] = []; @@ -96,19 +175,51 @@ export class TopstepXClient implements PlatformClient { private callbackOnSyncRequest: () => Promise; private hub: signalR.HubConnection | null = null; private subscribedAccountIds = new Set(); + /** SubscribeAccounts done on the current hub connection. */ + private accountsFeedSubscribed = false; + + /** ensureHub is single-flight; the generation lets a flight declared + * wedged be superseded without its late finally clearing the new flag. */ + private hubSetupInFlight = false; + private hubSetupSince = 0; + private hubSetupGeneration = 0; + /** ensureHub was asked for while a flight ran (e.g. the first sync's + * accounts landed mid-start) — run once more if the hub came up. */ + private hubRecheckWanted = false; + /** When the hub was first seen Connecting/Reconnecting/Disconnecting (0 = it wasn't). */ + private hubStalledSince = 0; + /** Last frame from the hub (event, invoke answer, handshake). */ + private lastInboundAt = 0; private syncInterval: ReturnType | null = null; private tokenRenewalInterval: ReturnType | null = null; private ocoPollInterval: ReturnType | null = null; private loginRetryTimer: ReturnType | null = null; + private watchdogInterval: ReturnType | null = null; private intentionalDisconnect = false; + + /** requestSync is single-flight too, with the same generation trick. */ private syncInFlight = false; + private syncInFlightSince = 0; + private syncGeneration = 0; + + private loginInFlight = false; + private lastLoginAt = 0; + private lastLoginError: string | null = null; + private consecutiveLoginFailures = 0; + private nextReconnectAt = 0; + /** When the current token was issued or last renewed (24h lifetime). */ + private tokenRefreshedAt = 0; /** Same hourly-with-retry throttle the Tradovate client uses for reports. */ private daysFetchInFlight = false; + private daysFetchInFlightSince = 0; private lastDaysFetchStartedAt = 0; private lastDaysFetchSucceededAt = 0; private nextDaysFetchRetryAt = 0; + /** The first full daily P&L pass has landed (cache fallbacks included). + * No sync counts as good before it — see requestSync. */ + private historyLoaded = false; /** Trades seen for the current CT trade date, per account — realizedPnL is * recomputed from these on every REST sync and every live trade event. */ @@ -134,7 +245,8 @@ export class TopstepXClient implements PlatformClient { this.name = name; this.apiKey = apiKey; this.callbackOnSyncRequest = callbackOnSyncRequest; - this.connectAndAuth(); + this.watchdogInterval = setInterval(() => this.watchdog(), WATCHDOG_INTERVAL_MS); + void this.connectAndAuth(); } // ── auth ───────────────────────────────────────────────────────────────── @@ -144,16 +256,48 @@ export class TopstepXClient implements PlatformClient { } private async post(path: string, body: unknown): Promise { - const res = await axios.post(`${API_BASE}${path}`, body, { headers: this.authHeaders() }); - const data = res.data as T & { success?: boolean; errorMessage?: string | null; errorCode?: number }; - if (data && data.success === false) { - throw new Error(`${path} failed: ${data.errorMessage ?? `errorCode=${data.errorCode}`}`); + const res = await axios.post(`${API_BASE}${path}`, body, { + headers: this.authHeaders(), + signal: AbortSignal.timeout(REST_DEADLINE_MS), + }).catch((err: unknown) => { + // The deadline abort surfaces as a bare "canceled" — say what it + // was. Still a plain Error: for an order it stays ambiguous. + if (axios.isCancel(err)) throw new Error(`${path} timed out after ${REST_DEADLINE_MS / 1000}s`); + throw err; + }); + const data = res.data as T & { success?: boolean; errorMessage?: string | null; errorCode?: number; orderId?: number | null }; + // The gateway answers every call with a JSON object. Anything else — + // an HTML maintenance or proxy page served with a 200, an empty body — + // is a failed call, never an empty result: through the callers' + // `?? []`, a positions answer would read as "flat", a trades answer + // as "nothing traded today", and the sync would certify both. For an + // order it stays a plain Error: it may have gone in. + if (!data || typeof data !== 'object') { + throw new Error(`${path} returned a non-JSON answer (HTTP ${res.status})`); + } + if (data.success === false) { + throw new GatewayRefusedError( + `${path} failed: ${data.errorMessage ?? `errorCode=${data.errorCode}`}`, + typeof data.errorCode === 'number' ? data.errorCode : undefined, + typeof data.orderId === 'number' && data.orderId > 0 ? data.orderId : undefined, + ); } return data; } - private connectAndAuth(): void { - if (this.intentionalDisconnect) return; + /** + * Log in, single-flight. Safe to call from anywhere (constructor, retry + * timer, token renewal, watchdog) — a second caller while one runs is a + * no-op, and it never rejects. + * + * First login starts the hub, the sync loop and its 60s interval. A + * RE-login only swaps the token and nudges a sync: it deliberately leaves + * the sync interval and syncComplete alone. Tearing those down before the + * new session was proven is what left a firm "waiting for sync" forever + * when the re-login chain wedged. + */ + private async connectAndAuth(): Promise { + if (this.intentionalDisconnect || this.loginInFlight) return; if (!this.name) { // loginKey requires the platform username the key was generated for. @@ -161,42 +305,89 @@ export class TopstepXClient implements PlatformClient { // can be traded until the username arrives — say so loudly, once. this.lastFetchErrors['login'] = 'TopstepX username missing — the API key alone cannot log in. Add the username on the dashboard.'; + this.lastLoginError = this.lastFetchErrors['login']; console.error(`[topstepx] ${this.lastFetchErrors['login']}`); return; } - this.login() - .then(async (token) => { - this.token = token; - console.log(`[topstepx] logged in as ${this.name}`); - delete this.lastFetchErrors['login']; + if (this.loginRetryTimer) { clearTimeout(this.loginRetryTimer); this.loginRetryTimer = null; } + this.nextReconnectAt = 0; + this.loginInFlight = true; + try { + // A hard deadline on top of axios's own: that one is a socket + // inactivity timer, which never starts while a proxy CONNECT + // hangs — and a login that never settles holds loginInFlight. + const token = await withTimeout(this.login(), LOGIN_TIMEOUT_MS, 'loginKey'); + if (this.intentionalDisconnect) return; - // Session tokens live 24h; re-validate every hour to stay fresh. - if (this.tokenRenewalInterval) clearInterval(this.tokenRenewalInterval); - this.tokenRenewalInterval = setInterval(() => void this.renewToken(), 60 * 60 * 1000); + this.token = token; + this.lastLoginAt = this.tokenRefreshedAt = Date.now(); + this.consecutiveLoginFailures = 0; + this.lastLoginError = null; + console.log(`[topstepx] logged in as ${this.name}`); + delete this.lastFetchErrors['login']; - if (this.syncInterval) { clearInterval(this.syncInterval); this.syncInterval = null; } - this.syncComplete = false; + // Session tokens live 24h; re-validate every hour to stay fresh. + if (this.tokenRenewalInterval) clearInterval(this.tokenRenewalInterval); + this.tokenRenewalInterval = setInterval(() => void this.renewToken(), 60 * 60 * 1000); - // Hub failure must never read as a login failure: REST polling - // carries every feature (slower), and subscribeAll retries the - // hub on each sync tick. - try { - await this.startHub(); - } catch (err) { - console.error('[topstepx] hub setup failed — running on REST polling only', err instanceof Error ? err.message : err); - } - - await this.requestSync(); + if (!this.syncInterval) { + // First login. The hub is never a hard dependency: REST polling + // carries every feature (slower), and each sync tick calls + // ensureHub again, so a hub that won't come up now gets retried. + void this.ensureHub(); this.syncInterval = setInterval(() => void this.requestSync(), 60_000); - }) - .catch((err) => { - const msg = err instanceof Error ? err.message : String(err); - this.lastFetchErrors['login'] = msg; - console.error(`[topstepx] login failed for ${this.name}: ${msg} — retrying in 60s`); - if (this.loginRetryTimer) clearTimeout(this.loginRetryTimer); - this.loginRetryTimer = setTimeout(() => this.connectAndAuth(), 60_000); - }); + void this.requestSync(); + } else { + // Re-login: the loop keeps running; just use the new session now. + void this.requestSync(); + void this.ensureHub(); + } + } catch (err) { + if (this.intentionalDisconnect) return; + this.consecutiveLoginFailures++; + const msg = errorMessage(err); + this.lastLoginError = msg; + this.lastFetchErrors['login'] = msg; + const backoff = Math.min( + LOGIN_RETRY_BASE_MS * 2 ** Math.min(this.consecutiveLoginFailures - 1, 10), + LOGIN_RETRY_MAX_MS, + ); + const delay = Math.round(backoff * (0.8 + Math.random() * 0.4)); + this.nextReconnectAt = Date.now() + delay; + console.error(`[topstepx] login failed for ${this.name}: ${msg} — attempt ${this.consecutiveLoginFailures}, retrying in ${Math.round(delay / 1000)}s`); + this.loginRetryTimer = setTimeout(() => { + this.loginRetryTimer = null; + this.nextReconnectAt = 0; + void this.connectAndAuth(); + }, delay); + } finally { + this.loginInFlight = false; + } + } + + /** + * Never-dead invariant, checked every minute. Every failure path above is + * supposed to leave a retry scheduled; this catches the ones that don't + * (and sessions that died without anything failing loudly): + * - no token, no login running, no retry pending → log in now; + * - logged in >10 min ago with no good sync for >10 min → the token may + * have died in a way validate didn't catch → log in again. + */ + private watchdog(): void { + if (!this.name || this.intentionalDisconnect) return; + if (this.loginInFlight || this.loginRetryTimer) return; + const now = Date.now(); + if (!this.token) { + console.warn(`[topstepx] watchdog: ${this.name} has no session and no login pending — logging in`); + void this.connectAndAuth(); + return; + } + if (now - this.lastLoginAt > SESSION_STALE_MS && now - this.lastSyncAt > SESSION_STALE_MS) { + const since = this.lastSyncAt > 0 ? `in ${Math.round((now - this.lastSyncAt) / 60_000)} min` : 'since login'; + console.warn(`[topstepx] watchdog: no good sync for ${this.name} ${since} — re-logging in`); + void this.connectAndAuth(); + } } /** LoginErrorCode names from the gateway swagger — the API itself sends @@ -218,6 +409,12 @@ export class TopstepXClient implements PlatformClient { const res = await axios.post(`${API_BASE}/api/Auth/loginKey`, { userName: this.name, apiKey: this.apiKey, + }, { + timeout: LOGIN_TIMEOUT_MS, + // Just after connectAndAuth's own deadline: that one gives the + // clean error; this one kills the request, so a stalled CONNECT + // can't still log in minutes later. + signal: AbortSignal.timeout(LOGIN_TIMEOUT_MS + 5_000), }); const data = res.data as { token?: string; success: boolean; errorCode: number; errorMessage: string | null }; if (!data.success || !data.token) { @@ -228,36 +425,203 @@ export class TopstepXClient implements PlatformClient { } private async renewToken(): Promise { + if (this.intentionalDisconnect || this.loginInFlight) return; + if (!this.token) { void this.connectAndAuth(); return; } + const sent = this.token; try { - const res = await axios.post(`${API_BASE}/api/Auth/validate`, {}, { headers: this.authHeaders() }); + const res = await withTimeout(axios.post(`${API_BASE}/api/Auth/validate`, {}, { + headers: this.authHeaders(), + timeout: VALIDATE_TIMEOUT_MS, + signal: AbortSignal.timeout(VALIDATE_TIMEOUT_MS + 5_000), + }), VALIDATE_TIMEOUT_MS, 'token validate'); + if (this.intentionalDisconnect) return; const data = res.data as { success: boolean; newToken?: string }; if (data.success && data.newToken) { + // A login that landed meanwhile holds a newer session — keep it. + if (this.token !== sent) return; this.token = data.newToken; + this.tokenRefreshedAt = Date.now(); console.log('[topstepx] session token renewed'); } else if (!data.success) { console.warn('[topstepx] token validate failed — re-authenticating'); - this.connectAndAuth(); + void this.connectAndAuth(); } } catch (err) { - console.error('[topstepx] token renewal error — re-authenticating', err); - this.connectAndAuth(); + if (this.intentionalDisconnect) return; + // Never the error object: its request config carries the Bearer token. + console.error(`[topstepx] token renewal error — re-authenticating: ${errorMessage(err)}`); + void this.connectAndAuth(); } } // ── realtime hub ───────────────────────────────────────────────────────── - private async startHub(): Promise { - if (this.hub) { - try { await this.hub.stop(); } catch { /* ignore */ } - this.hub = null; + /** + * Bring the user hub to connected-and-subscribed, as far as one pass can. + * Called on login and every sync tick, and never awaited by anything that + * matters: REST polling carries every feature on its own (slower), so a + * wedged hub must never hold up a sync. + * + * - no hub → build a fresh one and start it; a failed start discards it + * (a HubConnection whose first start failed never auto-reconnects), so + * the next pass builds another; + * - Disconnected (automatic reconnect gave up) → start it again, discard + * on failure; + * - Connecting/Reconnecting/Disconnecting for > 2 min → stuck; discard it + * and build a fresh one; + * - Connected → subscribe whatever isn't subscribed yet. + * + * Single-flight; a flight running past HUB_SETUP_WEDGED_MS is superseded + * (every await in it has a deadline, so that is belt and braces). + */ + private async ensureHub(): Promise { + if (this.intentionalDisconnect || !this.token) return; + if (this.hubSetupInFlight) { + const runningMs = Date.now() - this.hubSetupSince; + if (runningMs < HUB_SETUP_WEDGED_MS) { + this.hubRecheckWanted = true; + return; + } + console.warn(`[topstepx] hub setup running for ${Math.round(runningMs / 1000)}s — looks wedged, starting over`); } - this.subscribedAccountIds.clear(); + const gen = ++this.hubSetupGeneration; + this.hubSetupInFlight = true; + this.hubSetupSince = Date.now(); + this.hubRecheckWanted = false; + let clean = false; + try { + clean = await this.ensureHubPass(gen); + } catch (err) { + console.error(`[topstepx] hub setup failed — running on REST polling only: ${errorMessage(err)}`); + } finally { + // A superseded flight must not clear its successor's flag. + if (gen === this.hubSetupGeneration) { + this.hubSetupInFlight = false; + this.hubSetupSince = 0; + // Asked for again mid-flight (typically: the first sync's + // accounts landed while the hub was starting). Only worth a + // rerun after a clean pass on a live hub — a failed start or + // subscribe waits for the next tick rather than retrying hot. + if (clean && this.hubRecheckWanted && !this.intentionalDisconnect + && this.hub?.state === signalR.HubConnectionState.Connected) { + this.hubRecheckWanted = false; + void this.ensureHub(); + } + } + } + } + /** One ensureHub pass. True when it ended connected with everything + * subscribed; false when it stopped short (the next call carries on). */ + private async ensureHubPass(gen: number): Promise { + const State = signalR.HubConnectionState; + let hub = this.hub; + + if (hub && (hub.state === State.Connecting || hub.state === State.Reconnecting || hub.state === State.Disconnecting)) { + const now = Date.now(); + if (!this.hubStalledSince) this.hubStalledSince = now; + // Automatic reconnect is still working on it — let it. + if (now - this.hubStalledSince < HUB_STALL_MS) return false; + console.warn(`[topstepx] hub stuck ${hub.state} for ${Math.round((now - this.hubStalledSince) / 1000)}s — discarding it`); + this.discardHub(); + hub = null; + } + + if (!hub) { + hub = this.buildHub(); + this.hub = hub; + this.subscribedAccountIds.clear(); + this.accountsFeedSubscribed = false; + if (!(await this.startHub(hub, gen))) return false; + console.log('[topstepx] user hub connected'); + } else if (hub.state === State.Disconnected) { + // Subscriptions died with the old connection. + this.subscribedAccountIds.clear(); + this.accountsFeedSubscribed = false; + if (!(await this.startHub(hub, gen))) return false; + console.log('[topstepx] user hub restarted'); + } + + if (hub.state !== State.Connected) return false; + this.hubStalledSince = 0; + return this.subscribeAll(hub, gen); + } + + /** Start `hub` against a deadline. False when it didn't come up (the hub + * is discarded) or was discarded / superseded while starting. */ + private async startHub(hub: signalR.HubConnection, gen: number): Promise { + try { + await withTimeout(hub.start(), HUB_START_TIMEOUT_MS, 'hub start'); + } catch (err) { + // REST polling still works without the hub; the next sync tick + // builds a fresh one. + console.error(`[topstepx] hub connect failed — falling back to REST polling: ${errorMessage(err)}`); + if (this.hub === hub) this.discardHub(); + return false; + } + if (this.hub !== hub) { + // Discarded (disconnect, or a stall check) while it was starting. + void hub.stop().catch(() => { /* discarded */ }); + return false; + } + if (gen !== this.hubSetupGeneration) return false; + this.hubStalledSince = 0; + this.lastInboundAt = Date.now(); + return true; + } + + /** Subscribe everything the current connection isn't subscribed to yet. + * A failed or timed-out invoke is left for the next pass. True when + * nothing was left over. */ + private async subscribeAll(hub: signalR.HubConnection, gen: number): Promise { + const stillCurrent = () => + this.hub === hub && gen === this.hubSetupGeneration && hub.state === signalR.HubConnectionState.Connected; + const invoke = async (method: string, ...args: unknown[]) => { + await withTimeout(hub.invoke(method, ...args), HUB_INVOKE_TIMEOUT_MS, method); + this.lastInboundAt = Date.now(); + }; + try { + if (!this.accountsFeedSubscribed) { + await invoke('SubscribeAccounts'); + if (!stillCurrent()) return false; + this.accountsFeedSubscribed = true; + } + for (const acc of this.accountList) { + if (this.subscribedAccountIds.has(acc.id)) continue; + await invoke('SubscribeOrders', acc.id); + await invoke('SubscribePositions', acc.id); + await invoke('SubscribeTrades', acc.id); + if (!stillCurrent()) return false; + this.subscribedAccountIds.add(acc.id); + } + return true; + } catch (err) { + console.error(`[topstepx] hub subscribe failed — retrying on the next sync: ${errorMessage(err)}`); + return false; + } + } + + /** Drop the current hub without waiting on it — stop() can hang as long + * as the start it interrupts, so it is fire-and-forget. The hub's + * handlers go quiet the moment this.hub stops pointing at it. */ + private discardHub(): void { + const hub = this.hub; + this.hub = null; + this.subscribedAccountIds.clear(); + this.accountsFeedSubscribed = false; + this.hubStalledSince = 0; + if (hub) void hub.stop().catch(() => { /* discarded */ }); + } + + /** A fresh HubConnection with every handler wired. Each handler first + * checks its hub is still the current one: a discarded connection that + * lingers (or reconnects on its own) must never write state. */ + private buildHub(): signalR.HubConnection { // signalr's Node path does an unconditional dynamic require('ws') / // require('eventsource') when constructing the connection — bundled // builds throw "dynamic usage of require is not supported" on it, so // the package is listed in next.config serverExternalPackages and any - // residual failure here is caught by the caller (REST polling covers + // residual failure here is caught by ensureHub (REST polling covers // everything the hub does, just slower). // With skipNegotiation the hub is a single websocket to a broker // endpoint, so it follows the operator's proxy like the REST calls do. @@ -282,6 +646,8 @@ export class TopstepXClient implements PlatformClient { }; hub.on('GatewayUserAccount', (raw: unknown) => { + if (this.hub !== hub) return; + this.lastInboundAt = Date.now(); const acc = unwrap(raw); if (typeof acc?.id !== 'number') return; this.bufferEvent('account', acc); @@ -299,6 +665,8 @@ export class TopstepXClient implements PlatformClient { }); hub.on('GatewayUserPosition', (raw: unknown) => { + if (this.hub !== hub) return; + this.lastInboundAt = Date.now(); const pos = unwrap(raw); if (typeof pos?.accountId !== 'number') return; this.bufferEvent('position', pos); @@ -306,6 +674,8 @@ export class TopstepXClient implements PlatformClient { }); hub.on('GatewayUserTrade', (raw: unknown) => { + if (this.hub !== hub) return; + this.lastInboundAt = Date.now(); const trade = unwrap(raw) as unknown as TsxTrade; if (typeof trade?.accountId !== 'number') return; this.bufferEvent('trade', trade); @@ -315,67 +685,43 @@ export class TopstepXClient implements PlatformClient { const waiters = this.fillWaiters.filter((w) => w.orderId === trade.orderId); this.fillWaiters = this.fillWaiters.filter((w) => w.orderId !== trade.orderId); for (const w of waiters) { - try { w.callback(trade); } catch (err) { console.error('[topstepx] fill callback error', err); } + try { w.callback(trade); } catch (err) { console.error(`[topstepx] fill callback error: ${errorMessage(err)}`); } } }); hub.on('GatewayUserOrder', (raw: unknown) => { + if (this.hub !== hub) return; + this.lastInboundAt = Date.now(); const order = unwrap(raw); if (typeof order?.id !== 'number') return; this.bufferEvent('order', order); this.handleOcoOrderEvent(order.id, order.status); }); + hub.onreconnecting(() => { + if (this.hub !== hub) return; + // The stall clock ensureHub judges "stuck Reconnecting" by. + if (!this.hubStalledSince) this.hubStalledSince = Date.now(); + }); + hub.onreconnected(() => { + if (this.hub !== hub) return; + this.lastInboundAt = Date.now(); + this.hubStalledSince = 0; console.log('[topstepx] hub reconnected — resubscribing'); this.subscribedAccountIds.clear(); - void this.subscribeAll(); + this.accountsFeedSubscribed = false; + void this.ensureHub(); void this.requestSync(); }); hub.onclose((err) => { - if (this.intentionalDisconnect) return; - console.warn('[topstepx] hub closed', err?.message ?? ''); + if (this.hub !== hub || this.intentionalDisconnect) return; + this.hubStalledSince = 0; + console.warn(`[topstepx] hub closed — the next sync restarts it${err ? `: ${errorMessage(err)}` : ''}`); }); - this.hub = hub; - try { - await hub.start(); - await this.subscribeAll(); - console.log('[topstepx] user hub connected'); - } catch (err) { - // REST polling still works without the hub; automatic reconnect - // only runs after a successful start, so retry via the next sync. - console.error('[topstepx] hub connect failed — falling back to REST polling', err); - } - } - - private async subscribeAll(): Promise { - if (!this.hub) return; - // A hub whose initial start failed never auto-reconnects — each sync - // tick gets another chance to bring it up. - if (this.hub.state === signalR.HubConnectionState.Disconnected) { - try { - await this.hub.start(); - this.subscribedAccountIds.clear(); - console.log('[topstepx] user hub connected'); - } catch { - return; // REST polling carries on; retry next sync - } - } - if (this.hub.state !== signalR.HubConnectionState.Connected) return; - try { - await this.hub.invoke('SubscribeAccounts'); - for (const acc of this.accountList) { - if (this.subscribedAccountIds.has(acc.id)) continue; - await this.hub.invoke('SubscribeOrders', acc.id); - await this.hub.invoke('SubscribePositions', acc.id); - await this.hub.invoke('SubscribeTrades', acc.id); - this.subscribedAccountIds.add(acc.id); - } - } catch (err) { - console.error('[topstepx] hub subscribe failed', err); - } + return hub; } private bufferEvent(entityType: string, entity: unknown): void { @@ -428,12 +774,30 @@ export class TopstepXClient implements PlatformClient { // ── sync ───────────────────────────────────────────────────────────────── + /** + * One REST sync of accounts, positions and today's trades. Single-flight, + * but a sync still running after SYNC_WEDGED_MS is presumed wedged and a + * fresh one starts: nothing may hold syncInFlight forever, because a firm + * that stops syncing stops trading. The superseded run checks its + * generation after every await and bows out without touching state, and + * its finally leaves the new run's flag alone. + */ private async requestSync(): Promise { - if (!this.token || this.syncInFlight) return; + if (!this.token || this.intentionalDisconnect) return; + const startedAt = Date.now(); + if (this.syncInFlight) { + const runningMs = startedAt - this.syncInFlightSince; + if (runningMs < SYNC_WEDGED_MS) return; + console.warn(`[topstepx] sync running for ${Math.round(runningMs / 1000)}s — looks wedged, starting a fresh one`); + } + const gen = ++this.syncGeneration; + const superseded = () => gen !== this.syncGeneration || this.intentionalDisconnect; this.syncInFlight = true; + this.syncInFlightSince = startedAt; try { const accRes = await this.post<{ accounts: { id: number; name: string; balance: number; canTrade: boolean; isVisible: boolean }[] }>( '/api/Account/search', { onlyActiveAccounts: false }); + if (superseded()) return; this.accountList = (accRes.accounts ?? []) .filter((a) => a.isVisible) @@ -444,13 +808,17 @@ export class TopstepXClient implements PlatformClient { const start = new Date(Date.now() - 2 * 24 * 60 * 60 * 1000).toISOString(); const end = new Date(Date.now() + 60 * 60 * 1000).toISOString(); const today = currentTradeDateCT(); + /** Accounts whose positions or trades didn't come back this pass. */ + const staleAccounts: string[] = []; for (const acc of this.accountList) { const balance = (accRes.accounts ?? []).find((a) => a.id === acc.id)?.balance ?? 0; + let accountFresh = true; try { const posRes = await this.post<{ positions: { accountId: number; contractId: string; type: number; size: number; averagePrice: number; creationTimestamp: string }[] }>( '/api/Position/searchOpen', { accountId: acc.id }); + if (superseded()) return; const open = (posRes.positions ?? []).filter((p) => p.size > 0); if (open.length > 0) { this.applyPosition(open[0]); @@ -458,12 +826,15 @@ export class TopstepXClient implements PlatformClient { delete this.positions[acc.id]; } } catch (err) { - console.error(`[topstepx] position sync failed for ${acc.name}`, err instanceof Error ? err.message : err); + accountFresh = false; + console.error(`[topstepx] position sync failed for ${acc.name}: ${errorMessage(err)}`); } + if (superseded()) return; try { const tradeRes = await this.post<{ trades: TsxTrade[] }>( '/api/Trade/search', { accountId: acc.id, startTimestamp: start, endTimestamp: end }); + if (superseded()) return; const map = new Map(); for (const t of tradeRes.trades ?? []) { if (t.voided) continue; @@ -472,38 +843,64 @@ export class TopstepXClient implements PlatformClient { } this.todayTrades[acc.id] = map; } catch (err) { - console.error(`[topstepx] trade sync failed for ${acc.name}`, err instanceof Error ? err.message : err); + accountFresh = false; + console.error(`[topstepx] trade sync failed for ${acc.name}: ${errorMessage(err)}`); } + if (superseded()) return; + if (!accountFresh) staleAccounts.push(acc.name); this.accountCashBalances[acc.id] = { amount: balance, realizedPnL: 0 }; this.recomputeRealized(acc.id); this.autoLiqThresholds[acc.id] = 0; // Withdrawal/deposit detection — the books must explain the balance. + // Never off a failed fetch: a fresh balance against stale trades + // would book the unseen P&L as a (permanent) fund event. try { - this.reconcileFunds(acc); + if (accountFresh) this.reconcileFunds(acc); } catch (err) { - console.error(`[topstepx] fund reconcile failed for ${acc.name}`, err instanceof Error ? err.message : err); + console.error(`[topstepx] fund reconcile failed for ${acc.name}: ${errorMessage(err)}`); } try { this.updateMllFloor(acc); } catch (err) { - console.error(`[topstepx] MLL reconstruction failed for ${acc.name}`, err instanceof Error ? err.message : err); + console.error(`[topstepx] MLL reconstruction failed for ${acc.name}: ${errorMessage(err)}`); } } console.log(`[topstepx] sync — ${this.accountList.length} account(s), ${Object.keys(this.positions).length} open position(s)`); - await this.subscribeAll(); + // Fire-and-forget: the hub subscribes new accounts on its own + // schedule and can never hold this sync up. + void this.ensureHub(); this.maybeFetchDaysTraded(); this.syncComplete = true; - await this.callbackOnSyncRequest(); + // lastSyncAt is what the scheduler trades on (isClientReady), so it + // certifies only a pass where every account's positions and trades + // came back: a stale-flat position or a stale realized P&L walks + // straight through the "already in trade" / "already traded today" + // gates. Nor before the first daily P&L pass has landed — until + // then the plan would run on no history at all. + if (staleAccounts.length > 0) { + console.warn(`[topstepx] sync incomplete — not refreshed: ${staleAccounts.join(', ')}; not counted as a good sync`); + } else if (this.historyLoaded) { + this.lastSyncAt = Date.now(); + } + // The resolver runs off this; its failure is not a sync failure. + try { + await this.callbackOnSyncRequest(); + } catch (err) { + console.error(`[topstepx] post-sync callback failed: ${errorMessage(err)}`); + } } catch (err) { - console.error('[topstepx] sync failed', err instanceof Error ? err.message : err); + console.error(`[topstepx] sync failed: ${errorMessage(err)}`); } finally { - this.syncInFlight = false; + if (gen === this.syncGeneration) { + this.syncInFlight = false; + this.syncInFlightSince = 0; + } } } @@ -679,7 +1076,7 @@ export class TopstepXClient implements PlatformClient { } private maybeFetchDaysTraded(): void { - if (this.daysFetchInFlight) return; + if (this.daysFetchInFlight || this.intentionalDisconnect) return; const now = Date.now(); const initialFetchDue = this.lastDaysFetchStartedAt === 0; @@ -693,11 +1090,18 @@ export class TopstepXClient implements PlatformClient { if (!initialFetchDue && !retryDue && !hourlyRefreshDue) return; this.daysFetchInFlight = true; + this.daysFetchInFlightSince = now; this.lastDaysFetchStartedAt = now; if (retryDue) this.nextDaysFetchRetryAt = 0; this.fetchDaysTraded() .then(({ failedAccounts }) => { + if (!this.historyLoaded && !this.intentionalDisconnect) { + // Readiness waited on this; sync again now rather than + // leaving the firm unready until the next 60s tick. + this.historyLoaded = true; + void this.requestSync(); + } if (failedAccounts > 0) { this.nextDaysFetchRetryAt = Date.now() + 5 * 60 * 1_000; console.warn(`[topstepx] ${failedAccounts} account(s) failed daily P&L fetch — retrying in 5 minutes`); @@ -707,10 +1111,11 @@ export class TopstepXClient implements PlatformClient { }) .catch((err) => { this.nextDaysFetchRetryAt = Date.now() + 5 * 60 * 1_000; - console.error('[topstepx] daily P&L fetch failed — retrying in 5 minutes', err); + console.error(`[topstepx] daily P&L fetch failed — retrying in 5 minutes: ${errorMessage(err)}`); }) .finally(() => { this.daysFetchInFlight = false; + this.daysFetchInFlightSince = 0; }); } @@ -738,6 +1143,7 @@ export class TopstepXClient implements PlatformClient { }; for (const account of this.accountList) { + if (this.intentionalDisconnect) break; // a discarded client fetches (and writes) nothing more if (onlyAccountIds && !onlyAccountIds.includes(account.id)) continue; // Fund transactions never come from the TopstepX API — only what the @@ -788,8 +1194,8 @@ export class TopstepXClient implements PlatformClient { delete this.lastFetchErrors[account.name]; this.updateMllFloor(account); } catch (err) { - const msg = err instanceof Error ? err.message : String(err); - console.error(`[topstepx] trade history failed for ${account.name}:`, msg); + const msg = errorMessage(err); + console.error(`[topstepx] trade history failed for ${account.name}: ${msg}`); failedAccounts++; if (cached.length > 0) { this.fullDailyPnL[account.id] = cached; @@ -841,7 +1247,7 @@ export class TopstepXClient implements PlatformClient { this.contractByName.set(match.name, match); return { id: match.id, name: match.name, tickSize: match.tickSize }; } catch (err) { - console.error(`[topstepx] contract search failed for ${productName}`, err instanceof Error ? err.message : err); + console.error(`[topstepx] contract search failed for ${productName}: ${errorMessage(err)}`); return null; } } @@ -872,6 +1278,29 @@ export class TopstepXClient implements PlatformClient { // ── orders ─────────────────────────────────────────────────────────────── + /** + * Everything an order needs before anything is sent: a session and the + * contract. Failing here means nothing reached the broker, which is the + * one case the scheduler may un-count a leg for — hence OrderNotPlacedError. + */ + private async contractForOrder(contractSymbol: string): Promise { + if (!this.token) throw new OrderNotPlacedError('TopstepX not logged in — order not sent'); + try { + return await this.resolveContractId(contractSymbol); + } catch (err) { + throw new OrderNotPlacedError(`TopstepX contract lookup failed for ${contractSymbol} — order not sent: ${errorMessage(err)}`); + } + } + + /** + * POST /api/Order/place. Only a failure that proves nothing was placed + * becomes OrderNotPlacedError: no session, a gateway `success: false` + * that names no order and no ambiguous code, or an HTTP refusal given + * before any handler ran. Everything else (timeouts, resets, 5xx) may + * have placed the order and stays a plain Error — rewrapped so the + * AxiosError, whose config carries the Bearer token, never travels on + * into someone's log. + */ private async placeOrder( accountId: number, contractId: string, @@ -881,18 +1310,36 @@ export class TopstepXClient implements PlatformClient { limitPrice?: number, stopPrice?: number, ): Promise { - const res = await this.post<{ orderId: number }>('/api/Order/place', { - accountId, - contractId, - type, - side: action === 'Buy' ? SIDE_BUY : SIDE_SELL, - size, - limitPrice: limitPrice ?? null, - stopPrice: stopPrice ?? null, - trailPrice: null, - customTag: null, - }); - return res.orderId; + if (!this.token) throw new OrderNotPlacedError('TopstepX not logged in — order not sent'); + try { + const res = await this.post<{ orderId: number }>('/api/Order/place', { + accountId, + contractId, + type, + side: action === 'Buy' ? SIDE_BUY : SIDE_SELL, + size, + limitPrice: limitPrice ?? null, + stopPrice: stopPrice ?? null, + trailPrice: null, + customTag: null, + }); + return res.orderId; + } catch (err) { + if (err instanceof GatewayRefusedError) { + const ambiguous = err.orderId !== undefined + || (err.errorCode !== undefined && AMBIGUOUS_PLACE_ERRORS.has(err.errorCode)); + if (!ambiguous) throw new OrderNotPlacedError(`TopstepX refused the order: ${err.message}`); + throw new Error(`TopstepX order outcome unknown: ${err.message}`); + } + const response = (err as { response?: { status?: unknown; data?: { orderId?: unknown } } } | null)?.response; + const status = response?.status; + // Same rule as a refusal: an answer that names an order means it exists. + const namesOrder = typeof response?.data?.orderId === 'number' && response.data.orderId > 0; + if (typeof status === 'number' && REFUSED_BEFORE_HANDLER.has(status) && !namesOrder) { + throw new OrderNotPlacedError(`TopstepX rejected the order request: ${errorMessage(err)}`); + } + throw new Error(`TopstepX order request failed (it may still have been placed): ${errorMessage(err)}`); + } } /** Place an order and wait for its fill (trade event, with a REST poll as @@ -905,7 +1352,7 @@ export class TopstepXClient implements PlatformClient { orderType: 'Market' | 'Limit', price?: number ): Promise> { - const contract = await this.resolveContractId(contractSymbol); + const contract = await this.contractForOrder(contractSymbol); const orderId = await this.placeOrder( accountId, contract.id, quantity, action, orderType === 'Market' ? TYPE_MARKET : TYPE_LIMIT, @@ -920,7 +1367,15 @@ export class TopstepXClient implements PlatformClient { settled = true; clearInterval(poll); this.fillWaiters = this.fillWaiters.filter((w) => w.orderId !== orderId); - reject(new Error(`Order ${orderId} placed but no fill within 30s (market may be closed)`)); + // The caller gives up on this order and places no exits, so a + // late fill would leave a position nothing manages — and one + // open position blocks every firm through the open-position + // gate. Cancel it, best effort. The rejection stays a plain + // Error: it may have filled after all. + console.warn(`[topstepx] order ${orderId} not filled within 30s — cancelling it`); + void this.cancelOrder(accountId, orderId).catch((err) => + console.error(`[topstepx] failed to cancel unfilled order ${orderId}: ${errorMessage(err)}`)); + reject(new Error(`Order ${orderId} placed but no fill within 30s (market may be closed) — cancel requested`)); }, 30_000); const settle = (fillPrice: number) => { @@ -966,7 +1421,7 @@ export class TopstepXClient implements PlatformClient { orderType: 'Market' | 'Limit', price?: number ): Promise<{ orderId?: number }> { - const contract = await this.resolveContractId(contractSymbol); + const contract = await this.contractForOrder(contractSymbol); const orderId = await this.placeOrder( accountId, contract.id, quantity, action, orderType === 'Market' ? TYPE_MARKET : TYPE_LIMIT, @@ -979,8 +1434,18 @@ export class TopstepXClient implements PlatformClient { * Emulated OCO exit: the gateway has no linked-order type, so place the * take-profit limit and the stop separately and cancel the survivor when * either fills (order events; a poll and the flat-position hook back it - * up). If the stop can't be placed the limit is cancelled and the error - * rethrown, so the caller's lone-limit fallback starts from a clean book. + * up). + * + * If the stop doesn't go in, what the caller may do next depends on what + * is left at the broker — it falls back to a lone limit only on + * OrderNotPlacedError, and places nothing more on a plain Error: + * - stop refused, limit cancel confirmed → nothing of the pair is working: + * OrderNotPlacedError, and the fallback starts from a clean book; + * - stop refused, limit cancel failed → the limit may still be working: + * plain Error (a fallback on top could double the exit); + * - stop outcome unknown → it may be working, so the limit is kept (the + * position keeps its take-profit, unpaired, as the fallback would have + * left it) and the error is plain. */ public async placeOcoExit( accountId: number, @@ -990,14 +1455,20 @@ export class TopstepXClient implements PlatformClient { limitPrice: number, stopPrice: number, ): Promise<{ orderId?: number }> { - const contract = await this.resolveContractId(contractSymbol); + const contract = await this.contractForOrder(contractSymbol); const limitId = await this.placeOrder(accountId, contract.id, quantity, exitAction, TYPE_LIMIT, limitPrice); let stopId: number; try { stopId = await this.placeOrder(accountId, contract.id, quantity, exitAction, TYPE_STOP, undefined, stopPrice); } catch (err) { - await this.cancelOrder(accountId, limitId).catch(() => { /* best effort */ }); - throw err; + if (!isOrderNotPlaced(err)) { + throw new Error(`OCO stop leg outcome unknown — limit ${limitId} left working, unpaired: ${errorMessage(err)}`); + } + const limitCancelled = await this.cancelOrder(accountId, limitId).then(() => true, () => false); + if (limitCancelled) { + throw new OrderNotPlacedError(`OCO stop leg refused, limit ${limitId} cancelled — no exit working: ${errorMessage(err)}`); + } + throw new Error(`OCO stop leg refused and limit ${limitId} could not be cancelled — it may still be working: ${errorMessage(err)}`); } this.ocoPairs.push({ accountId, limitId, stopId, done: false, placedAt: Date.now() }); @@ -1018,7 +1489,7 @@ export class TopstepXClient implements PlatformClient { const sibling = pair.limitId === orderId ? pair.stopId : pair.limitId; console.log(`[topstepx] OCO leg ${orderId} filled — cancelling sibling ${sibling}`); void this.cancelOrder(pair.accountId, sibling).catch((err) => - console.error(`[topstepx] failed to cancel OCO sibling ${sibling}`, err instanceof Error ? err.message : err)); + console.error(`[topstepx] failed to cancel OCO sibling ${sibling}: ${errorMessage(err)}`)); } } @@ -1072,15 +1543,40 @@ export class TopstepXClient implements PlatformClient { // ── lifecycle ──────────────────────────────────────────────────────────── + public health(): ClientHealth { + return { + platform: 'topstepx', + authenticated: !!this.token, + transport: this.hub ? String(this.hub.state) : 'none', + syncComplete: this.syncComplete, + lastSyncAt: this.lastSyncAt, + lastLoginAt: this.lastLoginAt, + lastLoginError: this.lastLoginError, + consecutiveLoginFailures: this.consecutiveLoginFailures, + nextReconnectAt: this.nextReconnectAt, + // Validate hands out fresh 24h tokens; this is the latest one's expiry. + tokenExpiresAt: this.token && this.tokenRefreshedAt ? this.tokenRefreshedAt + TOKEN_LIFETIME_MS : 0, + lastInboundAt: this.lastInboundAt, + pendingRequests: this.fillWaiters.length, + syncInFlightSince: this.syncInFlight ? this.syncInFlightSince : 0, + daysFetchInFlightSince: this.daysFetchInFlight ? this.daysFetchInFlightSince : 0, + }; + } + + /** Final: nothing this client scheduled fires afterwards, and every + * in-flight login / sync / hub setup checks intentionalDisconnect (or + * that its hub is still current) before touching state again. */ public disconnect(): void { this.intentionalDisconnect = true; if (this.syncInterval) { clearInterval(this.syncInterval); this.syncInterval = null; } if (this.tokenRenewalInterval) { clearInterval(this.tokenRenewalInterval); this.tokenRenewalInterval = null; } if (this.ocoPollInterval) { clearInterval(this.ocoPollInterval); this.ocoPollInterval = null; } if (this.loginRetryTimer) { clearTimeout(this.loginRetryTimer); this.loginRetryTimer = null; } + if (this.watchdogInterval) { clearInterval(this.watchdogInterval); this.watchdogInterval = null; } + this.nextReconnectAt = 0; this.daysFetchInFlight = false; + this.daysFetchInFlightSince = 0; this.nextDaysFetchRetryAt = 0; - void this.hub?.stop().catch(() => { /* ignore */ }); - this.hub = null; + this.discardHub(); } } diff --git a/lib/tradovate-class.ts b/lib/tradovate-class.ts index e2a7531..2de94f3 100644 --- a/lib/tradovate-class.ts +++ b/lib/tradovate-class.ts @@ -7,7 +7,7 @@ import { POINT_VALUES } from './trading-logic'; import { getCachedContract, resolveContracts } from './contract-resolver'; import { saveDailyPnL, loadDailyPnL, saveAccountMeta, loadAccountMeta, saveFundTransactions, loadFundTransactions, saveEndedCycleStart } from './db'; import { brokerWebSocket } from './outbound-proxy'; -import { FILLS_FALLBACK_NOTE } from './platform-client'; +import { FILLS_FALLBACK_NOTE, OrderNotPlacedError, SYNC_STALE_MS, errorMessage, type ClientHealth } from './platform-client'; /** * Entity types Tradovate sends over the user-sync socket. Keep this as the @@ -26,6 +26,111 @@ type TradovateEntityType = | 'position' | 'cashBalance'; +/** + * What auth/accesstokenrequest can answer. HTTP 200 is not success on its + * own: a rate-limited login comes back 200 with no token and a penalty ticket + * instead (p-ticket, plus p-time in seconds to wait before presenting it, and + * p-captcha when Tradovate wants a human), and a refused one with errorText. + */ +type LoginResponseBody = Partial & { + 'p-ticket'?: string; + 'p-time'?: number; + 'p-captcha'?: boolean; + errorText?: string; +}; + +type LoginOutcome = + | { ok: true; info: AuthLoginResponse } + | { ok: false; error: string; retryInMs?: number }; + +/** A listener for pushed entity events. One-shot: it is removed once it fires. + * `cancel` (sendOrder's fill wait) rejects the waiter when the client is + * discarded, so no caller is left hanging on a dead instance. */ +type IndirectCallback = { + entityType: TradovateEntityType; + eventType: 'Created' | 'Updated'; + // Since the entity is not always the same, we need a validator to check if the response is the one we are looking for + validator: (response: any) => boolean; + callback: (response: any) => void; + cancel?: (reason: string) => void; +}; + +/* + * Connection supervision. + * + * This client used to be `login().then(...)` with exactly one retry scheduled + * from ws.onclose. One failed re-login — weekend maintenance, a network blip, + * a rate-limit penalty answered as HTTP 200 without a token — left it dead for + * the life of the process, with syncComplete frozen, and every firm on the + * instance waited on it until someone ran a pm2 restart. A session could also + * die without onclose ever firing (a socket stuck CLOSING, sync answers that + * just stop), and an order sent on a closed socket waited forever. + * + * Now: every failure path ends in scheduleReconnect(); a 15s watchdog notices + * sockets that went quiet, never authorized, or got stuck, and re-arms a + * reconnect if nothing is pending; nothing waits on the broker without a + * deadline. + */ +/** Reconnect after a socket drop on a still-good session. */ +const SOCKET_DROP_RECONNECT_MS = 5_000; +/** Reconnect when the session itself is suspect — straight to a fresh login. */ +const FORCED_RELOGIN_DELAY_MS = 1_000; +/** Login failures back off 5s, 10s, 20s, … capped at 5 min. */ +const BACKOFF_BASE_MS = 5_000; +const BACKOFF_MAX_MS = 5 * 60_000; +/** A captcha demand means we've logged in too often — go quiet for a long while. */ +const CAPTCHA_RETRY_MS = 15 * 60_000; +const PENALTY_MAX_WAIT_MS = 5 * 60_000; +const MAX_PENALTY_ROUNDS = 2; +/** Deadline for login / token renewal / contract lookups over REST. */ +const HTTP_TIMEOUT_MS = 15_000; +/** Report calls answer with a p-ticket to poll when they're slow, so a single + * HTTP call taking this long is a hung connection, not a big report. */ +const REPORT_TIMEOUT_MS = 60_000; +const LIVENESS_CHECK_MS = 15_000; +const HANDSHAKE_TIMEOUT_MS = 30_000; +const AUTHORIZE_TIMEOUT_MS = 30_000; +/** Tradovate heartbeats every few seconds and we sync every 60s — this much + * silence means the session is gone even if the socket says OPEN. */ +const INBOUND_SILENCE_MS = 150_000; +const STUCK_CLOSING_MS = 15_000; +const TOKEN_RENEW_CHECK_MS = 2 * 60_000; +const TOKEN_RENEW_WINDOW_MS = 15 * 60_000; +/** A failed renewal this close to expiry logs in afresh instead of waiting. */ +const TOKEN_RELOGIN_WINDOW_MS = 2 * 60_000; +/** A reconnect reuses the current token only if it has at least this long left. */ +const TOKEN_REUSE_MIN_MS = 5 * 60_000; +/** An order ack later than this is treated as lost (ambiguous — it may have been placed). */ +const ACK_TIMEOUT_MS = 15_000; +const FILL_TIMEOUT_MS = 30_000; +const CONTRACT_FIND_TIMEOUT_MS = 15_000; +/** On-demand front-month resolve (several Yahoo lookups, then Tradovate). */ +const CONTRACT_RESOLVE_DEADLINE_MS = 30_000; +const READY_STATE_NAMES = ['CONNECTING', 'OPEN', 'CLOSING', 'CLOSED']; + +function reconnectBackoffMs(failures: number): number { + return Math.min(BACKOFF_BASE_MS * 2 ** Math.max(0, failures - 1), BACKOFF_MAX_MS); +} + +/** ±20%, so several clients on one IP that failed together don't retry in lockstep. */ +function jitter(ms: number): number { + return Math.round(ms * (0.8 + Math.random() * 0.4)); +} + +function fmtDelay(ms: number): string { + return ms >= 60_000 ? `${(ms / 60_000).toFixed(1)}m` : `${(ms / 1000).toFixed(1)}s`; +} + +/** Settles like `work`, or rejects once `ms` pass first. The work itself + * carries on — only the wait for it is cut short. */ +function withDeadline(work: Promise, ms: number, what: string): Promise { + let timer: ReturnType | undefined; + const deadline = new Promise((_, reject) => { + timer = setTimeout(() => reject(new Error(`${what} took longer than ${ms / 1000}s`)), ms); + }); + return Promise.race([work, deadline]).finally(() => clearTimeout(timer)); +} + export class TradovateClient { private name: string; private password: string; @@ -87,26 +192,64 @@ export class TradovateClient { /** True once the first requestSync has completed and positions/balances are populated. */ public syncComplete = false; + /** Epoch ms of the last sync that came back good (0 = never). */ + public lastSyncAt = 0; + + // ── Connection supervisor state (see the comment above the constants) ── + /** Single-flight guard: true while connectAndAuth() is logging in / opening. */ + private connecting = false; + private reconnectTimer: ReturnType | null = null; + /** When reconnectTimer (or a login penalty wait) fires; 0 = nothing scheduled. */ + private nextReconnectAt = 0; + private consecutiveLoginFailures = 0; + /** Sessions in a row that died before producing a good sync (never + * opened, never authorized, sync refused) — they back off like login + * failures instead of reconnecting (or re-logging in) every few seconds. */ + private consecutiveSocketFailures = 0; + /** Last successful login. */ + private lastLoginAt = 0; + private lastLoginError: string | null = null; + /** The current token is suspect (refused, never authorized a socket, sync + * answers stopped): the next connect logs in instead of reusing it. */ + private needsLogin = true; + /** Socket creation time; reset to the open time once it opens. */ + private socketOpenedAt = 0; + /** The current socket got as far as presenting the token. */ + private authorizeSent = false; + /** When the current socket's authorize was accepted (0 = not yet). */ + private authorizedAt = 0; + private lastInboundAt = 0; + /** First time the watchdog saw the current socket CLOSING/CLOSED. */ + private closingSeenAt = 0; + /** When the oldest still-unanswered syncrequest was sent (0 = none). */ + private syncRequestedAt = 0; + private livenessInterval: ReturnType | null = null; + private renewInFlight = false; + /** The current token's expirationTime was (nearly) past on arrival: this + * machine's clock is off, so the expiry is ignored (noteTokenExpiry()). */ + private tokenExpiryUntrusted = false; + /** A login rate-limit penalty wait in progress — disconnect() cuts it short. */ + private penaltyWait: { timer: ReturnType; resolve: () => void } | null = null; - private ws!: WebSocket; + private ws: WebSocket | null = null; private callbackOnSyncRequest: () => Promise; /** Incrementing ID for outgoing WebSocket messages — ensures concurrent orders don't clobber each other's callbacks. */ private nextMsgId = 100; private getMsgId(): number { return this.nextMsgId++; } - // Events that we sent out, and tradovate gives us a response for the id we sent out + // Events that we sent out, and tradovate gives us a response for the id we sent out. + // One-shot: removed when the answer arrives, and all dropped with the socket. private directEventCallbacks: { - [id: number]: (response: any) => void; + [id: number]: (response: any, status?: number) => void; } = {}; - private indirectEventCallbacks: { - entityType: TradovateEntityType; - eventType: 'Created' | 'Updated'; - // Since the entity is not always the same, we need a validator to check if the response is the one we are looking for - validator: (response: any) => boolean; - callback: (response: any) => void; - }[] = []; + /** Rejecters for awaited direct requests (acks, contract finds), so a socket + * going away fails them instead of leaving the caller waiting forever. */ + private pendingRejecters: { + [id: number]: (reason: string) => void; + } = {}; + private indirectEventCallbacks: IndirectCallback[] = []; constructor( @@ -117,156 +260,538 @@ export class TradovateClient { this.name = name; this.password = password; this.callbackOnSyncRequest = callbackOnSyncRequest; - this.connectAndAuth(); + + // One watchdog and one renewal check for the life of the client — + // they span reconnects, and only disconnect() stops them. + this.livenessInterval = setInterval(() => this.checkLiveness(), LIVENESS_CHECK_MS); + this.tokenRenewalInterval = setInterval(() => this.maybeRenewToken(), TOKEN_RENEW_CHECK_MS); + void this.connectAndAuth(); } - private connectAndAuth(): void { - this.login().then((res) => { - if (!res?.accessToken) { - console.log('Failed to login', res); - return; - } + /** Log prefix — several Tradovate firms can share one instance's log. */ + private get tag(): string { + return `[tradovate ${this.name}]`; + } - console.log('Logged in', res); - this.accessInfo = res; - - // Clear and restart token renewal interval - if (this.tokenRenewalInterval) clearInterval(this.tokenRenewalInterval); - this.tokenRenewalInterval = setInterval(() => { - if ( - new Date(this.accessInfo.expirationTime).getTime() < - new Date().getTime() + 15 * 60 * 1000 - ) { - this.renewAccessToken(); - } - }, 2 * 60 * 1000); - - // Clear stale WS intervals before opening a new connection - if (this.syncInterval) { clearInterval(this.syncInterval); this.syncInterval = null; } - if (this.heartbeatInterval) { clearInterval(this.heartbeatInterval); this.heartbeatInterval = null; } - - // Reset sync state so scheduler waits for the new sync to complete - this.syncComplete = false; - - const randomnumber = Math.random().toString(36).substring(2, 15); - // Broker endpoint — dialled through the operator's proxy when one - // is configured (lib/outbound-proxy.ts). - this.ws = brokerWebSocket(`wss://demo.tradovateapi.com/v1/websocket?r=${randomnumber}`); - - this.ws.onopen = () => { - console.log('Connected to websocket'); - this.ws.send('authorize\n2\n\n' + this.accessInfo.accessToken); - this.directEventCallbacks[2] = (response: any) => { - this.requestSync(); - this.syncInterval = setInterval(() => this.requestSync(), 60000); - - // Every 2.5 seconds send a heartbeat - this.heartbeatInterval = setInterval(() => { - this.ws.send('[]'); - }, 2500); - }; - }; - - this.ws.onmessage = (event) => { - if (event.data[0] === 'a') { - const dataString = event.data.slice(1); - - const data: ( - | { - i?: number; // Id - s?: number; // Status - d?: any; // Data - } - | { - e?: string; - d?: { - entityType: TradovateEntityType; - eventType: 'Created' | 'Updated'; - entity: any; - }; - } - )[] = JSON.parse(dataString); - - for (const response of data) { - if ( - 'i' in response && - response.i && - this.directEventCallbacks[response.i] - ) { - this.directEventCallbacks[response.i](response.d); - } - - if ('e' in response && response.e) { - // Handle callbacks - for (const info of this.indirectEventCallbacks) { - if ( - response.d?.entityType === info?.entityType && - response.d?.eventType === info?.eventType && - info?.validator(response.d.entity) - ) { - info?.callback(response.d.entity); - } - } - - // Buffer recent entity events (last 50) - if (response.d?.entityType) { - this.recentEntityEvents.push({ entityType: response.d.entityType, eventType: response.d.eventType, entity: response.d.entity, ts: Date.now() }); - if (this.recentEntityEvents.length > 50) this.recentEntityEvents.shift(); - } - - // Update positions from WebSocket position events - if (response.d?.entityType === 'position' && response.d?.entity) { - const pos = response.d.entity; - if (pos.netPos !== 0) { - this.positions[pos.accountId] = { - contractId: pos.contractId, - netPos: pos.netPos, - netPrice: pos.netPrice, - timestamp: new Date(pos.timestamp), - }; - } else { - delete this.positions[pos.accountId]; - } - } - - // Update cash balances from WebSocket cashBalance events - if (response.d?.entityType === 'cashBalance' && response.d?.entity) { - const cb = response.d.entity; - if (cb.accountId) { - this.accountCashBalances[cb.accountId] = { - amount: cb.amount, - realizedPnL: cb.realizedPnL, - }; - } - } - } - } - } else if (event.data[0] === 'h') { - // Heartbeat response - } - }; - - this.ws.onerror = (event) => { - console.error('Error on websocket', event); - }; - - this.ws.onclose = (event) => { + /** + * Logs in (unless the current token is still good) and opens a fresh + * socket. Single-flight, never throws, never leaves a rejection unhandled: + * every way it can fail ends in scheduleReconnect(). + */ + private async connectAndAuth(): Promise { + if (this.connecting || this.intentionalDisconnect) return; + this.connecting = true; + try { + // A reconnect after a plain socket drop reuses a token with life + // left in it — fewer logins, fewer rate-limit penalties while + // Tradovate flaps during maintenance. Anything that makes the token + // suspect sets needsLogin. + if (this.needsLogin || !this.tokenValidFor(TOKEN_REUSE_MIN_MS)) { + const outcome = await this.attemptLogin(); if (this.intentionalDisconnect) return; - console.warn(`[tradovate] WebSocket closed (code=${event.code}) — reconnecting in 5s`); - if (this.syncInterval) { clearInterval(this.syncInterval); this.syncInterval = null; } - if (this.heartbeatInterval) { clearInterval(this.heartbeatInterval); this.heartbeatInterval = null; } - setTimeout(() => this.connectAndAuth(), 5_000); - }; + if (!outcome.ok) { + this.consecutiveLoginFailures++; + this.lastLoginError = outcome.error; + this.scheduleReconnect( + `login failed: ${outcome.error} (${this.consecutiveLoginFailures} in a row)`, + outcome.retryInMs ?? reconnectBackoffMs(this.consecutiveLoginFailures), + ); + return; + } + this.accessInfo = outcome.info; + this.noteTokenExpiry(); + this.lastLoginAt = Date.now(); + this.lastLoginError = null; + this.consecutiveLoginFailures = 0; + this.needsLogin = false; + console.log(`${this.tag} logged in — token expires ${outcome.info.expirationTime}`); + } + this.openSocket(); + } catch (err) { + // Nothing above is expected to throw (brokerWebSocket can, on a bad + // proxy setting) — but a throw must never end the reconnect loop. + this.consecutiveSocketFailures++; + this.scheduleReconnect(`connect failed: ${errorMessage(err)}`, reconnectBackoffMs(this.consecutiveSocketFailures)); + } finally { + this.connecting = false; + } + } + + /** + * One login attempt, riding out up to MAX_PENALTY_ROUNDS rate-limit + * penalties: Tradovate answers a throttled login with a p-ticket to present + * again after p-time seconds. p-captcha means it wants a human — never try + * to solve it, back off for a long while instead. + */ + private async attemptLogin(): Promise { + let pTicket: string | undefined; + for (let round = 0; ; round++) { + let body: LoginResponseBody | undefined; + try { + body = await this.login(pTicket); + } catch (err) { + // axios rejects every non-2xx (and timeouts); a penalty ticket + // could ride on one of those too. + const data = (err as { response?: { data?: unknown } })?.response?.data as LoginResponseBody | undefined; + if (data && typeof data === 'object' && data['p-ticket']) { + body = data; + } else { + const text = data && typeof data === 'object' && typeof data.errorText === 'string' ? `: ${data.errorText}` : ''; + return { ok: false, error: `${errorMessage(err)}${text}` }; + } + } + if (this.intentionalDisconnect) return { ok: false, error: 'client disconnected' }; + + if (body?.accessToken) return { ok: true, info: body as AuthLoginResponse }; + if (body?.['p-captcha']) { + return { ok: false, error: 'Tradovate asked for a captcha (too many logins)', retryInMs: CAPTCHA_RETRY_MS }; + } + const ticket = body?.['p-ticket']; + const pTime = Number(body?.['p-time']); + if (ticket && pTime > 0) { + if (round >= MAX_PENALTY_ROUNDS) { + return { ok: false, error: `still rate-limited after ${round} penalty waits` }; + } + const waitMs = Math.min(pTime * 1000, PENALTY_MAX_WAIT_MS); + console.warn(`${this.tag} login rate-limited — waiting ${fmtDelay(waitMs)} to retry with the penalty ticket`); + pTicket = ticket; + await this.penaltySleep(waitMs); + if (this.intentionalDisconnect) return { ok: false, error: 'client disconnected' }; + continue; + } + return { ok: false, error: body?.errorText ? String(body.errorText) : 'login answered without an access token' }; + } + } + + private penaltySleep(ms: number): Promise { + return new Promise((resolve) => { + this.nextReconnectAt = Date.now() + ms; + const timer = setTimeout(() => { + this.penaltyWait = null; + this.nextReconnectAt = 0; + resolve(); + }, ms); + this.penaltyWait = { timer, resolve }; }); } - private requestSync(): void { - this.directEventCallbacks[3] = (response: any) => { - try { - if (!response) { - console.error('[requestSync] Received null/undefined response — auth may have failed'); - return; + /** + * Arms the one reconnect timer. Never stacks: if one is already pending it + * stays as it is (a login backoff isn't cut short by a later socket event). + */ + private scheduleReconnect(reason: string, delayMs = SOCKET_DROP_RECONNECT_MS): void { + if (this.intentionalDisconnect) return; + if (this.reconnectTimer) { + console.warn(`${this.tag} ${reason} — reconnect already due in ${fmtDelay(Math.max(0, this.nextReconnectAt - Date.now()))}`); + return; + } + const wait = jitter(delayMs); + this.nextReconnectAt = Date.now() + wait; + console.warn(`${this.tag} ${reason} — reconnecting in ${fmtDelay(wait)}`); + this.reconnectTimer = setTimeout(() => { + this.reconnectTimer = null; + this.nextReconnectAt = 0; + void this.connectAndAuth(); + }, wait); + } + + private openSocket(): void { + // Only ever one live socket per client — a leftover would keep feeding + // callbacks and heartbeats alongside the new one. + if (this.ws) this.dropSocket('replaced by a new connection'); + + const randomnumber = Math.random().toString(36).substring(2, 15); + // Broker endpoint — dialled through the operator's proxy when one + // is configured (lib/outbound-proxy.ts). + const ws = brokerWebSocket(`wss://demo.tradovateapi.com/v1/websocket?r=${randomnumber}`); + this.ws = ws; + this.socketOpenedAt = Date.now(); + this.authorizeSent = false; + this.authorizedAt = 0; + this.closingSeenAt = 0; + // Reset sync state so scheduler waits for the new sync to complete + this.syncComplete = false; + + // Every handler first checks it still belongs to the current socket: + // a replaced socket's late events must not touch the new session. + ws.onopen = () => { + if (this.ws !== ws) return; + this.socketOpenedAt = Date.now(); // the authorize deadline runs from here + console.log(`${this.tag} websocket connected — authorizing`); + this.directEventCallbacks[2] = (_response: unknown, status?: number) => this.onAuthorized(ws, status); + if (this.sendFrame('authorize\n2\n\n' + this.accessInfo.accessToken)) { + this.authorizeSent = true; + } else { + this.abandonSocket('authorize could not be sent'); + } + }; + + ws.onmessage = (event) => { + if (this.ws !== ws) return; + // Any frame ('o', 'h', 'a', …) proves the session is alive. + this.lastInboundAt = Date.now(); + this.handleFrame(event.data); + }; + + ws.onerror = (event) => { + if (this.ws !== ws) return; + // onclose follows and does the recovery; just note it (no event dump). + const msg = 'message' in event ? (event as ErrorEvent).message : ''; + console.warn(`${this.tag} websocket error${typeof msg === 'string' && msg ? `: ${msg}` : ''}`); + }; + + ws.onclose = (event) => { + if (this.ws !== ws) return; + this.abandonSocket(`websocket closed (code=${event.code})`); + }; + } + + private onAuthorized(ws: WebSocket, status?: number): void { + if (this.ws !== ws) return; + if (typeof status === 'number' && status !== 200) { + // The token was refused — another socket with it would be refused too. + this.abandonSocket(`websocket authorize refused (status ${status})`, true); + return; + } + this.authorizedAt = Date.now(); + console.log(`${this.tag} websocket authorized — syncing`); + + this.clearSessionIntervals(); + this.requestSync(); + this.syncInterval = setInterval(() => { + if (this.ws === ws) this.requestSync(); + }, 60000); + + // Every 2.5 seconds send a heartbeat + this.heartbeatInterval = setInterval(() => { + if (this.ws === ws) this.sendFrame('[]'); + }, 2500); + } + + private handleFrame(raw: unknown): void { + // 'o' open, 'h' heartbeat, 'c' close — nothing to do beyond liveness. + if (typeof raw !== 'string' || raw[0] !== 'a') return; + + let data: ( + | { + i?: number; // Id + s?: number; // Status + d?: any; // Data + } + | { + e?: string; + d?: { + entityType: TradovateEntityType; + eventType: 'Created' | 'Updated'; + entity: any; + }; + } + )[]; + try { + data = JSON.parse(raw.slice(1)); + } catch { + console.warn(`${this.tag} unparseable frame (${raw.length} bytes) ignored`); + return; + } + if (!Array.isArray(data)) return; + + for (const response of data) { + if (!response || typeof response !== 'object') continue; + + if ( + 'i' in response && + response.i && + this.directEventCallbacks[response.i] + ) { + // One-shot: removed before it runs, so a handler that re-registers + // its id (the fixed ids 2/3/10/30 are per request) isn't undone. + const id = response.i; + const callback = this.directEventCallbacks[id]; + delete this.directEventCallbacks[id]; + delete this.pendingRejecters[id]; + try { + callback(response.d, response.s); + } catch (err) { + console.error(`${this.tag} handler for request ${id} failed: ${errorMessage(err)}`); } + } + + if ('e' in response && response.e) { + // Handle callbacks — all one-shot (fill waits), so take them out + // of the list before running them. + const matched = this.indirectEventCallbacks.filter((info) => + response.d?.entityType === info.entityType && + response.d?.eventType === info.eventType && + info.validator(response.d.entity) + ); + if (matched.length > 0) { + this.indirectEventCallbacks = this.indirectEventCallbacks.filter((info) => !matched.includes(info)); + for (const info of matched) { + try { + info.callback(response.d?.entity); + } catch (err) { + console.error(`${this.tag} ${info.entityType} handler failed: ${errorMessage(err)}`); + } + } + } + + // Buffer recent entity events (last 50) + if (response.d?.entityType) { + this.recentEntityEvents.push({ entityType: response.d.entityType, eventType: response.d.eventType, entity: response.d.entity, ts: Date.now() }); + if (this.recentEntityEvents.length > 50) this.recentEntityEvents.shift(); + } + + // Update positions from WebSocket position events + if (response.d?.entityType === 'position' && response.d?.entity) { + const pos = response.d.entity; + if (pos.netPos !== 0) { + this.positions[pos.accountId] = { + contractId: pos.contractId, + netPos: pos.netPos, + netPrice: pos.netPrice, + timestamp: new Date(pos.timestamp), + }; + } else { + delete this.positions[pos.accountId]; + } + } + + // Update cash balances from WebSocket cashBalance events + if (response.d?.entityType === 'cashBalance' && response.d?.entity) { + const cb = response.d.entity; + if (cb.accountId) { + this.accountCashBalances[cb.accountId] = { + amount: cb.amount, + realizedPnL: cb.realizedPnL, + }; + } + } + } + } + } + + /** + * Gives up on the current socket (if any) and schedules the way back. + * Used for every way a session ends — onclose, and the watchdog for the + * ways that never fire onclose. `relogin` means the token itself is + * suspect: log in afresh, after a short delay. + */ + private abandonSocket(reason: string, relogin = false): void { + if (this.intentionalDisconnect) return; + if (relogin) this.needsLogin = true; + // A connect in flight opens a fresh socket anyway (and logs in first + // if needsLogin was just set before its login) — don't race it. + if (this.connecting) return; + + const hadSocket = this.ws !== null; + const wasAuthorized = this.authorizedAt > 0; + const tokenPresented = this.authorizeSent; + // lastSyncAt only moves on a good sync, and one on this socket can't + // predate its open. + const synced = wasAuthorized && this.lastSyncAt >= this.socketOpenedAt; + this.dropSocket(reason); + + let delay = relogin ? FORCED_RELOGIN_DELAY_MS : SOCKET_DROP_RECONNECT_MS; + if (hadSocket && !synced) { + // A session that died before it produced a single good sync backs + // off if it keeps happening, rather than reconnecting every few + // seconds — and, when it forces a login (a sync refused with 401 + // on every fresh session), rather than logging in every second. + // If it presented the token and got no yes, the token is suspect + // too: log in afresh. A handshake that never completed says + // nothing about the token — keep it, so a websocket-host outage + // doesn't turn into a login every few seconds (and a rate-limit + // penalty on top). + if (tokenPresented && !wasAuthorized) this.needsLogin = true; + this.consecutiveSocketFailures++; + if (this.consecutiveSocketFailures > 1) delay = reconnectBackoffMs(this.consecutiveSocketFailures); + } + this.scheduleReconnect(reason, delay); + } + + /** Detaches and closes the current socket and clears everything that + * belonged to its session. Pending requests fail with `reason`. */ + private dropSocket(reason: string): void { + const ws = this.ws; + this.ws = null; + if (ws) { + ws.onopen = null; + ws.onmessage = null; + ws.onerror = null; + ws.onclose = null; + try { ws.close(); } catch { /* already closing */ } + } + this.clearSessionIntervals(); + this.syncComplete = false; + this.authorizeSent = false; + this.authorizedAt = 0; + this.closingSeenAt = 0; + this.syncRequestedAt = 0; + this.failPendingRequests(reason); + } + + private clearSessionIntervals(): void { + if (this.syncInterval) { clearInterval(this.syncInterval); this.syncInterval = null; } + if (this.heartbeatInterval) { clearInterval(this.heartbeatInterval); this.heartbeatInterval = null; } + } + + /** Answers to requests sent on a socket can't arrive on the next one: + * reject the awaited ones, drop the internal ones (authorize, sync, + * product list — re-sent on the new socket). Fill watchers stay: fills + * pushed after the reconnect still arrive. */ + private failPendingRequests(reason: string): void { + const rejecters = Object.values(this.pendingRejecters); + this.pendingRejecters = {}; + this.directEventCallbacks = {}; + for (const reject of rejecters) reject(reason); + } + + /** Every 15s: catch the ways a session dies without onclose, and make sure + * something is always pending while the client is meant to be connected. */ + private checkLiveness(): void { + // A connect in flight is bounded (HTTP deadlines, capped penalty waits). + if (this.intentionalDisconnect || this.connecting) return; + try { + const problem = this.livenessProblem(Date.now()); + if (problem) { + this.abandonSocket(problem.reason, problem.relogin); + return; + } + if (!this.ws && !this.reconnectTimer) this.scheduleReconnect('nothing scheduled'); + } catch (err) { + console.error(`${this.tag} liveness check failed: ${errorMessage(err)}`); + } + } + + private livenessProblem(now: number): { reason: string; relogin?: boolean } | null { + const ws = this.ws; + if (!ws) return null; + + const state = ws.readyState; + if (state === WebSocket.CLOSING || state === WebSocket.CLOSED) { + // onclose normally takes it from here — unless it never fires. + if (!this.closingSeenAt) this.closingSeenAt = now; + else if (now - this.closingSeenAt >= STUCK_CLOSING_MS) return { reason: 'websocket stuck closing' }; + return null; + } + this.closingSeenAt = 0; + + // Renewal normally keeps this from happening; a session on an expired + // token is dead whatever the socket says. + const expiresAt = this.tokenExpiresAt(); + if (expiresAt > 0 && expiresAt <= now) return { reason: 'access token expired', relogin: true }; + + if (state === WebSocket.CONNECTING) { + return now - this.socketOpenedAt > HANDSHAKE_TIMEOUT_MS ? { reason: 'websocket handshake timed out' } : null; + } + + if (!this.authorizedAt) { + return now - this.socketOpenedAt > AUTHORIZE_TIMEOUT_MS + ? { reason: 'websocket authorize never answered', relogin: true } + : null; + } + if (now - Math.max(this.lastInboundAt, this.authorizedAt) > INBOUND_SILENCE_MS) { + return { reason: `no frames for ${INBOUND_SILENCE_MS / 1000}s` }; + } + if (now - this.authorizedAt > SYNC_STALE_MS && now - this.lastSyncAt > SYNC_STALE_MS) { + return { reason: 'sync responses stopped', relogin: true }; + } + return null; + } + + /** Sends a frame on the current socket if it is open. Never throws. */ + private sendFrame(frame: string): boolean { + const ws = this.ws; + if (!ws || ws.readyState !== WebSocket.OPEN) return false; + try { + ws.send(frame); + return true; + } catch (err) { + console.warn(`${this.tag} websocket send failed: ${errorMessage(err)}`); + return false; + } + } + + /** Why `ws` can't carry a request right now, or null if it can. */ + private socketProblem(ws: WebSocket | null): string | null { + if (!ws) return 'Tradovate socket not connected'; + if (ws.readyState !== WebSocket.OPEN) return `Tradovate socket ${READY_STATE_NAMES[ws.readyState] ?? 'not open'}`; + if (!this.authorizedAt) return 'Tradovate socket not authorized yet'; + return null; + } + + /** + * Sends one request on the socket and settles on its answer; it can't + * hang. No answer within `timeoutMs`, or the socket going away first, + * rejects with a plain Error — ambiguous, the broker may have acted on it. + * Failing before anything was sent rejects with OrderNotPlacedError. + * `onAnswer` runs synchronously inside the frame handler, so a fill + * watcher it registers still sees events later in the same frame. + */ + private sendRequest( + endpoint: string, + rest: string, + timeoutMs: number, + onAnswer: (data: unknown, status?: number) => T | PromiseLike, + ): Promise { + const ws = this.ws; + const problem = this.socketProblem(ws); + if (!ws || problem) return Promise.reject(new OrderNotPlacedError(`${endpoint} not sent — ${problem}`)); + + const id = this.getMsgId(); + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { + delete this.directEventCallbacks[id]; + delete this.pendingRejecters[id]; + reject(new Error(`${endpoint}: no answer from Tradovate within ${timeoutMs / 1000}s — outcome unknown`)); + }, timeoutMs); + this.directEventCallbacks[id] = (data: unknown, status?: number) => { + clearTimeout(timer); + try { + resolve(onAnswer(data, status)); + } catch (err) { + reject(err); + } + }; + this.pendingRejecters[id] = (reason: string) => { + clearTimeout(timer); + reject(new Error(`${endpoint}: ${reason} before Tradovate answered — outcome unknown`)); + }; + try { + ws.send(`${endpoint}\n${id}\n${rest}`); + } catch (err) { + clearTimeout(timer); + delete this.directEventCallbacks[id]; + delete this.pendingRejecters[id]; + reject(new OrderNotPlacedError(`${endpoint} not sent — ${errorMessage(err)}`)); + } + }); + } + + /** Runs the sync-complete handler without letting it reject unhandled. */ + private notifySyncComplete(): void { + try { + this.callbackOnSyncRequest().catch((err) => { + console.error(`${this.tag} sync-complete handler failed: ${errorMessage(err)}`); + }); + } catch (err) { + console.error(`${this.tag} sync-complete handler failed: ${errorMessage(err)}`); + } + } + + private requestSync(): void { + if (this.socketProblem(this.ws)) return; + this.directEventCallbacks[3] = (response: any, status?: number) => { + this.syncRequestedAt = 0; + // A refused or empty answer must not wipe the accounts, positions + // and balances we have — nor pass for a fresh sync. + if (!response || typeof response === 'string' || (typeof status === 'number' && status !== 200)) { + console.error(`${this.tag} sync request failed (status ${status ?? '?'}): ${typeof response === 'string' && response ? response : 'empty response'}`); + // 401: the session is gone — no point waiting for the watchdog. + if (status === 401) this.abandonSocket('sync refused (status 401)', true); + return; + } + try { // liquidateOnly flag lives in accountRiskStatuses const riskStatusById: { [accountId: number]: { liquidateOnly?: string } } = ( response.accountRiskStatuses || [] @@ -324,31 +849,41 @@ export class TradovateClient { {} as { [accountId: number]: { contractId: number; netPos: number; netPrice: number; timestamp: Date } } ); - console.log(`[requestSync] ${this.accountList.length} account(s), ${Object.keys(this.accountCashBalances).length} balance(s)`); + this.lastSyncAt = Date.now(); + // A session counts as working once it has synced (abandonSocket). + this.consecutiveSocketFailures = 0; + console.log(`${this.tag} sync:${this.accountList.length} account(s), ${Object.keys(this.accountCashBalances).length} balance(s)`); this.maybeFetchDaysTraded(); - if (this.products.length > 0) { + // The product list only feeds fetchInstrumentFees. Wait for it + // on a session's first sync, but not for ever: still unanswered + // at the next sync, it is lost — sync without it (a late answer + // still loads it) rather than hold syncComplete false, and every + // firm on the instance with it, while the syncs themselves are good. + if (this.products.length > 0 || this.directEventCallbacks[30]) { this.syncComplete = true; - this.callbackOnSyncRequest(); + this.notifySyncComplete(); return; } this.directEventCallbacks[30] = (products: any) => { if (Array.isArray(products) && products.length > 0) { this.products = products.map((p: any) => ({ id: p.id, name: p.name })); - console.log(`Loaded ${this.products.length} products`); + console.log(`${this.tag} loaded ${this.products.length} products`); } this.syncComplete = true; - this.callbackOnSyncRequest(); + this.notifySyncComplete(); }; - this.ws.send('product/list\n30\n\n'); + this.sendFrame('product/list\n30\n\n'); } catch (err) { - console.error('[requestSync] Error processing sync response:', err); + console.error(`${this.tag} error processing sync response: ${errorMessage(err)}`); } }; - this.ws.send('user/syncrequest\n3\n\n{"splitResponses":false}'); + if (this.sendFrame('user/syncrequest\n3\n\n{"splitResponses":false}') && !this.syncRequestedAt) { + this.syncRequestedAt = Date.now(); + } } private maybeFetchDaysTraded(): void { @@ -379,7 +914,7 @@ export class TradovateClient { .then(({ failedAccounts }) => { if (failedAccounts > 0) { this.nextDaysFetchRetryAt = Date.now() + 5 * 60 * 1_000; - console.warn(`[fetchDaysTraded] ${failedAccounts} account(s) failed during ${reason} — retrying in 5 minutes`); + console.warn(`${this.tag} fetchDaysTraded: ${failedAccounts} account(s) failed during ${reason} — retrying in 5 minutes`); return; } @@ -387,7 +922,8 @@ export class TradovateClient { }) .catch((err) => { this.nextDaysFetchRetryAt = Date.now() + 5 * 60 * 1_000; - console.error(`[fetchDaysTraded] ${reason} failed — retrying in 5 minutes`, err); + // errorMessage(): an AxiosError logged whole dumps the Bearer token. + console.error(`${this.tag} fetchDaysTraded ${reason} failed — retrying in 5 minutes: ${errorMessage(err)}`); }) .finally(() => { this.daysFetchInFlight = false; @@ -477,6 +1013,9 @@ export class TradovateClient { const maxAttempts = 3; let lastErr: unknown = null; for (let attempt = 1; attempt <= maxAttempts; attempt++) { + // A discarded client stops polling rather than keep a report + // conversation going on a session nobody uses any more. + if (this.intentionalDisconnect) throw new Error('client disconnected'); try { let reportData = (await axios.post( 'https://rpt-demo.tradovateapi.com/v1/reports/requestreport', @@ -490,16 +1029,17 @@ export class TradovateClient { { name: 'account', value: accountName }, ], }, - { headers: authHeaders } + { headers: authHeaders, timeout: REPORT_TIMEOUT_MS } )).data; let pollAttempts = 0; while (reportData?.['p-ticket'] && pollAttempts < 30) { const pTicket: string = reportData['p-ticket']; const pTime: number = Math.max(1, reportData['p-time'] ?? 1); await new Promise((r) => setTimeout(r, pTime * 1000)); + if (this.intentionalDisconnect) throw new Error('client disconnected'); reportData = (await axios.get( 'https://rpt-demo.tradovateapi.com/v1/reports/getreport', - { params: { 'p-ticket': pTicket }, headers: authHeaders } + { params: { 'p-ticket': pTicket }, headers: authHeaders, timeout: REPORT_TIMEOUT_MS } )).data; pollAttempts++; } @@ -508,7 +1048,7 @@ export class TradovateClient { lastErr = err; if (!isTransient(err) || attempt === maxAttempts) throw err; const backoffMs = 500 * Math.pow(2, attempt - 1); // 500ms, 1s, 2s - console.warn(`[requestReport] ${accountName} ${name}: transient error (attempt ${attempt}/${maxAttempts}), retrying in ${backoffMs}ms`); + console.warn(`${this.tag} requestReport ${accountName} ${name}: transient error (${errorMessage(err)}, attempt ${attempt}/${maxAttempts}), retrying in ${backoffMs}ms`); await new Promise((r) => setTimeout(r, backoffMs)); } } @@ -529,6 +1069,9 @@ export class TradovateClient { }; for (const account of this.accountList) { + // A discarded client (credentials changed, firm removed) stops here + // rather than keep writing rows next to its replacement's. + if (this.intentionalDisconnect) break; if (onlyAccountIds && !onlyAccountIds.includes(account.id)) continue; // Load last known fund date and fund transactions from DB const storedFundDate = loadAccountMeta(account.id, 'last_fund_date'); @@ -704,13 +1247,14 @@ export class TradovateClient { saveDailyPnL(account.id, account.name, fresh); // upsert only fresh entries — preserves older cached rows this.lastFetchErrors[account.name] = FILLS_FALLBACK_NOTE; } catch (err) { + if (this.intentionalDisconnect) break; const msg = err instanceof Error ? err.message : String(err); - console.error(`[fetchDaysTraded] both reports failed for ${account.name}:`, msg); + console.error(`${this.tag} fetchDaysTraded: both reports failed for ${account.name}: ${msg}`); failedAccounts++; // Cache already loaded and filtered at top of loop — just log the error if (cached.length > 0) { this.lastFetchErrors[account.name] = `${msg} (using ${cached.length} cached entries from database)`; - console.log(`[fetchDaysTraded] using ${cached.length} cached entries from DB for ${account.name}`); + console.log(`${this.tag} fetchDaysTraded: using ${cached.length} cached entries from DB for ${account.name}`); } else { this.dailyPnL[account.id] ??= []; this.daysTraded[account.id] ??= 0; @@ -722,7 +1266,7 @@ export class TradovateClient { return { failedAccounts }; } - private async login(): Promise { + private async login(pTicket?: string): Promise { const chl = '' + (Date.now() - 1581e9); const sec = computeSec(this.name, this.password, this.deviceId, chl); @@ -733,6 +1277,9 @@ export class TradovateClient { return btoa(r); }; + // Non-2xx and timeouts reject (callers log errorMessage(), never the + // AxiosError itself — its config carries this body). The abort signal + // backs the timeout up for a connect that never gets a socket. const res = await axios.post('https://live.tradovateapi.com/v1/auth/accesstokenrequest', { name: this.name, password: generateEncodedPassword({ name: this.name, password: this.password }), @@ -744,33 +1291,100 @@ export class TradovateClient { chl: chl, sec: sec, enc: true, - }); + ...(pTicket ? { 'p-ticket': pTicket } : {}), + }, { timeout: HTTP_TIMEOUT_MS, signal: AbortSignal.timeout(HTTP_TIMEOUT_MS + 5_000) }); - if (res.status !== 200) { - console.error('Failed to login', res.data); + return res.data ?? {}; + } + + /** Access token expiry as epoch ms (0 = none, unparseable, or not + * comparable with this machine's clock — see noteTokenExpiry()). */ + private tokenExpiresAt(): number { + if (this.tokenExpiryUntrusted) return 0; + const raw = this.accessInfo?.expirationTime; + if (!raw) return 0; + const t = new Date(raw).getTime(); + return Number.isFinite(t) ? t : 0; + } + + /** + * Called whenever a token arrives (login or renewal). Tradovate hands out + * tokens with well over an hour to live, so one that is already expired — + * or nearly — by our clock means this machine's clock is ahead of + * Tradovate's, not that the token is dead. Trusting that expiry would have + * the watchdog "expire" every fresh token and log in again every 15s (a + * rate-limit penalty, then a captcha, for every client on this IP). + * Treat the expiry as unknown instead: renew every 2 minutes and log in on + * reconnect, as the code did before it read the expiry at all. + */ + private noteTokenExpiry(): void { + const raw = this.accessInfo?.expirationTime; + const t = raw ? new Date(raw).getTime() : NaN; + const untrusted = Number.isFinite(t) && t - Date.now() < TOKEN_RENEW_WINDOW_MS; + if (untrusted && !this.tokenExpiryUntrusted) { + console.warn(`${this.tag} new token expires ${raw}, which is (nearly) past by this machine's clock — check the system clock; ignoring the expiry`); } + this.tokenExpiryUntrusted = untrusted; + } - return res.data; + /** Holding a token known to stay valid for at least `ms` more. */ + private tokenValidFor(ms: number): boolean { + return !!this.accessInfo?.accessToken && this.tokenExpiresAt() > Date.now() + ms; + } + + /** The 2-minute renewal check. Never rejects, never overlaps itself. */ + private maybeRenewToken(): void { + if (this.intentionalDisconnect || this.connecting || this.renewInFlight || !this.accessInfo?.accessToken) return; + const now = Date.now(); + const expiresAt = this.tokenExpiresAt(); + // An unparseable expiry renews anyway — an extra renewal beats a silent expiry. + if (expiresAt > 0 && expiresAt - now > TOKEN_RENEW_WINDOW_MS) return; + // Already expired: renewal can't save it; the watchdog / next connect logs in. + if (expiresAt > 0 && expiresAt <= now) return; + + this.renewInFlight = true; + this.renewAccessToken() + .catch((err) => console.error(`${this.tag} token renewal failed: ${errorMessage(err)}`)) + .finally(() => { this.renewInFlight = false; }); } private async renewAccessToken(): Promise { - const res = await axios.get('https://live.tradovateapi.com/v1/auth/renewAccessToken', { - headers: { - Authorization: `Bearer ${this.accessInfo.accessToken}`, - }, - }); + const token = this.accessInfo?.accessToken; + if (!token) return; + try { + const res = await axios.get('https://live.tradovateapi.com/v1/auth/renewAccessToken', { + headers: { + Authorization: `Bearer ${token}`, + }, + timeout: HTTP_TIMEOUT_MS, + signal: AbortSignal.timeout(HTTP_TIMEOUT_MS + 5_000), + }); + if (this.intentionalDisconnect) return; + // A re-login while this was in flight already replaced the token. + if (this.accessInfo?.accessToken !== token) return; - if (res.status !== 200) { - console.error('Failed to renew access token', res.data); + const body = res.data as LoginResponseBody | undefined; + if (!body?.accessToken) { + throw new Error(body?.errorText ? String(body.errorText) : 'renewal answered without an access token'); + } + // Merge, so a renewal answer without expirationTime can't leave it undefined. + this.accessInfo = { ...this.accessInfo, ...body } as AuthLoginResponse; + this.noteTokenExpiry(); + console.log(`${this.tag} renewed access token — expires ${this.accessInfo.expirationTime}`); + } catch (err) { + if (this.intentionalDisconnect || this.accessInfo?.accessToken !== token) return; + console.warn(`${this.tag} token renewal failed: ${errorMessage(err)}`); + const status = (err as { response?: { status?: unknown } })?.response?.status; + const expiresAt = this.tokenExpiresAt(); + if (status === 401 || (expiresAt > 0 && expiresAt - Date.now() < TOKEN_RELOGIN_WINDOW_MS)) { + this.abandonSocket('token renewal failed', true); + } } - - console.log('Renewed access token'); - this.accessInfo = res.data; } async getAccountList(): Promise { if (!this.accessInfo?.accessToken) { - console.log('Not authenticated'); + console.log(`${this.tag} not authenticated`); return []; } @@ -778,6 +1392,7 @@ export class TradovateClient { headers: { Authorization: `Bearer ${this.accessInfo.accessToken}`, }, + timeout: HTTP_TIMEOUT_MS, }); return res.data; @@ -790,22 +1405,39 @@ export class TradovateClient { const cached = getCachedContract(productName); if (cached) return { id: cached.id, name: cached.name, tickSize: cached.tickSize }; - // If the cache is cold, resolve on demand with the same logic the settings page uses. - const resolved = await resolveContracts([productName], this.accessInfo.accessToken); - const contract = resolved[productName]; - if (contract) { - return { id: contract.id, name: contract.name, tickSize: contract.tickSize }; + // Lookups fail (a 401 on an expiring token, a network blip) — that must + // come back as "no contract", not a throw that aborts every firm's trade. + try { + // If the cache is cold, resolve on demand with the same logic the settings page uses. + // Under a deadline: its lookups (Yahoo, then Tradovate) must not + // hold every firm's trade. A late answer still fills the cache. + const resolved = await withDeadline( + resolveContracts([productName], this.accessInfo.accessToken), + CONTRACT_RESOLVE_DEADLINE_MS, + 'contract resolve', + ); + const contract = resolved[productName]; + if (contract) { + return { id: contract.id, name: contract.name, tickSize: contract.tickSize }; + } + } catch (err) { + console.warn(`${this.tag} contract resolve for ${productName} failed: ${errorMessage(err)} — trying contract/suggest`); } - // Last resort: fall back to Tradovate's ordered suggestions. - const res = await axios.get( - `https://demo.tradovateapi.com/v1/contract/suggest?t=${encodeURIComponent(productName)}&l=20`, - { headers: { Authorization: `Bearer ${this.accessInfo.accessToken}` } } - ); - const contracts: Array<{ id: number; name: string; status: string; providerTickSize: number }> = res.data ?? []; - const match = contracts.find((c) => c.name.startsWith(productName)); - if (!match) return null; - return { id: match.id, name: match.name, tickSize: match.providerTickSize ?? 0.25 }; + try { + // Last resort: fall back to Tradovate's ordered suggestions. + const res = await axios.get( + `https://demo.tradovateapi.com/v1/contract/suggest?t=${encodeURIComponent(productName)}&l=20`, + { headers: { Authorization: `Bearer ${this.accessInfo.accessToken}` }, timeout: HTTP_TIMEOUT_MS } + ); + const contracts: Array<{ id: number; name: string; status: string; providerTickSize: number }> = res.data ?? []; + const match = contracts.find((c) => c.name.startsWith(productName)); + if (!match) return null; + return { id: match.id, name: match.name, tickSize: match.providerTickSize ?? 0.25 }; + } catch (err) { + console.error(`${this.tag} contract lookup for ${productName} failed: ${errorMessage(err)}`); + return null; + } } async fetchInstrumentFees(symbols: string[]): Promise<{ [symbol: string]: number }> { @@ -818,7 +1450,7 @@ export class TradovateClient { const res = await axios.post( 'https://demo.tradovateapi.com/v1/contract/getproductfeeparams', { productIds }, - { headers: { Authorization: `Bearer ${this.accessInfo.accessToken}` } } + { headers: { Authorization: `Bearer ${this.accessInfo.accessToken}` }, timeout: HTTP_TIMEOUT_MS } ); const result: { [symbol: string]: number } = {}; @@ -835,7 +1467,7 @@ export class TradovateClient { (param.orderRoutingFee ?? 0); result[product.name] = parseFloat(raw.toFixed(4)); console.log( - `[fees] ${product.name}: clearing=${param.clearingFee ?? 0}` + + `${this.tag} fees ${product.name}: clearing=${param.clearingFee ?? 0}` + ` exchange=${param.exchangeFee ?? 0}` + ` nfa=${param.nfaFee ?? 0}` + ` brokerage=${param.brokerageFee ?? 0}` + @@ -852,20 +1484,72 @@ export class TradovateClient { async requestContractsFromSocket(names: string[]): Promise<{ [name: string]: Contract; }> { - if (!this.ws) { - console.log('Websocket not connected'); + // Callers expect an object — a dead socket or a lost answer is {} with + // a warning, never a promise left pending. + try { + return await this.sendRequest('contract/finds', `names=${names.join(',')}`, CONTRACT_FIND_TIMEOUT_MS, (response) => { + const items: Contract[] = Array.isArray(response) ? response : []; + return items.reduce((acc: { [name: string]: Contract }, item: Contract) => { + return { ...acc, [item.name]: item }; + }, {} as { [name: string]: Contract }); + }); + } catch (err) { + console.warn(`${this.tag} contract/finds failed: ${errorMessage(err)}`); return {}; } + } - return new Promise((resolve) => { - this.directEventCallbacks[10] = (response: any) => { - resolve( - (response || []).reduce((acc: { [name: string]: Contract }, item: Contract) => { - return { ...acc, [item.name]: item }; - }, {} as { [name: string]: Contract }) - ); + /** A string or empty ack is the broker refusing the order — nothing was placed. */ + private static acceptedAck(response: unknown, endpoint: string): { orderId?: number; id?: number; commandStatus?: string } { + if (typeof response === 'string') throw new OrderNotPlacedError(response || `${endpoint} refused`); + if (!response) throw new OrderNotPlacedError(`Empty response from ${endpoint}`); + if (typeof response !== 'object') return {}; + const ack = response as { orderId?: number; id?: number; commandStatus?: string; failureReason?: string; failureText?: string }; + // PlaceOrderResult / PlaceOCOResult: a refusal (SessionClosed, + // LiquidationOnly, …) carries failureReason and no order id — no order + // exists. Silently taking it as placed left a position with no exit + // (placeoco) or waited 30s for a fill that could never come. + if (ack.orderId == null && ack.id == null + && ((ack.failureReason && ack.failureReason !== 'Success') || ack.failureText)) { + throw new OrderNotPlacedError(`${endpoint} refused: ${ack.failureText || ack.failureReason}`); + } + return ack; + } + + /** Resolves with the fill event for `orderId` (it carries the real + * execution price). Registered synchronously from the ack handler, so a + * fill later in the same frame as the ack is not missed. */ + private waitForFill(orderId: number | undefined): Promise> { + // Fills are pushed on the event stream, independently of the ack: one + // that got ahead of it (an earlier frame, or earlier in the same one) + // is already in the event buffer. Missed, a filled entry came back + // "outcome unknown" and the caller placed no exit on an open position. + const early = orderId == null ? undefined : this.recentEntityEvents.find((ev) => + ev.entityType === 'fill' && ev.eventType === 'Created' && ev.entity?.orderId === orderId); + if (early) { + console.log(`${this.tag} fill (ahead of its ack):`, JSON.stringify(early.entity)); + return Promise.resolve(early.entity); + } + return new Promise((resolve, reject) => { + const watcher: IndirectCallback = { + entityType: 'fill', + eventType: 'Created', + validator: (item: any) => item?.orderId === orderId, + callback: (fill: any) => { + clearTimeout(timeout); + console.log(`${this.tag} fill:`, JSON.stringify(fill)); + resolve(fill); + }, + cancel: (reason: string) => { + clearTimeout(timeout); + reject(new Error(`Order ${orderId} acknowledged but ${reason} before its fill arrived — outcome unknown`)); + }, }; - this.ws.send(`contract/finds\n10\nnames=${names.join(',')}`); + const timeout = setTimeout(() => { + this.indirectEventCallbacks = this.indirectEventCallbacks.filter((info) => info !== watcher); + reject(new Error(`Order ${orderId} acknowledged but no fill within 30s (market may be closed)`)); + }, FILL_TIMEOUT_MS); + this.indirectEventCallbacks.push(watcher); }); } @@ -877,58 +1561,45 @@ export class TradovateClient { orderType: 'Market' | 'Limit', price?: number ): Promise> { - if (!this.ws) throw new Error('WebSocket not connected'); + // Checked before the audit frame so a dead socket is a clean + // "not placed" rather than a send into nothing. + const problem = this.socketProblem(this.ws); + if (problem) throw new OrderNotPlacedError(`order/placeorder not sent — ${problem}`); - this.ws.send( + if (!this.sendFrame( `user/registeraudituseraction\n25\n\n${JSON.stringify({ accountId, actionType: action + orderType, details: `${action} ${quantity} ${contractSymbol} ${orderType}${price ? ` @ ${price}` : ''}, TIF Day`, })}` - ); + )) { + throw new OrderNotPlacedError('order/placeorder not sent — Tradovate socket rejected the audit frame'); + } - return new Promise((resolve, reject) => { - const msgId = this.getMsgId(); - this.directEventCallbacks[msgId] = (response: any) => { - console.log(`[sendOrder] raw ack:`, JSON.stringify(response)); - if (typeof response === 'string' || !response) { - reject(new Error(typeof response === 'string' ? response : 'Empty response from order/placeorder')); - return; - } + return this.sendRequest( + 'order/placeorder', + `\n${JSON.stringify({ + accountId, + action, + symbol: contractSymbol, + orderQty: quantity, + orderType, + price, + timeInForce: 'Day', + text: 'DOM', + })}`, + ACK_TIMEOUT_MS, + (response) => { + console.log(`${this.tag} sendOrder raw ack:`, JSON.stringify(response)); + const ack = TradovateClient.acceptedAck(response, 'order/placeorder'); // Tradovate returns the command object: { id, commandStatus, orderId, ... } - const orderId = response.orderId ?? response.id; - console.log(`[sendOrder] orderId=${orderId} status=${response.commandStatus}`); + const orderId = ack.orderId ?? ack.id; + console.log(`${this.tag} sendOrder orderId=${orderId} status=${ack.commandStatus}`); // Wait for the fill event — it carries the real execution price - const timeout = setTimeout(() => { - reject(new Error(`Order ${orderId} acknowledged but no fill within 30s (market may be closed)`)); - }, 30000); - - this.indirectEventCallbacks.push({ - entityType: 'fill', - eventType: 'Created', - validator: (item: any) => item?.orderId === orderId, - callback: (fill: any) => { - clearTimeout(timeout); - console.log(`[sendOrder] fill:`, JSON.stringify(fill)); - resolve(fill); - }, - }); - }; - - this.ws.send( - `order/placeorder\n${msgId}\n\n${JSON.stringify({ - accountId, - action, - symbol: contractSymbol, - orderQty: quantity, - orderType, - price, - timeInForce: 'Day', - text: 'DOM', - })}` - ); - }); + return this.waitForFill(orderId); + }, + ); } /** Place an order and resolve as soon as the command is acknowledged (does not wait for fill). */ @@ -940,31 +1611,24 @@ export class TradovateClient { orderType: 'Market' | 'Limit', price?: number ): Promise<{ orderId?: number }> { - if (!this.ws) throw new Error('WebSocket not connected'); - - const msgId = this.getMsgId(); - return new Promise((resolve, reject) => { - this.directEventCallbacks[msgId] = (response: any) => { - if (typeof response === 'string') { - reject(new Error(response)); - return; - } - resolve({ orderId: response?.orderId ?? response?.id }); - }; - - this.ws.send( - `order/placeorder\n${msgId}\n\n${JSON.stringify({ - accountId, - action, - symbol: contractSymbol, - orderQty: quantity, - orderType, - price, - timeInForce: 'Day', - text: 'DOM', - })}` - ); - }); + return this.sendRequest( + 'order/placeorder', + `\n${JSON.stringify({ + accountId, + action, + symbol: contractSymbol, + orderQty: quantity, + orderType, + price, + timeInForce: 'Day', + text: 'DOM', + })}`, + ACK_TIMEOUT_MS, + (response) => { + const ack = TradovateClient.acceptedAck(response, 'order/placeorder'); + return { orderId: ack.orderId ?? ack.id }; + }, + ); } /** Place an OCO exit pair after an entry fill: a limit take-profit linked @@ -979,49 +1643,78 @@ export class TradovateClient { limitPrice: number, stopPrice: number, ): Promise<{ orderId?: number }> { - if (!this.ws) throw new Error('WebSocket not connected'); - - const msgId = this.getMsgId(); - return new Promise((resolve, reject) => { - this.directEventCallbacks[msgId] = (response: any) => { - if (typeof response === 'string') { - reject(new Error(response)); - return; - } - resolve({ orderId: response?.orderId ?? response?.id }); - }; - - this.ws.send( - `order/placeoco\n${msgId}\n\n${JSON.stringify({ - accountId, + return this.sendRequest( + 'order/placeoco', + `\n${JSON.stringify({ + accountId, + action: exitAction, + symbol: contractSymbol, + orderQty: quantity, + orderType: 'Limit', + price: limitPrice, + timeInForce: 'Day', + text: 'DOM', + other: { action: exitAction, - symbol: contractSymbol, - orderQty: quantity, - orderType: 'Limit', - price: limitPrice, + orderType: 'Stop', + stopPrice, timeInForce: 'Day', text: 'DOM', - other: { - action: exitAction, - orderType: 'Stop', - stopPrice, - timeInForce: 'Day', - text: 'DOM', - }, - })}` - ); - }); + }, + })}`, + ACK_TIMEOUT_MS, + (response) => { + const ack = TradovateClient.acceptedAck(response, 'order/placeoco'); + return { orderId: ack.orderId ?? ack.id }; + }, + ); } - /** Close the WebSocket connection and stop all intervals. Call before discarding the instance. */ + /** Liveness snapshot for /api/health and the registry supervisor. No tokens. */ + public health(): ClientHealth { + const now = Date.now(); + const expiresAt = this.tokenExpiresAt(); + return { + platform: 'tradovate', + authenticated: !!this.accessInfo?.accessToken && (expiresAt === 0 || expiresAt > now), + transport: this.ws ? (READY_STATE_NAMES[this.ws.readyState] ?? String(this.ws.readyState)) : 'none', + syncComplete: this.syncComplete, + lastSyncAt: this.lastSyncAt, + lastLoginAt: this.lastLoginAt, + lastLoginError: this.lastLoginError, + consecutiveLoginFailures: this.consecutiveLoginFailures, + nextReconnectAt: this.nextReconnectAt, + tokenExpiresAt: expiresAt, + lastInboundAt: this.lastInboundAt, + pendingRequests: Object.keys(this.directEventCallbacks).length + this.indirectEventCallbacks.length, + syncInFlightSince: this.syncRequestedAt, + daysFetchInFlightSince: this.daysFetchInFlight ? this.lastDaysFetchStartedAt : 0, + }; + } + + /** Close the WebSocket connection and stop all timers. Call before discarding + * the instance — nothing it scheduled fires afterwards. */ public disconnect(): void { this.intentionalDisconnect = true; - if (this.syncInterval) { clearInterval(this.syncInterval); this.syncInterval = null; } - if (this.heartbeatInterval) { clearInterval(this.heartbeatInterval); this.heartbeatInterval = null; } + if (this.reconnectTimer) { clearTimeout(this.reconnectTimer); this.reconnectTimer = null; } + this.nextReconnectAt = 0; + if (this.livenessInterval) { clearInterval(this.livenessInterval); this.livenessInterval = null; } if (this.tokenRenewalInterval) { clearInterval(this.tokenRenewalInterval); this.tokenRenewalInterval = null; } + if (this.penaltyWait) { + // Cut the wait short; the login loop sees intentionalDisconnect and stops. + const { timer, resolve } = this.penaltyWait; + this.penaltyWait = null; + clearTimeout(timer); + resolve(); + } + // Closes the socket with its handlers detached, clears sync/heartbeat, + // rejects pending acks (ambiguous: they may have been placed). + this.dropSocket('client disconnected'); + const watchers = this.indirectEventCallbacks; + this.indirectEventCallbacks = []; + for (const info of watchers) info.cancel?.('client disconnected'); this.daysFetchInFlight = false; this.nextDaysFetchRetryAt = 0; - try { this.ws?.close(); } catch { /* ignore */ } } /** Register a one-time callback for when a fill arrives for a given orderId. */ diff --git a/lib/version.ts b/lib/version.ts index d265b81..29e490b 100644 --- a/lib/version.ts +++ b/lib/version.ts @@ -22,12 +22,16 @@ export interface BuildVersion { time: string; } -let cached: BuildVersion | null | undefined; +// Per process, not per bundle: instrumentation.ts and the route handlers load +// their own copies of this module, and each would otherwise read HEAD at its own +// first call — after an update's pull, a late first call reports code that isn't +// running yet. +const _g = globalThis as typeof globalThis & { __buildVersion?: BuildVersion | null }; /** null when this isn't a git checkout or git isn't installed — both mean * "no version to report", and the master shows the machine as an old build. */ export function getBuildVersion(): BuildVersion | null { - if (cached !== undefined) return cached; + if (_g.__buildVersion !== undefined) return _g.__buildVersion; try { const out = execFileSync('git', ['log', '-1', '--format=%h %cI'], { cwd: process.cwd(), @@ -35,9 +39,9 @@ export function getBuildVersion(): BuildVersion | null { stdio: ['ignore', 'pipe', 'ignore'], }).trim(); const [commit, time] = out.split(' '); - cached = commit && time ? { commit, time } : null; + _g.__buildVersion = commit && time ? { commit, time } : null; } catch { - cached = null; + _g.__buildVersion = null; } - return cached; + return _g.__buildVersion; } diff --git a/scripts/start-all.mjs b/scripts/start-all.mjs index 70bba17..f295bc2 100644 --- a/scripts/start-all.mjs +++ b/scripts/start-all.mjs @@ -4,10 +4,12 @@ * * Run by the "at log on" scheduled task. Safe to run by hand at any time. * - * The warm-up is not cosmetic. getClients() in lib/clients.ts is lazily - * bootstrapped — Tradovate clients, the contract resolver, the reporter and the - * persisted-schedule resume all start inside it — so until something makes an - * HTTP request the process sits idle and none of that happens. + * The warm-up used to be what started trading: getClients() in lib/clients.ts + * was only bootstrapped by the first HTTP request. Builds with + * instrumentation.ts boot it when the server starts, so the warm-up is now a + * readiness check — it waits for each instance to answer, loads the dashboard + * state once, and logs the scheduler and /api/health status. (On an older + * build it still does the bootstrapping, so it stays.) */ import { spawnSync } from 'node:child_process'; import { ensurePythonDeps } from './ensure-python-deps.mjs'; @@ -58,7 +60,7 @@ for (const instance of loadInstances()) { continue; } - log(`warming up ${instance.name} (this is what triggers the client bootstrap)`); + log(`warming up ${instance.name}`); try { await fetch(`${base}/api/state`, { signal: AbortSignal.timeout(60_000) }); } catch (err) { @@ -71,6 +73,27 @@ for (const instance of loadInstances()) { ? `${instance.name}: scheduler resumed — ${status.action} ${status.symbol}` : `${instance.name}: scheduler is stopped`); } catch { /* non-critical */ } + + // One line of client health. Right after boot a client or two may still + // be logging in, so this reports; it doesn't fail the run. Builds from + // before /api/health answer 404 — nothing to say then. + try { + const res = await fetch(`${base}/api/health`, { signal: AbortSignal.timeout(10_000) }); + if (res.status !== 404) { + const health = await res.json(); + if (!res.ok) { + log(`${instance.name}: /api/health failed — ${health.error ?? `HTTP ${res.status}`}`); + } else { + // Firms without a client yet aren't failing to sync — same + // rule as the route's own `ok` (and the updater's line). + const clients = (Array.isArray(health.clients) ? health.clients : []) + .filter((c) => c?.hasClient !== false); + const waiting = clients.filter((c) => !c.ready).map((c) => c.firm); + log(`${instance.name}: ${clients.length - waiting.length}/${clients.length} firm(s) ready` + + (waiting.length > 0 ? ` — not ready yet: ${waiting.join(', ')}` : '')); + } + } + } catch { /* non-critical */ } } if (failed) process.exit(1);