Broker sessions rest while nothing needs them
A firm's client used to stay logged in around the clock, renewing on a fixed timer overnight and at weekends. Now, once a firm has nothing open, nothing pending and nothing left to trade before its next session for a random 15–45 minutes, its client disconnects and the dashboard shows it Resting. It logs back in a random 20–50 minutes before the next session opens, at a check-in every 3–7 Central daytime hours, or at once when trading starts, a manual leg, copy trade, manual order or AutoBuyer run needs it. Token renewals fall at random points (Tradovate 15–45 min before expiry, TopstepX every 45–150 min). Settings → Trading → Rest idle broker sessions (session_rest) turns it off. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Brandon Li
co-authored by
Claude Opus 5.5
Claude Opus 5.5
parent
6dd2528fe2
commit
5eb0ea7760
+306
-2
@@ -14,8 +14,10 @@ import { applyBrokerHttpDefaults, applyBrokerProxy } from './outbound-proxy';
|
||||
import axios from './http-client';
|
||||
import { startMasterLink } from './master-link';
|
||||
import { startUpdaterWatchdog } from './updater-watchdog';
|
||||
import { resumeSchedulerIfPersisted } from './auto-trade';
|
||||
import { getSchedulerStatus, nextSessionOpenFor, resumeSchedulerIfPersisted, sessionDemand, type SessionDemand } from './auto-trade';
|
||||
import { superviseQuotes, SUPERVISE_INTERVAL_MS, type QuoteClient } from './live-quotes';
|
||||
import { getSetting } from './db';
|
||||
import { drawBetween, planRest, REST_LINGER_MAX_MS, REST_LINGER_MIN_MS } from './trading-logic';
|
||||
|
||||
// Use global to persist the client pool across HMR reloads in dev mode.
|
||||
// Keyed by the master's firm id, which is a uuid string.
|
||||
@@ -35,6 +37,10 @@ const g = global as typeof globalThis & {
|
||||
__clientSupervisorTimer?: ReturnType<typeof setInterval>;
|
||||
__clientSupervision?: Map<string, SupervisionEntry>;
|
||||
__quoteSupervisorTimer?: ReturnType<typeof setInterval>;
|
||||
__sessionGovernorTimer?: ReturnType<typeof setInterval>;
|
||||
__sessionRest?: Map<string, RestEntry>;
|
||||
__sessionIdle?: Map<string, IdleEntry>;
|
||||
__sessionWaking?: Map<string, WakeEntry>;
|
||||
};
|
||||
|
||||
function ensureMap(): Map<string, PlatformClient> {
|
||||
@@ -222,6 +228,9 @@ export function getClients(): Map<string, PlatformClient> {
|
||||
// Rebuild any client that has been wedged for too long.
|
||||
startClientSupervisor();
|
||||
|
||||
// Log idle sessions out, and back in before they're needed.
|
||||
startSessionGovernor();
|
||||
|
||||
// Live quotes off one futures client, for the spread every entry
|
||||
// pays and the Auto draw.
|
||||
startQuoteSupervisor();
|
||||
@@ -291,6 +300,13 @@ function triggerContractResolve(): void {
|
||||
try { authenticated = c.health().authenticated; } catch { /* no health: trust the token */ }
|
||||
if (authenticated) { accessToken = token; break; }
|
||||
}
|
||||
if (!accessToken && tradovateAllResting(map)) {
|
||||
// Nothing to resolve with until one logs back in, and every wake
|
||||
// resolves (wakeFirm) — no half hour of retries meanwhile.
|
||||
console.log('[contract-resolver] Tradovate sessions are resting — resolving at the next login');
|
||||
g.__contractResolveRetries = 0;
|
||||
return;
|
||||
}
|
||||
if (!accessToken) {
|
||||
const retries = (g.__contractResolveRetries ?? 0) + 1;
|
||||
if (retries > CONTRACT_RESOLVE_MAX_RETRIES) {
|
||||
@@ -327,6 +343,19 @@ function triggerContractResolve(): void {
|
||||
}
|
||||
}
|
||||
|
||||
/** Every Tradovate client is a resting snapshot (and there is at least one). */
|
||||
function tradovateAllResting(map: Map<string, PlatformClient>): boolean {
|
||||
let resting = 0;
|
||||
for (const [id, c] of map) {
|
||||
let platform: string | undefined;
|
||||
try { platform = c.health().platform; } catch { /* unknown: count it awake */ }
|
||||
if (platform !== 'tradovate') continue;
|
||||
if (!isResting(id)) return false;
|
||||
resting++;
|
||||
}
|
||||
return resting > 0;
|
||||
}
|
||||
|
||||
function scheduleDailyResolve(): void {
|
||||
// Clear any existing timer (HMR safety)
|
||||
if (g.__contractResolverTimer) clearInterval(g.__contractResolverTimer);
|
||||
@@ -407,7 +436,8 @@ function superviseLiveQuotes(): void {
|
||||
const map = ensureMap();
|
||||
const candidates = getTradableFirms().filter(isFuturesFirm).flatMap((firm) => {
|
||||
const client = map.get(firm.id);
|
||||
return client ? [{ firmId: firm.id, firmName: firm.name, platform: firm.platform, client: client as unknown as QuoteClient }] : [];
|
||||
// A resting session holds no token; its feed (if it had one) drops.
|
||||
return client && !isResting(firm.id) ? [{ firmId: firm.id, firmName: firm.name, platform: firm.platform, client: client as unknown as QuoteClient }] : [];
|
||||
});
|
||||
superviseQuotes(candidates, resolveRootsHere())
|
||||
.catch((err) => console.error(`[quotes] supervision failed: ${errorMessage(err)}`));
|
||||
@@ -499,6 +529,12 @@ function superviseClients(): void {
|
||||
entry.unhealthySince = 0;
|
||||
}
|
||||
|
||||
// Logged out on purpose: rebuilding it would wake it.
|
||||
if (isResting(id)) {
|
||||
entry.unhealthySince = 0;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (isClientReady(client, now)) {
|
||||
entry.unhealthySince = 0;
|
||||
continue;
|
||||
@@ -541,3 +577,271 @@ function superviseClients(): void {
|
||||
console.error(`[clients] supervisor tick failed: ${errorMessage(err)}`);
|
||||
}
|
||||
}
|
||||
|
||||
// ── Session rest ─────────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Logs a firm's broker session out while nothing needs it, and back in before
|
||||
* something does.
|
||||
*
|
||||
* A session used to stay up from boot to shutdown: logged in all night and all
|
||||
* weekend, renewing its token on a timer throughout. Now a firm that has been
|
||||
* idle for a random 15–45 minutes — nothing held, nothing pending, nothing
|
||||
* left to trade before its next session (sessionDemand in lib/auto-trade.ts)
|
||||
* — has its client disconnected: the client keeps its last snapshot for the
|
||||
* dashboards and the report, but holds no token and runs no timers. A fresh
|
||||
* client logs in a random 20–50 minutes before the firm's next session opens
|
||||
* (planRest), on the same path a restart takes. A long rest is broken by a
|
||||
* check-in every few hours of the Central daytime, which refreshes balances
|
||||
* and finds new accounts: one that can still trade keeps the session up. A
|
||||
* command that needs the firm — trading started, a manual leg, a copy, an
|
||||
* AutoBuyer run — wakes it at once.
|
||||
*
|
||||
* Trading never sees a resting firm (isSessionAway, in firmClients): it holds
|
||||
* nothing, so it neither trades nor gates, and its lane doesn't wait for it.
|
||||
* A woken one stays out of sight until its first good sync (WAKE_GRACE_MS at
|
||||
* most), so a check-in never holds up the firms still trading.
|
||||
*
|
||||
* Setting session_rest = '0' keeps every session up, as before.
|
||||
*/
|
||||
const GOVERNOR_INTERVAL_MS = 60_000;
|
||||
const WAKE_GRACE_MS = 15 * 60_000;
|
||||
|
||||
interface RestEntry {
|
||||
/** The disconnected client, left in the pool as a snapshot. */
|
||||
client: PlatformClient;
|
||||
since: number;
|
||||
/** When its next session wants it back (null: only a command or a check-in will). */
|
||||
wakeAt: number | null;
|
||||
checkInAt: number | null;
|
||||
/** How long before its next session's open it logs back in (drawn per rest). */
|
||||
leadMs: number;
|
||||
/** The trade date whose accounts were all done, or null: the market was closed. */
|
||||
afterTradeDate: string | null;
|
||||
/** What wakeAt was planned from: the Central day (its open delay may be
|
||||
* drawn later) and whether the scheduler was running. */
|
||||
plannedDay: string;
|
||||
plannedRunning: boolean;
|
||||
}
|
||||
|
||||
interface IdleEntry {
|
||||
client: PlatformClient;
|
||||
since: number;
|
||||
lingerMs: number;
|
||||
}
|
||||
|
||||
interface WakeEntry {
|
||||
client: PlatformClient;
|
||||
since: number;
|
||||
}
|
||||
|
||||
const CENTRAL_DAY = new Intl.DateTimeFormat('en-CA', { timeZone: 'America/Chicago' });
|
||||
const CENTRAL_TIME = new Intl.DateTimeFormat('en-US', {
|
||||
timeZone: 'America/Chicago', weekday: 'short', hour: 'numeric', minute: '2-digit',
|
||||
});
|
||||
|
||||
/** "Mon 6:12 PM CT". */
|
||||
export function fmtCentralTime(ms: number): string {
|
||||
return `${CENTRAL_TIME.format(ms)} CT`;
|
||||
}
|
||||
|
||||
function restMap(): Map<string, RestEntry> {
|
||||
if (!g.__sessionRest) g.__sessionRest = new Map();
|
||||
return g.__sessionRest;
|
||||
}
|
||||
|
||||
function idleMap(): Map<string, IdleEntry> {
|
||||
if (!g.__sessionIdle) g.__sessionIdle = new Map();
|
||||
return g.__sessionIdle;
|
||||
}
|
||||
|
||||
function wakingMap(): Map<string, WakeEntry> {
|
||||
if (!g.__sessionWaking) g.__sessionWaking = new Map();
|
||||
return g.__sessionWaking;
|
||||
}
|
||||
|
||||
export function sessionRestEnabled(): boolean {
|
||||
return getSetting('session_rest') !== '0';
|
||||
}
|
||||
|
||||
/** The firm's session is resting: its client is a disconnected snapshot. */
|
||||
export function isResting(firmId: string): boolean {
|
||||
const entry = g.__sessionRest?.get(firmId);
|
||||
return !!entry && g.__tradovateClients?.get(firmId) === entry.client;
|
||||
}
|
||||
|
||||
/** Logging back in after a rest, and not synced yet. */
|
||||
export function isWaking(firmId: string, now = Date.now()): boolean {
|
||||
const entry = g.__sessionWaking?.get(firmId);
|
||||
if (!entry) return false;
|
||||
if (g.__tradovateClients?.get(firmId) !== entry.client
|
||||
|| isClientReady(entry.client, now)
|
||||
|| now - entry.since >= WAKE_GRACE_MS) {
|
||||
g.__sessionWaking?.delete(firmId);
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/** Out of trading's sight: resting, or not yet back from a rest. */
|
||||
export function isSessionAway(firmId: string, now = Date.now()): boolean {
|
||||
return isResting(firmId) || isWaking(firmId, now);
|
||||
}
|
||||
|
||||
/** A resting firm's rest, for /api/health and the dashboards; null when awake. */
|
||||
export function sessionRestInfo(firmId: string): { since: number; wakeAt: number | null; checkInAt: number | null } | null {
|
||||
if (!isResting(firmId)) return null;
|
||||
const entry = restMap().get(firmId)!;
|
||||
return { since: entry.since, wakeAt: entry.wakeAt, checkInAt: entry.checkInAt };
|
||||
}
|
||||
|
||||
/** When a resting firm logs back in by itself (its wake or its check-in,
|
||||
* whichever comes first); null when awake or resting until a command. */
|
||||
export function restingWakeAt(firmId: string): number | null {
|
||||
const info = sessionRestInfo(firmId);
|
||||
if (!info) return null;
|
||||
const times = [info.wakeAt, info.checkInAt].filter((t): t is number => t !== null);
|
||||
return times.length > 0 ? Math.min(...times) : null;
|
||||
}
|
||||
|
||||
/** Log a resting firm back in now. True if it was resting. */
|
||||
export function wakeFirm(firmId: string, why: string): boolean {
|
||||
const resting = isResting(firmId);
|
||||
restMap().delete(firmId);
|
||||
if (!resting) return false;
|
||||
idleMap().delete(firmId);
|
||||
const firm = getTradableFirms().find((f) => f.id === firmId);
|
||||
if (!firm) return false; // no longer tradable: reconcileClients drops it
|
||||
console.log(`[sessions] ${firm.name}: logging back in — ${why}`);
|
||||
removeClient(firmId);
|
||||
const client = initClient(firm.id, firm.name, firm.platform, firm);
|
||||
wakingMap().set(firmId, { client, since: Date.now() });
|
||||
// A rest can span the midnight contract resolve; catch up once logged in.
|
||||
if (firm.platform === 'tradovate') setTimeout(() => triggerContractResolve(), 15_000);
|
||||
return true;
|
||||
}
|
||||
|
||||
/** Log every resting firm back in now; how many there were. */
|
||||
export function wakeAllFirms(why: string): number {
|
||||
let woken = 0;
|
||||
for (const id of [...restMap().keys()]) {
|
||||
if (wakeFirm(id, why)) woken++;
|
||||
}
|
||||
return woken;
|
||||
}
|
||||
|
||||
function startSessionGovernor(): void {
|
||||
if (g.__sessionGovernorTimer) clearInterval(g.__sessionGovernorTimer);
|
||||
g.__sessionGovernorTimer = setInterval(governSessions, GOVERNOR_INTERVAL_MS);
|
||||
}
|
||||
|
||||
function governSessions(): void {
|
||||
// Runs inside setInterval: nothing may escape.
|
||||
try {
|
||||
const map = ensureMap();
|
||||
const now = Date.now();
|
||||
// Clients replaced or dropped since (a credentials change, a firm gone).
|
||||
for (const [id, entry] of [...restMap()]) if (map.get(id) !== entry.client) restMap().delete(id);
|
||||
for (const [id, entry] of [...idleMap()]) if (map.get(id) !== entry.client) idleMap().delete(id);
|
||||
|
||||
if (!sessionRestEnabled()) {
|
||||
idleMap().clear();
|
||||
if (restMap().size > 0) wakeAllFirms('session rest switched off');
|
||||
return;
|
||||
}
|
||||
|
||||
for (const firm of getTradableFirms()) {
|
||||
const client = map.get(firm.id);
|
||||
if (!client) continue;
|
||||
try {
|
||||
const rest = restMap().get(firm.id);
|
||||
if (rest) governResting(firm, rest, now);
|
||||
else governAwake(firm, client, now);
|
||||
} catch (err) {
|
||||
console.error(`[sessions] ${firm.name}: check failed: ${errorMessage(err)}`);
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
console.error(`[sessions] governor tick failed: ${errorMessage(err)}`);
|
||||
}
|
||||
}
|
||||
|
||||
type GovernedFirm = ReturnType<typeof getTradableFirms>[number];
|
||||
|
||||
function governResting(firm: GovernedFirm, rest: RestEntry, now: number): void {
|
||||
if (rest.wakeAt !== null && now >= rest.wakeAt) {
|
||||
wakeFirm(firm.id, 'its next session opens soon');
|
||||
return;
|
||||
}
|
||||
if (rest.checkInAt !== null && now >= rest.checkInAt) {
|
||||
wakeFirm(firm.id, 'check-in');
|
||||
return;
|
||||
}
|
||||
// Plan again once the Central day turns (its open delay is drawn then)
|
||||
// or the scheduler starts or stops.
|
||||
const running = getSchedulerStatus().running;
|
||||
const day = CENTRAL_DAY.format(now);
|
||||
if (day === rest.plannedDay && running === rest.plannedRunning) return;
|
||||
rest.plannedDay = day;
|
||||
rest.plannedRunning = running;
|
||||
const opensAt = running ? nextSessionOpenFor(firm, now, rest.afterTradeDate) : null;
|
||||
const wakeAt = opensAt === null ? null : Math.max(now, opensAt - rest.leadMs);
|
||||
if (wakeAt === rest.wakeAt) return;
|
||||
rest.wakeAt = wakeAt;
|
||||
if (wakeAt !== null && rest.checkInAt !== null && rest.checkInAt >= wakeAt) rest.checkInAt = null;
|
||||
console.log(`[sessions] ${firm.name}: next login ${wakeAt !== null ? `moved to ${fmtCentralTime(wakeAt)}` : 'when trading starts'}`);
|
||||
}
|
||||
|
||||
function governAwake(firm: GovernedFirm, client: PlatformClient, now: number): void {
|
||||
const demand = sessionDemand(firm, client, now);
|
||||
if (demand.needed) {
|
||||
idleMap().delete(firm.id);
|
||||
return;
|
||||
}
|
||||
const idle = idleMap().get(firm.id);
|
||||
if (!idle) {
|
||||
idleMap().set(firm.id, {
|
||||
client, since: now, lingerMs: Math.round(drawBetween(REST_LINGER_MIN_MS, REST_LINGER_MAX_MS)),
|
||||
});
|
||||
return;
|
||||
}
|
||||
if (now - idle.since < idle.lingerMs || demand.wait) return;
|
||||
|
||||
const opensAt = demand.scheduled ? nextSessionOpenFor(firm, now, demand.afterTradeDate) : null;
|
||||
// Scheduled, with nothing opening in a week: something is off — stay up.
|
||||
if (demand.scheduled && opensAt === null) return;
|
||||
const plan = planRest(now, opensAt);
|
||||
if (!plan) return; // its session opens too soon for a rest to be worth a login
|
||||
idleMap().delete(firm.id);
|
||||
restClient(firm, client, plan, demand, now);
|
||||
}
|
||||
|
||||
function restClient(
|
||||
firm: GovernedFirm,
|
||||
client: PlatformClient,
|
||||
plan: NonNullable<ReturnType<typeof planRest>>,
|
||||
demand: SessionDemand,
|
||||
now: number,
|
||||
): void {
|
||||
try {
|
||||
client.disconnect();
|
||||
} catch (err) {
|
||||
// Half torn down is still not trading; the wake builds a fresh one.
|
||||
console.error(`[sessions] ${firm.name}: disconnect failed: ${errorMessage(err)}`);
|
||||
}
|
||||
restMap().set(firm.id, {
|
||||
client,
|
||||
since: now,
|
||||
wakeAt: plan.wakeAt,
|
||||
checkInAt: plan.checkInAt,
|
||||
leadMs: plan.leadMs,
|
||||
afterTradeDate: demand.afterTradeDate,
|
||||
plannedDay: CENTRAL_DAY.format(now),
|
||||
plannedRunning: demand.scheduled,
|
||||
});
|
||||
const why = demand.afterTradeDate ? `every account is done for ${demand.afterTradeDate}`
|
||||
: demand.scheduled ? 'its market is closed' : 'trading is stopped';
|
||||
const back = plan.wakeAt !== null ? `back at ${fmtCentralTime(plan.wakeAt)}` : 'back when trading starts';
|
||||
const checkIn = plan.checkInAt !== null ? `, check-in ${fmtCentralTime(plan.checkInAt)}` : '';
|
||||
console.log(`[sessions] ${firm.name}: idle (${why}) — logged out; ${back}${checkIn}`);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user