Files
autofirmer-expanded/lib/reporter.ts
T
Brandon LiandClaude Opus 5.5 5ec5abde6e Remove broker session rest
Reverts 062ec07 and 5eb0ea7: every firm's broker session stays logged in
around the clock again, renewing its token on the fixed timer as before.
The tree is back to 6dd2528's. A session_rest row left in settings is
unused.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-21 23:39:53 -05:00

582 lines
29 KiB
TypeScript
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
/**
* Reporter module — pushes aggregated state to the master dashboard.
*
* Every 30 seconds, collects firm-level stats + scheduler state and sends them
* to the configured master_dashboard_url. Silently skips if not configured.
*
* Two transports, in that order of preference: the control WebSocket in
* `master-link.ts` when it is up, and the original POST /api/report when it is
* not. The socket already exists for commands, so reusing it saves a request
* per tick — but a master on an older build has no socket to offer, and the
* POST keeps those working unchanged.
*/
import { getSetting, loadEndedCycleStart } from './db';
import { pushReport } from './master-link';
import { getBuildVersion } from './version';
import { tradeMixFor } from './trade-mix';
import { updaterStatus } from './updater-status';
import { getClients } from './clients';
import { getAccountTypeById, getKeptRuleIdsForFirm, getTradableFirms } from './firm-catalog';
import { getAccountConfig } from './account-type-resolver';
import { getSchedulerStatus } from './auto-trade';
import { botStopsActive, combineCmeSessions, computeDayPlan, currentTradeDateCT, cycleCompletion, endedCycleCompletion, LEG_COMPLETION_EPSILON, projectPerfectPath, resolveEffectiveConfig } from './trading-logic';
import type { AccountTypeStats, FirmStats, InstrumentStats, SymbolStats } from './master-protocol';
import { accountNextLeg, accountLegShape } from './leg-estimate';
import { sizingHoldHours } from './person-size';
import { instrumentStatsFor, isFuturesFirm } from './instruments';
import { accountStatusFields, brokerFacts, emptyFirmStats, firmStatusFields, historyFresh, keptRuleIdsField } from './platform-client';
function collectFirmStats() {
// Only firms this machine has credentials for. A firm it cannot log into
// has nothing to say about, and reporting it as zero accounts would fill
// the master's table with rows for firms it does not run.
const firms = getTradableFirms();
const clients = getClients();
// One raw trade date for the whole report: the combined views below roll
// it forward, the completion check needs it as is.
const rawToday = currentTradeDateCT();
// projectPerfectPath's `loops` test: a type whose completed cycle keeps
// going pays out (a funded account), one that stops just passes.
const loops = (cfg: { payoutable?: boolean; withdrawalStages: unknown[] }): boolean =>
cfg.payoutable === true || cfg.withdrawalStages.length > 0;
const completionKind = (cfg: Parameters<typeof loops>[0], drain: boolean): 'pass' | 'payout' | 'drain' =>
(drain ? 'drain' : loops(cfg) ? 'payout' : 'pass');
return firms.map((f) => {
const client = clients.get(f.id);
// Absent (null) while this machine's catalog can't vouch for them.
const keptRuleIds = getKeptRuleIdsForFirm(f.id);
// accounts: [] only once synced (see emptyFirmStats).
if (!client || client.accountList.length === 0) return emptyFirmStats(f, client, keptRuleIds);
let totalAccounts = 0;
let accountsTraded = 0;
let inTrade = 0;
let targetHitCount = 0;
let partialCount = 0;
let lossCount = 0;
let personalDllCount = 0;
let completedCount = 0;
// The firm's whole broker list, dead accounts included — the master
// tells an account the firm erased from one still listed, and spots
// a newly arrived funded account.
const accounts: NonNullable<FirmStats['accounts']> = [];
// Per account type, so the master can say what the accounts still to
// trade actually are. Keyed by account type id; unassigned accounts
// share the null bucket — they inflate "left to trade" indefinitely,
// which is precisely the thing worth being able to see. Always the
// live type's id, label and size: accounts on a kept rule set count
// in their live type's slice (and in its keptRules).
const byType = new Map<string | null, AccountTypeStats>();
const bucket = (cfg: { id: string; label: string; accountSize: number } | null): AccountTypeStats => {
const key = cfg?.id ?? null;
let entry = byType.get(key);
if (!entry) {
const live = key ? getAccountTypeById(key) ?? cfg : null;
entry = {
accountTypeId: key,
label: live?.label ?? 'Unassigned',
accountSize: live?.accountSize ?? 0,
totalAccounts: 0,
accountsTraded: 0,
inTrade: 0,
targetHit: 0,
partial: 0,
loss: 0,
};
byType.set(key, entry);
}
return entry;
};
for (const acc of client.accountList) {
const reportedCash = client.accountCashBalances[acc.id];
const cash = reportedCash ?? { amount: 0, realizedPnL: 0 };
const autoLiqThreshold = client.autoLiqThresholds[acc.id] ?? 0;
const resolved = getAccountConfig(acc.id);
const cfg = resolved.accountType;
// undefined until the account's fund history has loaded. The master
// clears a completed cycle once the withdrawal it was waiting for
// shows up here, so an account that hasn't loaded reports no count
// at all rather than a zero: undercounting only leaves the entry
// standing, while a wrong zero would clear one early.
const loadedFundTxns = client.fundTransactions?.[acc.id];
const fundTxns = loadedFundTxns ?? [];
accounts.push({
name: acc.name,
size: cfg?.accountSize ?? 0,
payoutable: cfg ? loops(cfg) : null,
...(loadedFundTxns ? { withdrawals: loadedFundTxns.filter((f) => f.amount < 0).length } : {}),
// For the master's account ledger. Data only: nothing here
// trades on it, and the totals below keep their own test.
...statusFieldsOrUnknown(() => accountStatusFields({
cash: reportedCash?.amount,
floor: autoLiqThreshold,
active: acc.active,
historyReady: client.peekHistoryReady ? client.peekHistoryReady(acc.id) : client.historyReady(acc.id),
accountTypeId: cfg?.id,
fundTransactions: loadedFundTxns,
...brokerFacts(f.platform, acc, client.brokerAccountMeta?.(acc.id)),
})),
});
// 0 = no floor; negative floors are reconstructed TopstepX MLLs.
const isDead = autoLiqThreshold !== 0 && cash.amount <= autoLiqThreshold;
if (isDead) continue;
const slice = bucket(cfg);
totalAccounts++;
slice.totalAccounts++;
if (resolved.keptRules) {
const keptId = resolved.keptRules.id;
const kept = (slice.keptRules ??= []).find((k) => k.id === keptId);
if (kept) kept.accounts++;
else slice.keptRules.push({ id: keptId, accounts: 1 });
}
// Same combined-day view as the trade path: for a combining firm,
// "today" spans the holiday stretch. Computed for every live
// account — including ones mid-trade — so the days-traded
// histogram below covers the whole slice. Accounts in the red or
// DLL-locked stay OUT of the histogram: the payout calendar
// projects from "on the winning path", which they are not.
// (Dead accounts never reach here — the auto-liq check above.)
const combined = cfg ? combineCmeSessions(
cfg.combineCmeHolidays, client.dailyPnL[acc.id] ?? [], rawToday, cash.realizedPnL,
) : null;
// Stage-resolved config, shared by the payout state and the
// day-outcome check below — a drain stage swaps in its own
// min-day and drops the profit target.
const effective = cfg ? resolveEffectiveConfig(
cfg.profitTarget, cfg.consistency, cfg.minTradingDays,
cfg.targetSameEquity, cfg.withdrawalStages,
client.priorProfit?.[acc.id] ?? 0, fundTxns, cfg.minDayPnL,
) : null;
// Days traded, for the master's per-day pass counts — the same
// number as the local dashboard's Days Traded column
// (app/api/state): only days reaching minDayPnL when it is set,
// today included once it books.
const dayOn = combined
? effective && effective.minDayPnL > 0
? combined.dailyPnL.filter((d) => d.pnl >= effective.minDayPnL).length
: combined.daysTraded
: null;
const dayEntry = dayOn === null ? null : ((slice.dayStats ??= {})[dayOn] ??= { total: 0, targetHit: 0 });
if (dayEntry) dayEntry.total++;
if (cfg && combined && acc.active && cash.amount - cfg.accountSize >= 0) {
const hist = (slice.daysHist ??= {});
hist[combined.daysTraded] = (hist[combined.daysTraded] ?? 0) + 1;
}
// Exact payout state for every live account — negative ones
// included, since the master sims their real required profit.
// client.dailyPnL is already the active-cycle slice, so a
// withdrawal resets days/maxDay/sum here automatically.
if (cfg && combined && effective) {
const r2 = (n: number) => Math.round(n * 100) / 100;
const equityProfit = r2(cash.amount - cfg.accountSize);
const qualifying = effective.minDayPnL === 0
? combined.dailyPnL
: combined.dailyPnL.filter((d) => d.pnl >= effective.minDayPnL);
const sum = r2(combined.dailyPnL.reduce((s, d) => s + d.pnl, 0));
// End-of-day equity-profit high-water mark for this cycle,
// floored at the $0 start: walk the P&L from the cycle base.
let run = r2(equityProfit - sum);
let highWater = Math.max(0, run);
for (const d of combined.dailyPnL) {
run = r2(run + d.pnl);
if (run > highWater) highWater = run;
}
const traded = combined.dailyPnL.filter((d) => d.pnl !== 0);
// The account's future payout chain on the perfect path —
// dates, dynamic amounts (withdrawal % of balance, capped,
// × profit split) and cumulative fair-walk reach probability.
// The master renders these directly instead of running its
// own Monte-Carlo for this account.
let futurePayouts: NonNullable<AccountTypeStats['payoutStates']>[number]['futurePayouts'];
try {
futurePayouts = projectPerfectPath({
accountId: acc.id,
cfg,
cycleDailyPnL: client.dailyPnL[acc.id] ?? [],
fundTransactions: fundTxns,
balance: cash.amount,
realizedPnL: cash.realizedPnL,
payout: {
withdrawalPct: cfg.withdrawalPct,
payoutCap: cfg.payoutCap,
profitSplit: cfg.profitSplit,
// Same default chain the master's calendar always
// used: configured trailing drawdown, else the
// max-loss size, else $2,000.
drawdown: (cfg.trailingDrawdown ?? 0) > 0
? cfg.trailingDrawdown!
: Math.abs(cfg.maxLoss) > 0 ? Math.abs(cfg.maxLoss) : 2000,
startHigh: highWater,
// Tradovate reports the account's REAL liquidation
// floor — anchor the trail there for exact room.
startThreshold: autoLiqThreshold !== 0
? r2(autoLiqThreshold - cfg.accountSize)
: undefined,
},
})
.filter((p) => p.withdrawal !== undefined || p.payoutReady)
.slice(0, 6)
.map((p) => ({
date: p.date,
gross: p.gross ?? 0,
net: p.net ?? 0,
prob: Math.round(p.prob * 10000) / 10000,
stage: p.stage,
...(p.drain ? { drain: true } : {}),
}));
if (futurePayouts.length === 0) futurePayouts = undefined;
} catch (err) {
console.error(`[reporter] payout projection failed for account ${acc.id}`, err);
}
(slice.payoutStates ??= []).push({
days: qualifying.length,
profit: equityProfit,
maxDay: qualifying.length > 0 ? Math.max(...qualifying.map((d) => d.pnl)) : 0,
sum,
high: highWater,
withdrawals: fundTxns.filter((f) => f.amount < 0).length,
target: effective.profitTarget,
consistency: effective.consistency,
minDays: effective.minTradingDays,
lastDay: traded.length > 0 ? traded.reduce((a, b) => (a.date > b.date ? a : b)).date : undefined,
...(futurePayouts ? { futurePayouts } : {}),
});
}
// Only once the account's history has loaded — right after a
// restart the missing rows and withdrawals would read as a fresh
// stage-1 cycle.
const historyLoaded = client.dailyPnL[acc.id] !== undefined
&& loadedFundTxns !== undefined;
// Catch-up for the cycle a fund transaction just ended: if it stood
// complete, name it, since its final day may never have been
// reported (a guard held it back, or this instance was offline).
// Named only — it isn't today's outcome, so no counter moves. Ahead
// of the in-trade check: the new cycle trading doesn't hide it.
if (cfg && effective && historyLoaded && historyFresh(client, acc)) {
const ended = endedCycleCompletion({
accountId: acc.id,
rawTradeDate: rawToday,
fullDailyPnL: client.fullDailyPnL[acc.id] ?? [],
cycleDailyPnL: client.dailyPnL[acc.id],
fundTransactions: fundTxns,
lastFundDate: client.lastFundDates?.[acc.id] ?? null,
endedCycleStart: loadEndedCycleStart(acc.id),
rawRealizedPnL: cash.realizedPnL,
equityProfit: cash.amount - cfg.accountSize,
cfg,
});
if (ended) {
(slice.completedAccounts ??= []).push({
name: acc.name,
date: ended.date,
stage: ended.stage,
kind: completionKind(cfg, ended.drain),
cycle: ended.cycle,
});
}
}
if (client.positions[acc.id]) {
inTrade++;
accountsTraded++;
slice.inTrade++;
slice.accountsTraded++;
continue;
}
// Did today finish the cycle? Checked ahead of the lockout branch:
// firms lock a passed account (Tradovate liquidate-only, TopstepX
// canTrade=false), and a pass is no loss. A 'today' only on
// freshly fetched rows, since the master keeps it.
if (cfg && combined && effective && historyLoaded) {
const completion = cycleCompletion({
accountId: acc.id,
rawTradeDate: rawToday,
cycleDailyPnL: client.dailyPnL[acc.id],
rawRealizedPnL: cash.realizedPnL,
combineCmeHolidays: cfg.combineCmeHolidays,
tradesEveryDay: cfg.tradesEveryDay === true,
effective,
equityProfit: cash.amount - cfg.accountSize,
splitEnabled: cfg.splitEnabled,
});
if (completion === 'today' && historyFresh(client, acc)) {
accountsTraded++;
slice.accountsTraded++;
targetHitCount++;
slice.targetHit = (slice.targetHit ?? 0) + 1;
completedCount++;
slice.completed = (slice.completed ?? 0) + 1;
if (dayEntry) {
dayEntry.targetHit++;
dayEntry.completed = (dayEntry.completed ?? 0) + 1;
}
(slice.completedAccounts ??= []).push({
name: acc.name,
date: combined.tradeDate,
stage: 1 + fundTxns.filter((f) => f.amount < 0).length,
kind: completionKind(cfg, effective.drain),
// The fund transaction that started this cycle — the
// catch-up names the same cycle by the same key.
cycle: client.lastFundDates?.[acc.id] ?? '',
});
continue;
}
// Passed on an earlier day and locked since: nothing traded
// today, so it reads as untraded — like a finished account
// that's still active — not as a DLL loss every day after.
if (completion === 'before' && !acc.active && combined.realizedPnL === 0) continue;
}
// DLL lockout: Tradovate flips the account to liquidate-only, so
// its day is over at a loss even when the booked realizedPnL
// hasn't caught up yet (or the account has no matching config).
if (!acc.active) {
accountsTraded++;
slice.accountsTraded++;
lossCount++;
slice.loss = (slice.loss ?? 0) + 1;
continue;
}
// Check if target was hit today — on the stage-resolved config, so
// a later stage's (or drain's) day counts the way it was traded.
if (cfg && combined && effective) {
const equityProfit = cash.amount - cfg.accountSize;
const plan = computeDayPlan(
acc.id, combined.tradeDate, effective.profitTarget, effective.consistency,
combined.dailyPnL, effective.minDayPnL, effective.minTradingDays, equityProfit, combined.realizedPnL,
cfg.splitEnabled, effective.drain
);
const targetHit = !plan ? false :
((plan.dayTarget === 0 && Math.abs(combined.realizedPnL) > 0 && combined.daysTraded <= effective.minTradingDays) ||
(combined.realizedPnL > 0 && plan.remainingToday <= LEG_COMPLETION_EPSILON));
if (targetHit || combined.realizedPnL !== 0) {
accountsTraded++;
slice.accountsTraded++;
// A grind-day fill (dayTarget 0) can flag targetHit on a
// negative day — a loss is still a loss, so red wins.
// A red day on an account the bot puts stops on (the
// obfuscation stops, or a hard daily loss limit) is its own
// personal DLL, booked on purpose — not a genuine loss. (A
// firm DLL lockout took the !acc.active branch above and
// stays a loss.)
if (botStopsActive(cfg) && combined.realizedPnL <= -25) {
personalDllCount++;
slice.personalDll = (slice.personalDll ?? 0) + 1;
} else if (combined.realizedPnL < 0) {
lossCount++;
slice.loss = (slice.loss ?? 0) + 1;
} else if (targetHit) {
targetHitCount++;
slice.targetHit = (slice.targetHit ?? 0) + 1;
if (dayEntry) dayEntry.targetHit++;
} else {
partialCount++;
slice.partial = (slice.partial ?? 0) + 1;
}
}
}
}
// Biggest shortfall first — the account types with the most left to do
// are the ones worth reading, and Unassigned sinks to the bottom only
// when it is genuinely small.
const accountTypes = [...byType.values()].sort((a, b) =>
(b.totalAccounts - b.accountsTraded) - (a.totalAccounts - a.accountsTraded)
|| a.label.localeCompare(b.label));
return {
firm: f.name, totalAccounts, accountsTraded, inTrade,
targetHit: targetHitCount, partial: partialCount, loss: lossCount,
personalDll: personalDllCount, completed: completedCount,
// Unknown until the firm has synced, not empty.
...(client.syncComplete ? { accounts } : {}),
accountTypes,
...symbolStatsFor(f, client, rawToday),
...instrumentStatsBlock(f, client, rawToday),
mix: tradeMixFor(f, client),
...firmStatusFields(f.id, client.syncComplete, client.lastSyncAt),
...keptRuleIdsField(keptRuleIds),
};
});
}
/** An account's status fields, or unknown if working them out throws — the
* ledger's data must never cost the rest of the report. Logged once. */
function statusFieldsOrUnknown(fields: () => ReturnType<typeof accountStatusFields>): ReturnType<typeof accountStatusFields> {
try {
return fields();
} catch (err) {
if (!_g.__accountStatusFailed) {
_g.__accountStatusFailed = true;
console.error('[reporter] account status failed — reporting it as unknown', err);
}
return { status: 'unknown', reason: 'status_error' };
}
}
/** The live accounts of a firm's largest prefix group first — the group a
* batch fires — each with an account type. */
// eslint-disable-next-line @typescript-eslint/no-explicit-any
function leadCandidates(client: any): { id: number; name: string }[] {
const groups = new Map<string, { id: number; name: string }[]>();
for (const acc of client.accountList as { id: number; name: string; active: boolean }[]) {
const cfg = getAccountConfig(acc.id).accountType;
const floor = client.autoLiqThresholds?.[acc.id] ?? 0;
const cash = client.accountCashBalances?.[acc.id];
if (!cfg || !acc.active || !cash || (floor !== 0 && cash.amount <= floor)) continue;
groups.set(cfg.prefix, [...(groups.get(cfg.prefix) ?? []), acc]);
}
return [...groups.values()].sort((a, b) => b.length - a.length).flat();
}
/**
* A futures firm's instruments as the Auto draw sees them, for the master's
* Firms page: entry cost, movement and chance for each, with holds sized for
* the next leg of the first live account (largest prefix group) that has one.
*/
function instrumentStatsBlock(
f: { id: string; name: string; platform: string; instruments?: string[]; bannedSymbols?: string[] },
// eslint-disable-next-line @typescript-eslint/no-explicit-any
client: any,
tradeDate: string,
): { instrumentStats?: InstrumentStats } {
if (!isFuturesFirm(f)) return {};
let lead: { account: string; leg: number; stopDollars: number; maxPositionSize: number } | null = null;
for (const acc of leadCandidates(client)) {
const shape = accountLegShape(client, acc, tradeDate);
if (shape) { lead = { ...shape, account: acc.name }; break; }
}
return { instrumentStats: instrumentStatsFor(f, lead) };
}
/**
* A DXtrade firm's next leg on each symbol-map candidate, for the master's
* Firms page: one live account from the firm's largest prefix group — the
* group a batch fires — with something left to trade today. Nothing for
* other platforms, or when no account has a leg to size.
*/
function symbolStatsFor(
f: { name: string; commissionPct?: number; targetHoldHours?: number; financingPctPerDay?: number },
// eslint-disable-next-line @typescript-eslint/no-explicit-any
client: any,
tradeDate: string,
): { symbolStats?: SymbolStats } {
if (!(client.lastCandidates?.length > 0)) return {};
const groups = new Map<string, { id: number; name: string }[]>();
for (const acc of client.accountList as { id: number; name: string; active: boolean }[]) {
const cfg = getAccountConfig(acc.id).accountType;
const floor = client.autoLiqThresholds?.[acc.id] ?? 0;
const cash = client.accountCashBalances?.[acc.id];
if (!cfg || !acc.active || !cash || (floor !== 0 && cash.amount <= floor)) continue;
groups.set(cfg.prefix, [...(groups.get(cfg.prefix) ?? []), acc]);
}
for (const group of [...groups.values()].sort((a, b) => b.length - a.length)) {
for (const acc of group) {
const next = accountNextLeg(client, acc, f, tradeDate);
if (!next || next.candidates.length === 0) continue;
return {
symbolStats: {
account: acc.name,
leg: next.leg,
targetHoldHours: Math.round(sizingHoldHours(f, tradeDate) * 10) / 10,
lastStand: next.lastStand,
at: new Date().toISOString(),
symbols: next.candidates.map((c) => ({
symbol: c.symbol, notional: c.notional, multiplier: c.multiplier, maxMultiplier: c.maxMultiplier,
holdHours: c.holdHours, cost: c.cost, winProbability: c.winProbability, winProbabilityNoFees: c.winProbabilityNoFees,
volatilityPerDay: c.volatilityPerDay, spreadFraction: c.spreadFraction,
})),
},
};
}
}
return {};
}
// HMR-safe global to avoid duplicate intervals
const _g = globalThis as typeof globalThis & {
__reporterInterval?: ReturnType<typeof setInterval>;
/** The last report was too big for the socket (logged on each change). */
__reportOverSocketLimit?: boolean;
/** accountStatusFields has thrown once (logged the first time only). */
__accountStatusFailed?: boolean;
};
/** The master's socket closes on a frame over 1,000,000 bytes (maxPayload in
* its server.mjs), and the link would resend it every tick — a report this
* big goes by POST instead. */
const SOCKET_REPORT_MAX_CHARS = 900_000;
/**
* Collect and send one report. Exported so the control link can fire an
* off-cycle one the moment a command has changed something — waiting up to 30s
* to see whether your own button worked is the wrong feedback loop.
*/
export async function sendReport(): Promise<void> {
const url = getSetting('master_dashboard_url');
const instanceName = getSetting('instance_name');
if (!url || !instanceName) return;
try {
const firms = collectFirmStats();
const scheduler = getSchedulerStatus();
const overSocketLimit = JSON.stringify(firms).length > SOCKET_REPORT_MAX_CHARS;
if (overSocketLimit !== (_g.__reportOverSocketLimit ?? false)) {
_g.__reportOverSocketLimit = overSocketLimit;
console.warn(overSocketLimit
? `[reporter] report over ${SOCKET_REPORT_MAX_CHARS} chars — sending it by POST, not the socket`
: '[reporter] report fits the socket again');
}
if (!overSocketLimit && pushReport(firms, scheduler)) return;
await fetch(new URL('/api/report', url), {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
instance: instanceName,
// Same identity the socket hello carries, so a rename resolves
// to the same person whichever transport the report takes.
id: getSetting('instance_id') || undefined,
firms,
scheduler,
version: getBuildVersion() ?? undefined,
updater: updaterStatus(),
}),
});
} catch {
// Master might be down — silently continue
}
}
export function startReporter() {
if (_g.__reporterInterval) {
clearInterval(_g.__reporterInterval);
}
_g.__reporterInterval = setInterval(sendReport, 30_000);
// Fire first report after a short delay to let clients sync
setTimeout(sendReport, 10_000);
console.log('[reporter] started — reporting every 30s');
}