diff --git a/lib/db.ts b/lib/db.ts index 763184b..01ad444 100644 --- a/lib/db.ts +++ b/lib/db.ts @@ -328,6 +328,22 @@ export function loadFundTransactions(accountId: number): { date: string; amount: return db.prepare('SELECT date, amount FROM fund_transactions WHERE account_id = ? ORDER BY date').all(accountId) as { date: string; amount: number }[]; } +/** Deletes one day's fund transaction and sets account meta keys back in the + * same transaction (null deletes the key), so no reader sees the row gone + * with the cycle it started still in place. TopstepX's reconcile uses it to + * take back a day whose bookings netted to fee noise. */ +export function dropFundTransaction(accountId: number, date: string, meta: { [key: string]: string | null }): void { + const del = db.prepare('DELETE FROM account_meta WHERE account_id = ? AND key = ?'); + const txn = db.transaction(() => { + db.prepare('DELETE FROM fund_transactions WHERE account_id = ? AND date = ?').run(accountId, date); + for (const [key, value] of Object.entries(meta)) { + if (value === null) del.run(accountId, key); + else saveAccountMeta(accountId, key, value); + } + }); + txn(); +} + // ── Account Meta ───────────────────────────────────────────────────────────── db.exec(` diff --git a/lib/topstepx-class.ts b/lib/topstepx-class.ts index 0cb3b4d..0b28b5a 100644 --- a/lib/topstepx-class.ts +++ b/lib/topstepx-class.ts @@ -24,10 +24,10 @@ import axios from 'axios'; import * as signalR from '@microsoft/signalr'; -import { currentTradeDateCT } from './trading-logic'; +import { currentTradeDateCT, FUND_EVENT_MIN, fundReconcileStep, type FundSighting } from './trading-logic'; import { getCachedContract } from './contract-resolver'; import { signalrWebSocket } from './outbound-proxy'; -import { saveDailyPnL, loadDailyPnL, loadAccountMeta, saveAccountMeta, loadFundTransactions, saveFundTransactions, saveEndedCycleStart } from './db'; +import { saveDailyPnL, loadDailyPnL, loadAccountMeta, saveAccountMeta, loadFundTransactions, saveFundTransactions, saveEndedCycleStart, dropFundTransaction } from './db'; import { getAccountConfig } from './account-type-resolver'; import { topstepxProductCode, topstepxRoot, topstepCommission, TOPSTEP_COMMISSION_FALLBACK } from './topstep-fees'; import type { PlatformClient, PlatformAccount, PlatformContract, ClientHealth } from './platform-client'; @@ -73,10 +73,10 @@ const TOKEN_LIFETIME_MS = 24 * 60 * 60 * 1000; * minutes after its caller gave up on it. Longer than the idle timeout, so * it only bites where that one can't. */ const REST_DEADLINE_MS = 90_000; -/** Balance the books don't explain past this is a fund event (reconcileFunds - * books it); under it, fee noise and rounding. The smallest real withdrawal - * in the drain flow is well above it. */ -const FUND_EVENT_MIN = 200; +/** account_meta: what the day's first fund booking replaced (its date, and + * last_fund_date and ended_cycle_start as they stood), so a reversal that + * nets the day to fee noise can put them back. */ +const FUND_DAY_BEFORE_META = 'fund_day_before'; /** Rejects if `promise` hasn't settled within `ms`. The underlying operation * keeps running (callers discard whatever it belonged to); Promise.race @@ -252,6 +252,9 @@ export class TopstepXClient implements PlatformClient { /** Accounts unbookedBalanceChange has warned about, until their books * settle — one line per episode, not one per tick. */ private unbookedWarned = new Set(); + /** Unexplained balances reconcileFunds has seen but not booked yet: one + * is booked only once a later check still sees it (fundReconcileStep). */ + private fundSightings = new Map(); /** See PlatformClient: when the last good pass started reading. */ public lastSyncStartedAt = 0; /** Accounts whose last order ended with its outcome unknown: each sync @@ -1039,51 +1042,108 @@ export class TopstepXClient implements PlatformClient { * and starts a new cycle; the very first check also books any historical * net withdrawal in one lump (fine — the last stage repeats). * - * FUND_EVENT_MIN keeps fee noise and rounding out. Skipped while a - * position is open (fills in flight) or while the trade history is - * known-stale; until it books, unbookedBalanceChange holds the account. + * FUND_EVENT_MIN keeps fee noise and rounding out. An amount is booked + * only once a later check still sees it: for one pass, a fill landing + * between the balance read and the trade read looks just like a fund + * event (fundReconcileStep). Skipped while a position is open (fills in + * flight) or while the trade history is known-stale; until it books, + * unbookedBalanceChange holds the account. */ /** Books a withdrawal/deposit the trades can't explain. True once it has * actually checked the books against the balance (false when it had to - * skip: a position open, no history or balance yet, no account type). */ + * skip: a position open, no history or balance yet, no account type; or + * when what it saw waits on a later check to be booked). */ private reconcileFunds(account: PlatformAccount): boolean { - if (this.positions[account.id]) return false; - if (this.lastFetchErrors[account.name]) return false; - const delta = this.unbookedBalance(account.id); - if (delta === null) return false; + const delta = this.positions[account.id] || this.lastFetchErrors[account.name] + ? null + : this.unbookedBalance(account.id); + if (delta === null) { + // A sighting is confirmed by the next check, not one after a skip. + this.fundSightings.delete(account.id); + return false; + } const cash = this.accountCashBalances[account.id]!; const startBal = getAccountConfig(account.id).accountType!.accountSize; const today = currentTradeDateCT(); const txns = this.fundTransactions[account.id] ?? []; + const existingToday = txns.find((f) => f.date === today)?.amount ?? null; + const before = this.fundDayBefore(account.id, today); + const step = fundReconcileStep({ + delta, + sighting: this.fundSightings.get(account.id) ?? null, + now: Date.now(), + existingToday, + undoable: before !== null, + }); + const books = Math.round((cash.amount - startBal - delta) * 100) / 100; + if (step.kind === 'unconfirmed') { + this.fundSightings.set(account.id, step.sighting); + if (step.fresh) console.log(`[topstepx] ${account.name}: balance ${cash.amount} vs books ${books} — ${delta} unexplained; booked if a later pass still sees it`); + return false; + } + this.fundSightings.delete(account.id); // The first check on this instance (or since a wipe) only has the // fetch window to explain the balance with: a lump it books is unseen // P&L and past payouts, not the end of a cycle this instance watched. const firstCheck = loadAccountMeta(account.id, 'funds_reconciled') === null; if (firstCheck) saveAccountMeta(account.id, 'funds_reconciled', today); - if (Math.abs(delta) < FUND_EVENT_MIN) return true; + if (step.kind === 'settled') return true; - console.log(`[topstepx] ${account.name}: balance ${cash.amount} vs books ${Math.round((cash.amount - startBal - delta) * 100) / 100} — recording ${delta < 0 ? 'withdrawal' : 'deposit'} of ${delta}`); - // A new cycle starts today: note where the one it ends began, by - // applyCycleSlice's rule for that date's own row, for the reporter's - // catch-up. A second event today doesn't start another cycle. - const prevFundDate = this.lastFundDates[account.id] ?? null; - if (!firstCheck && prevFundDate !== today) { - const prevAmt = prevFundDate ? (txns.find((f) => f.date === prevFundDate)?.amount ?? null) : null; - saveEndedCycleStart(account.id, { fundDate: prevFundDate ?? '', includesDay: !(prevAmt !== null && prevAmt < 0) }); + if (step.kind === 'drop') { + // A transient booked today and then reversed: the cycle it started + // never happened, so the markers go back to what it replaced. + console.log(`[topstepx] ${account.name}: balance ${cash.amount} vs books ${books} — today's fund bookings net to ${step.net}, a transient and its reversal; taking today's row back`); + dropFundTransaction(account.id, today, { + last_fund_date: before!.lastFundDate, + ended_cycle_start: before!.endedCycleStart, + [FUND_DAY_BEFORE_META]: null, + }); + this.lastFundDates[account.id] = before!.lastFundDate; + } else { + console.log(`[topstepx] ${account.name}: balance ${cash.amount} vs books ${books} — recording ${delta < 0 ? 'withdrawal' : 'deposit'} of ${delta}`); + // The day's first booking: keep what it replaces. + if (existingToday === null) { + saveAccountMeta(account.id, FUND_DAY_BEFORE_META, JSON.stringify({ + date: today, + lastFundDate: loadAccountMeta(account.id, 'last_fund_date'), + endedCycleStart: loadAccountMeta(account.id, 'ended_cycle_start'), + })); + } + // A new cycle starts today: note where the one it ends began, by + // applyCycleSlice's rule for that date's own row, for the reporter's + // catch-up. A second event today doesn't start another cycle. + const prevFundDate = this.lastFundDates[account.id] ?? null; + if (!firstCheck && prevFundDate !== today) { + const prevAmt = prevFundDate ? (txns.find((f) => f.date === prevFundDate)?.amount ?? null) : null; + saveEndedCycleStart(account.id, { fundDate: prevFundDate ?? '', includesDay: !(prevAmt !== null && prevAmt < 0) }); + } + // Same-day events merge into one row (the table keys on + // account+date): step.amount is today's delta on top of whatever + // is already booked for today. + saveFundTransactions(account.id, account.name, [{ date: today, amount: step.amount }]); + saveAccountMeta(account.id, 'last_fund_date', today); + this.lastFundDates[account.id] = today; } - // Same-day events merge into one row (the table keys on account+date), - // so add today's delta onto whatever is already booked for today. - const existingToday = txns.find((f) => f.date === today)?.amount ?? 0; - saveFundTransactions(account.id, account.name, [ - { date: today, amount: Math.round((existingToday + delta) * 100) / 100 }, - ]); - saveAccountMeta(account.id, 'last_fund_date', today); this.fundTransactions[account.id] = loadFundTransactions(account.id); - this.lastFundDates[account.id] = today; this.applyCycleSlice(account.id); return true; } + /** What today's first fund booking replaced — null when today has none + * that reconcileFunds made (none yet, or a row it didn't book). */ + private fundDayBefore(accountId: number, today: string): { lastFundDate: string | null; endedCycleStart: string | null } | null { + const raw = loadAccountMeta(accountId, FUND_DAY_BEFORE_META); + if (raw === null) return null; + try { + const saved = JSON.parse(raw) as { date?: unknown; lastFundDate?: unknown; endedCycleStart?: unknown }; + const strOrNull = (v: unknown) => v === null || typeof v === 'string'; + if (saved.date !== today || !strOrNull(saved.lastFundDate) || !strOrNull(saved.endedCycleStart)) return null; + return { lastFundDate: saved.lastFundDate as string | null, endedCycleStart: saved.endedCycleStart as string | null }; + } catch { + return null; + } + } + /** * Balance the books don't explain: balance less the starting balance * ($0 for Express, the funded size otherwise), all trade P&L (today's off @@ -1107,7 +1167,7 @@ export class TopstepXClient implements PlatformClient { /** * See PlatformClient. The hub moves the balance the moment a payout - * lands, but reconcileFunds books it only in the next sync — and not + * lands, but reconcileFunds books it only a sync or two later — and not * while a position is open. Planned on in between, the payout reads as * lost profit: on 2026-10-06 two Express accounts that had finished * their cycle re-planned as stage 1, fired a full 3×NQ leg 28 seconds @@ -1421,6 +1481,7 @@ export class TopstepXClient implements PlatformClient { public forgetHistory(accountId: number, accountName: string): void { this.historyLoadedIds.delete(accountId); this.reconcilePendingIds.delete(accountId); + this.fundSightings.delete(accountId); delete this.dailyPnL[accountId]; delete this.fullDailyPnL[accountId]; delete this.daysTraded[accountId]; diff --git a/lib/trading-logic.ts b/lib/trading-logic.ts index d613cbc..4322d7d 100644 --- a/lib/trading-logic.ts +++ b/lib/trading-logic.ts @@ -1107,6 +1107,74 @@ export function unbookedPayoutDrop( return drop >= PAYOUT_HOLD_MIN ? drop : null; } +// ── fund events from the balance (TopstepX) ────────────────────────────────── + +/** Balance the books don't explain past this is a fund event (TopstepX's + * reconcileFunds books it); under it, fee noise and rounding. The smallest + * real withdrawal in the drain flow is well above it. */ +export const FUND_EVENT_MIN = 200; +/** A sighting is booked only once a later pass, at least this much later, + * sees the same amount... */ +export const FUND_CONFIRM_MS = 30_000; +/** ...to within this (fee noise between the two passes). */ +export const FUND_CONFIRM_TOLERANCE = 5; + +/** An unexplained balance seen, not yet booked: the amount and when it was + * first seen at about that amount. */ +export interface FundSighting { delta: number; since: number } + +export type FundStep = + /** Under FUND_EVENT_MIN: nothing to book. */ + | { kind: 'settled' } + /** Not yet seen twice far enough apart. `sighting` is what to keep for + * the next pass; `fresh` when it is new (or the amount moved). */ + | { kind: 'unconfirmed'; sighting: FundSighting; fresh: boolean } + /** Book today's row as `amount` (today's existing row merged in). */ + | { kind: 'book'; amount: number } + /** Today's bookings net under FUND_EVENT_MIN: a transient and its + * reversal. Take today's row back and leave `net` unbooked. */ + | { kind: 'drop'; net: number }; + +/** + * One fresh pass's decision on the balance the books don't explain. + * + * The pass reads the balance first and the positions and trades after it, so + * a fill landing in between shows in the trades but not yet in the balance + * (or, the other way, in the balance before its trade row). For one pass the + * books are off by the whole fill. On 2026-10-05 two Express accounts booked a + * $3,001.16 "deposit" that way, and the next pass booked the $3,002.16 + * "withdrawal" that undid it. The two merged into a −$1.00 row, the + * commission drift of the day. A negative row is a withdrawal: it started a + * new cycle, dropped the day's −$3,003.32 out of it and moved both accounts + * to their drain stage. A real fund event stays put, so an amount is booked + * only when a pass at least FUND_CONFIRM_MS after the first sighting still + * sees it. + * + * `undoable`: today's row was booked by this reconcile alone, with what it + * replaced saved (the first booking of a day saves it), so it can be taken + * back whole. A transient that outlasts the confirmation and is then + * reversed nets under FUND_EVENT_MIN, and the day is dropped, not left as a + * tiny row. + */ +export function fundReconcileStep(opts: { + delta: number; + sighting: FundSighting | null; + now: number; + /** Today's fund row, null when there is none. */ + existingToday: number | null; + undoable: boolean; +}): FundStep { + const { delta, sighting, now, existingToday } = opts; + if (Math.abs(delta) < FUND_EVENT_MIN) return { kind: 'settled' }; + if (sighting === null || Math.abs(sighting.delta - delta) > FUND_CONFIRM_TOLERANCE) { + return { kind: 'unconfirmed', sighting: { delta, since: now }, fresh: true }; + } + if (now - sighting.since < FUND_CONFIRM_MS) return { kind: 'unconfirmed', sighting, fresh: false }; + const amount = round2((existingToday ?? 0) + delta); + if (existingToday !== null && opts.undoable && Math.abs(amount) < FUND_EVENT_MIN) return { kind: 'drop', net: amount }; + return { kind: 'book', amount }; +} + // ── fees ───────────────────────────────────────────────────────────────────── /** An entry's fee when the balance update hasn't shown it yet — at least one diff --git a/scripts/check-trading-logic.mjs b/scripts/check-trading-logic.mjs index f614a89..9b122ae 100644 --- a/scripts/check-trading-logic.mjs +++ b/scripts/check-trading-logic.mjs @@ -771,3 +771,114 @@ test('tick reasons fold their amounts, so like reasons count together', () => { assert.equal(foldReason("waiting for the broker to roll the account's day"), "waiting for the broker to roll the account's day"); assert.equal(foldReason('leg cap reached (5 today)'), 'leg cap reached (5 today)'); }); + +// ── fund events from the balance (TopstepX reconcileFunds) ────────────────── + +import { fundReconcileStep, FUND_CONFIRM_MS } from '../lib/trading-logic.ts'; + +/** reconcileFunds' bookkeeping for one trade date, pass by pass: `balance` + * and `trades` (all trade P&L, today's included) as each pass read them. + * `existing` is a row already on the day that reconcileFunds didn't book. */ +function reconcileDay(passes, existing = null) { + let row = existing; + let undoable = false; // the day's first booking saves what it replaces + let sighting = null; + const steps = []; + for (const { at, balance, trades } of passes) { + const delta = Math.round((balance - trades - (row ?? 0)) * 100) / 100; + const step = fundReconcileStep({ delta, sighting, now: at * 1000, existingToday: row, undoable }); + steps.push(step); + if (step.kind === 'unconfirmed') { sighting = step.sighting; continue; } + sighting = null; + if (step.kind === 'book') { if (row === null) undoable = true; row = step.amount; } + if (step.kind === 'drop') { row = null; undoable = false; } + } + return { row, kinds: steps.map((s) => s.kind), steps }; +} + +test('fund reconcile: a balance read before a fill the trades already show books nothing', () => { + // 2026-10-05, EXPRESS-V2-CT-DLL-39029-23275136: the pass read the balance + // (−2.16), then the trades with the −3,001.16 stop-out in them. One pass + // later the balance had it (−3,004.32, $1.00 of commission off-book). + // Booked as it came, that was +3,001.16, then −3,002.16, merged to −1.00. + const express = reconcileDay([ + { at: 0, balance: -2.16, trades: -3003.32 }, + { at: 60, balance: -3004.32, trades: -3003.32 }, + ]); + assert.deepEqual(express.kinds, ['unconfirmed', 'settled']); + assert.equal(express.steps[0].sighting.delta, 3001.16); + assert.equal(express.row, null); + + // 2026-10-08, 150KTC-SKU-V2-DLL-39029-99745612 (balance less the $150K + // start): +2,474.01 then −2,492.01, which left a −18.00 row. + const combine = reconcileDay([ + { at: 0, balance: 2413.43, trades: -60.58 }, + { at: 60, balance: -78.58, trades: -60.58 }, + ]); + assert.deepEqual(combine.kinds, ['unconfirmed', 'settled']); + assert.equal(combine.row, null); +}); + +test('fund reconcile: a real withdrawal books once a later pass still sees it', () => { + // 2026-10-14, EXPRESS-V2-CT-DLL-39029-43788281: a $4,620 payout. + const payout = reconcileDay([ + { at: 0, balance: 4601.42, trades: 9221.42 }, + { at: 10, balance: 4601.42, trades: 9221.42 }, // too soon to confirm + { at: 60, balance: 4601.42, trades: 9221.42 }, + ]); + assert.deepEqual(payout.kinds, ['unconfirmed', 'unconfirmed', 'book']); + assert.equal(payout.steps[1].fresh, false); + assert.equal(payout.steps[1].sighting.since, 0); // the clock runs from the first sighting + assert.equal(payout.row, -4620); + + // Fee noise between the passes still confirms; the later amount books. + assert.equal(reconcileDay([ + { at: 0, balance: 4601.42, trades: 9221.42 }, + { at: FUND_CONFIRM_MS / 1000, balance: 4598.42, trades: 9221.42 }, + ]).row, -4623); + + // An amount that moves starts over: the second sighting is the new one. + const moved = reconcileDay([ + { at: 0, balance: 4601.42, trades: 9221.42 }, + { at: 60, balance: 4901.42, trades: 9221.42 }, + { at: 120, balance: 4901.42, trades: 9221.42 }, + ]); + assert.deepEqual(moved.kinds, ['unconfirmed', 'unconfirmed', 'book']); + assert.equal(moved.steps[1].fresh, true); + assert.equal(moved.row, -4320); +}); + +test('fund reconcile: a transient that outlasts the confirmation is taken back whole', () => { + // The balance lags for two passes, so +3,001.16 books; when it catches up, + // the reversal nets the day to −1.00 and the day is dropped, not left as + // a −1.00 "withdrawal" that splits the cycle. + const lagging = reconcileDay([ + { at: 0, balance: -2.16, trades: -3003.32 }, + { at: 60, balance: -2.16, trades: -3003.32 }, + { at: 120, balance: -3004.32, trades: -3003.32 }, + { at: 180, balance: -3004.32, trades: -3003.32 }, + ]); + assert.deepEqual(lagging.kinds, ['unconfirmed', 'book', 'unconfirmed', 'drop']); + assert.equal(lagging.steps[1].amount, 3001.16); + assert.equal(lagging.steps[3].net, -1); + assert.equal(lagging.row, null); + + // A payout landing with the reversal is real: the day keeps its row. + const withPayout = reconcileDay([ + { at: 0, balance: -2.16, trades: -3003.32 }, + { at: 60, balance: -2.16, trades: -3003.32 }, + { at: 120, balance: -7624.32, trades: -3003.32 }, + { at: 180, balance: -7624.32, trades: -3003.32 }, + ]); + assert.deepEqual(withPayout.kinds, ['unconfirmed', 'book', 'unconfirmed', 'book']); + assert.equal(withPayout.row, -4621); + + // Only a day reconcileFunds booked itself is taken back: a row it found + // there (an import) merges as before. + const imported = reconcileDay([ + { at: 0, balance: -1, trades: 0 }, + { at: 60, balance: -1, trades: 0 }, + ], -4620); + assert.deepEqual(imported.kinds, ['unconfirmed', 'book']); + assert.equal(imported.row, -1); +});