From ed56e306d15205ff2f020b43912fd95b77d28f24 Mon Sep 17 00:00:00 2001 From: Brandon Li Date: Mon, 19 Oct 2026 06:08:23 -0500 Subject: [PATCH] Persist one browser user agent for Axios and WebSockets --- README.md | 19 +- app/api/debug/route.ts | 2 +- app/api/firms/[id]/debug-contracts/route.ts | 2 +- app/api/settings/route.ts | 3 + app/settings/page.tsx | 13 ++ eslint.config.mjs | 23 +++ lib/clients.ts | 7 +- lib/contract-resolver.ts | 6 +- lib/db.ts | 12 ++ lib/dxtrade-class.ts | 2 +- lib/http-client.ts | 36 ++++ lib/hyperliquid-feed.ts | 4 +- lib/instruments.ts | 3 +- lib/live-quotes.ts | 5 +- lib/master-link.ts | 3 +- lib/outbound-proxy.ts | 76 ++++---- lib/request-user-agent.ts | 16 ++ lib/topstepx-class.ts | 7 +- lib/tradovate-class.ts | 2 +- package.json | 2 +- scripts/check-request-user-agent.mjs | 201 ++++++++++++++++++++ 21 files changed, 380 insertions(+), 64 deletions(-) create mode 100644 lib/http-client.ts create mode 100644 lib/request-user-agent.ts create mode 100644 scripts/check-request-user-agent.mjs diff --git a/README.md b/README.md index 6313542..8c5b5cf 100644 --- a/README.md +++ b/README.md @@ -184,7 +184,24 @@ It is read-only unless `DX_CONFIRM=yes DX_ACCOUNT=` are set too, which adds one minimum-size live round trip. `npm run check` runs the pure-logic checks (fees, stops, sizing, trading -hours, the DXtrade selector, the futures draw and instrument lists). +hours, the DXtrade selector, the futures draw and instrument lists), plus local +HTTP/WebSocket checks against a temporary database. + +### Outbound user agent + +On first server load, each instance randomly selects a desktop Chrome, +Firefox, or Edge user agent and saves it as `user_agent` in its settings. +Settings → Network shows the selected value. It stays the same across requests, +WebSocket reconnects, page reloads, and server restarts, and travels with backups. +An existing saved value is reused; an absent or blank value is generated automatically. + +All server Axios calls use `lib/http-client.ts`; import Axios from +`@/lib/http-client` when adding requests. Its defaults and interceptor also apply +to `axios.create()` clients, so per-request headers cannot accidentally replace +the saved user agent. ESLint rejects direct application imports of the Axios package. +Use `brokerWebSocket` or `signalrWebSocket` from `lib/outbound-proxy.ts` for broker +sockets, and `directWebSocket` for control sockets. All share the same user agent; +the broker proxy still applies only to broker traffic. --- diff --git a/app/api/debug/route.ts b/app/api/debug/route.ts index b76a6b4..8944cf5 100644 --- a/app/api/debug/route.ts +++ b/app/api/debug/route.ts @@ -3,7 +3,7 @@ import { getClients, resetClients } from '@/lib/clients'; import { getTradableFirms } from '@/lib/firm-catalog'; import { getAccountConfig } from '@/lib/account-type-resolver'; import { combineCmeSessions, computeDayPlan, resolveEffectiveConfig, currentTradeDateCT } from '@/lib/trading-logic'; -import axios from 'axios'; +import axios from '@/lib/http-client'; /** * Per-account snapshot of everything computeDayPlan sees, plus the book-balance diff --git a/app/api/firms/[id]/debug-contracts/route.ts b/app/api/firms/[id]/debug-contracts/route.ts index 2bf8bea..4e0c6b6 100644 --- a/app/api/firms/[id]/debug-contracts/route.ts +++ b/app/api/firms/[id]/debug-contracts/route.ts @@ -1,6 +1,6 @@ import { NextRequest, NextResponse } from 'next/server'; import { getClients } from '@/lib/clients'; -import axios from 'axios'; +import axios from '@/lib/http-client'; export async function GET( req: NextRequest, diff --git a/app/api/settings/route.ts b/app/api/settings/route.ts index 52998d1..ca8d50b 100644 --- a/app/api/settings/route.ts +++ b/app/api/settings/route.ts @@ -1,6 +1,7 @@ import { NextRequest, NextResponse } from 'next/server'; import { getSetting, setSetting } from '@/lib/db'; import { scheduleBackupPush } from '@/lib/backup-sync'; +import { getRequestUserAgent } from '@/lib/request-user-agent'; const VALID_KEYS = ['max_concurrent_accounts', 'tick_interval_seconds', 'master_dashboard_url', 'instance_name', 'trading_hours', 'twocaptcha_api_key', 'twocaptcha_proxy', 'master_backup_token', 'proxy_url'] as const; type SettingKey = typeof VALID_KEYS[number]; @@ -13,6 +14,8 @@ export async function GET() { // Read-only on purpose (not in VALID_KEYS): the master issues this uuid, // and editing it by hand would detach the instance from its person. result.instance_id = getSetting('instance_id'); + // Generated once per instance. Callers cannot rotate an active identity. + result.user_agent = getRequestUserAgent(); result.backup_last_push_at = getSetting('backup_last_push_at'); result.backup_last_error = getSetting('backup_last_error'); return NextResponse.json(result); diff --git a/app/settings/page.tsx b/app/settings/page.tsx index 83b5f42..bd8e2a3 100644 --- a/app/settings/page.tsx +++ b/app/settings/page.tsx @@ -47,6 +47,7 @@ interface AppSettings { trading_hours: string | null; master_backup_token: string | null; proxy_url: string | null; + user_agent: string | null; backup_last_push_at: string | null; backup_last_error: string | null; } @@ -77,6 +78,7 @@ export default function SettingsPage() { const [saved, setSaved] = useState(false); const [backupToken, setBackupToken] = useState(''); const [proxyUrl, setProxyUrl] = useState(''); + const [userAgent, setUserAgent] = useState(''); const [backupStatus, setBackupStatus] = useState<{ at: string | null; error: string | null }>({ at: null, error: null }); const [restoreList, setRestoreList] = useState(null); const [restoreBusy, setRestoreBusy] = useState(false); @@ -130,6 +132,7 @@ export default function SettingsPage() { if (s.proxy_url != null) { setProxyUrl(s.proxy_url); } + setUserAgent(s.user_agent ?? ''); setBackupStatus({ at: s.backup_last_push_at, error: s.backup_last_error || null }); }); @@ -440,6 +443,16 @@ export default function SettingsPage() { className="w-72 rounded-lg border border-slate-200 bg-slate-50 px-3 py-1.5 text-sm font-mono text-slate-800 focus:outline-none focus:ring-2 focus:ring-blue-500" /> +
+

User agent

+

+ Chrome, Firefox, or Edge is randomly selected on first load and saved. + All server Axios requests and WebSocket connections reuse it, including after restarts. +

+

+ {userAgent || 'Loading…'} +

+
{/* ── Instruments ── */} diff --git a/eslint.config.mjs b/eslint.config.mjs index 05e726d..0801666 100644 --- a/eslint.config.mjs +++ b/eslint.config.mjs @@ -5,6 +5,29 @@ import nextTs from "eslint-config-next/typescript"; const eslintConfig = defineConfig([ ...nextVitals, ...nextTs, + { + files: ['**/*.{js,mjs,cjs,ts,tsx}'], + ignores: ['lib/http-client.ts', 'lib/outbound-proxy.ts'], + rules: { + 'no-restricted-imports': ['error', { + patterns: [{ + group: ['axios', 'axios/*'], + message: 'Import Axios from @/lib/http-client so requests keep the saved user agent, proxy and timeout.', + }], + }], + 'no-restricted-modules': ['error', { + paths: [{ + name: 'axios', + message: 'Use @/lib/http-client for the configured Axios client.', + }], + patterns: ['axios/*'], + }], + 'no-restricted-syntax': ['error', { + selector: 'ImportExpression[source.value=/^axios(\\/|$)/]', + message: 'Import @/lib/http-client for the configured Axios client.', + }], + }, + }, // Override default ignores of eslint-config-next. globalIgnores([ // Default ignores of eslint-config-next: diff --git a/lib/clients.ts b/lib/clients.ts index b900db2..e08cc52 100644 --- a/lib/clients.ts +++ b/lib/clients.ts @@ -11,6 +11,7 @@ import { resolveContracts } from './contract-resolver'; import { startReporter, sendReport } from './reporter'; import { startBackupSync } from './backup-sync'; import { applyBrokerHttpDefaults, applyBrokerProxy } from './outbound-proxy'; +import axios from './http-client'; import { startMasterLink } from './master-link'; import { startUpdaterWatchdog } from './updater-watchdog'; import { resumeSchedulerIfPersisted } from './auto-trade'; @@ -185,10 +186,10 @@ export function getClients(): Map { try { // Before anything dials out: a timeout on every broker HTTP call, // and the operator's proxy (if any) under the broker clients. - // Master traffic stays direct by design. (outbound-proxy applies + // Master traffic stays direct by design. (http-client applies // both when it loads, too; these are idempotent.) - applyBrokerHttpDefaults(); - applyBrokerProxy(); + applyBrokerHttpDefaults(axios.defaults); + applyBrokerProxy(axios.defaults); // Pull the catalog before anything else — without it there are no // firms to start clients for. Subsequent changes re-reconcile and diff --git a/lib/contract-resolver.ts b/lib/contract-resolver.ts index e82e259..29804d2 100644 --- a/lib/contract-resolver.ts +++ b/lib/contract-resolver.ts @@ -12,7 +12,7 @@ * Results are cached and refreshed periodically (default: every 30 minutes). */ -import axios from 'axios'; +import axios from './http-client'; import { errorMessage } from './platform-client'; interface ContractInfo { @@ -82,7 +82,7 @@ async function getContinuousExchange(product: string): Promise { try { const res = await axios.get( `https://query1.finance.yahoo.com/v8/finance/chart/${encodeURIComponent(product + '=F')}`, - { params: { range: '1d', interval: '1d' }, headers: { 'User-Agent': 'Mozilla/5.0' } } + { params: { range: '1d', interval: '1d' } } ); const meta = res.data?.chart?.result?.[0]?.meta; if (!meta) return null; @@ -117,7 +117,7 @@ async function findHighestVolumeContract( try { const res = await axios.get( `https://query1.finance.yahoo.com/v8/finance/chart/${encodeURIComponent(ticker)}`, - { params: { range: '1d', interval: '1d' }, headers: { 'User-Agent': 'Mozilla/5.0' } } + { params: { range: '1d', interval: '1d' } } ); const result = res.data?.chart?.result?.[0]; const meta = result?.meta; diff --git a/lib/db.ts b/lib/db.ts index 60dc78a..2187469 100644 --- a/lib/db.ts +++ b/lib/db.ts @@ -389,6 +389,18 @@ export function setSetting(key: string, value: string): void { db.prepare(`INSERT INTO settings (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value`).run(key, value); } +/** Seed a generated setting once, including across simultaneous server workers. + * Acquire the write lock before reading so they cannot choose different values. */ +export function getOrCreateSetting(key: string, create: () => string): string { + return db.transaction(() => { + const saved = getSetting(key); + if (saved?.trim()) return saved; + const value = create(); + setSetting(key, value); + return value; + }).immediate(); +} + // ── Instruments ────────────────────────────────────────────────────────────── const SYMBOLS = ['NQ','MNQ','ES','MES','YM','MYM','RTY','M2K','GC','MGC','SI','CL','MCL','NG','ZB','ZN','ZF','6E','6J','6B']; diff --git a/lib/dxtrade-class.ts b/lib/dxtrade-class.ts index ec32bee..5e72a44 100644 --- a/lib/dxtrade-class.ts +++ b/lib/dxtrade-class.ts @@ -31,7 +31,7 @@ * account shows as accountStatus ≠ FULL_TRADING → active=false. */ -import axios from 'axios'; +import axios from './http-client'; import { currentTradeDateCT, FIRM_RESET_UTC_HOUR, DEFAULT_TARGET_HOLD_HOURS, rankForTarget } from './trading-logic'; import { saveDailyPnL, loadDailyPnL, loadAccountMeta, saveAccountMeta, loadFundTransactions, saveFundTransactions, diff --git a/lib/http-client.ts b/lib/http-client.ts new file mode 100644 index 0000000..8734060 --- /dev/null +++ b/lib/http-client.ts @@ -0,0 +1,36 @@ +/** All application Axios imports belong here. New calls and axios.create() + * clients inherit the instance's browser identity, proxy and timeout. */ +import axios, { type AxiosInstance } from 'axios'; +import { applyBrokerHttpDefaults, applyBrokerProxy } from './outbound-proxy'; +import { getRequestUserAgent } from './request-user-agent'; + +const g = globalThis as typeof globalThis & { __configuredHttpClients?: WeakSet }; +const configured = g.__configuredHttpClients ??= new WeakSet(); + +function configure(client: T): T { + if (configured.has(client)) return client; + applyBrokerHttpDefaults(client.defaults); + // Builds import route modules but must not choose an instance's identity. + if (process.env.NEXT_PHASE !== 'phase-production-build') { + client.defaults.headers.common['User-Agent'] = getRequestUserAgent(); + try { + applyBrokerProxy(client.defaults); + } catch (err) { + console.error(`[proxy] could not apply proxy_url at load: ${err instanceof Error ? err.message : String(err)}`); + } + } + client.interceptors.request.use((config) => { + // Force replacement even for differently cased headers or `false`. + // This also replaces a stale per-call UA without losing auth headers. + config.headers.set('User-Agent', getRequestUserAgent(), true); + return config; + }, undefined, { synchronous: true }); + + // Axios copies defaults into child clients, but not interceptors. + const create = client.create.bind(client); + client.create = (options) => configure(create(options)); + configured.add(client); + return client; +} + +export default configure(axios); diff --git a/lib/hyperliquid-feed.ts b/lib/hyperliquid-feed.ts index c292321..c4c4686 100644 --- a/lib/hyperliquid-feed.ts +++ b/lib/hyperliquid-feed.ts @@ -12,8 +12,6 @@ * No key, no account: POST https://api.hyperliquid.xyz/info. */ -import axios from 'axios'; - const INFO_URL = 'https://api.hyperliquid.xyz/info'; const TIMEOUT_MS = 10_000; const META_TTL_MS = 60 * 60_000; @@ -21,6 +19,8 @@ const META_TTL_MS = 60 * 60_000; const DEXES = ['', 'xyz']; async function info(body: Record): Promise { + // Keep the pure quote helpers usable without opening the instance database. + const { default: axios } = await import('./http-client'); const res = await axios.post(INFO_URL, body, { timeout: TIMEOUT_MS, signal: AbortSignal.timeout(TIMEOUT_MS + 5_000), diff --git a/lib/instruments.ts b/lib/instruments.ts index 839b709..311ef86 100644 --- a/lib/instruments.ts +++ b/lib/instruments.ts @@ -19,7 +19,7 @@ * entry path asks spreadBlock before sending. */ -import axios from 'axios'; +import axios from './http-client'; import { getInstruments, getSetting } from './db'; import { getTradableFirms, getTradingSettings } from './firm-catalog'; import { entrySpread, quoteSource } from './live-quotes'; @@ -98,7 +98,6 @@ async function fetchMarket(root: string): Promise { try { const res = await axios.get(`https://query1.finance.yahoo.com/v8/finance/chart/${encodeURIComponent(spec.yahoo)}`, { params: { interval: '60m', range: '7d' }, - headers: { 'User-Agent': 'Mozilla/5.0' }, timeout: FETCH_TIMEOUT_MS, signal: AbortSignal.timeout(FETCH_TIMEOUT_MS + 5_000), }); diff --git a/lib/live-quotes.ts b/lib/live-quotes.ts index 68a49d1..3366ee2 100644 --- a/lib/live-quotes.ts +++ b/lib/live-quotes.ts @@ -106,14 +106,13 @@ class TopstepxFeed implements Feed { constructor(readonly firmId: string, readonly firmName: string, token: () => string | null) { // Like the user hub: one websocket, through the operator's proxy. - const proxiedWs = signalrWebSocket(); + const configuredWs = signalrWebSocket(); this.hub = new signalR.HubConnectionBuilder() .withUrl(MARKET_HUB, { skipNegotiation: true, transport: signalR.HttpTransportType.WebSockets, accessTokenFactory: () => token() ?? '', - // eslint-disable-next-line @typescript-eslint/no-explicit-any - ...(proxiedWs ? { WebSocket: proxiedWs as any } : {}), + ...{ WebSocket: configuredWs }, }) .withAutomaticReconnect() .configureLogging(signalR.LogLevel.Warning) diff --git a/lib/master-link.ts b/lib/master-link.ts index ab1215a..e29ce41 100644 --- a/lib/master-link.ts +++ b/lib/master-link.ts @@ -25,6 +25,7 @@ import { pushBackup } from './backup-sync'; import { getBuildVersion } from './version'; import { updaterStatus } from './updater-status'; import { runUpdateNow } from './updater-watchdog'; +import { directWebSocket } from './outbound-proxy'; import type { Command, InstanceCapabilities, @@ -239,7 +240,7 @@ function connect() { let socket: WebSocket; try { - socket = new WebSocket(url); + socket = directWebSocket(url); } catch (err: any) { scheduleReconnect(err?.message ?? 'could not open socket'); return; diff --git a/lib/outbound-proxy.ts b/lib/outbound-proxy.ts index 0e1dd2f..3f3f6c0 100644 --- a/lib/outbound-proxy.ts +++ b/lib/outbound-proxy.ts @@ -21,20 +21,20 @@ * process default is the scoped choice. It covers axios only; websockets and * the SignalR hub keep their own liveness handling in the clients. * - * Both defaults are applied when this module loads. Every broker client - * imports it, and that matters: each server bundle (instrumentation.ts, the - * route handlers) can carry its own copy of axios, and a copy that never had - * the defaults applied would dial the brokers direct, with no timeout. + * lib/http-client applies these defaults to every Axios client it exports, + * including clients created with axios.create(), in every server bundle. */ -import axios from 'axios'; -import { ProxyAgent, type Dispatcher } from 'undici'; +import type { AxiosDefaults } from 'axios'; +import { ProxyAgent, WebSocket as UndiciWebSocket, type Dispatcher } from 'undici'; +import NodeWebSocket from 'ws'; import { socksDispatcher } from 'fetch-socks'; import { HttpProxyAgent } from 'http-proxy-agent'; import { HttpsProxyAgent } from 'https-proxy-agent'; import { SocksProxyAgent } from 'socks-proxy-agent'; import type { Agent } from 'node:http'; import { getSetting } from './db'; +import { getRequestUserAgent } from './request-user-agent'; const SCHEMES = ['http:', 'https:', 'socks5:', 'socks5h:'] as const; @@ -120,18 +120,18 @@ export function getProxyUrl(): string | null { /** Puts the proxy under every axios call. All axios usage in this app is * broker/quote traffic (TopstepX, Tradovate REST, contract resolver), so the * process-wide default is the scoped choice, not a global one. Idempotent; - * runs when this module loads and again first in the getClients() bootstrap. + * runs when the shared HTTP client loads and in the getClients() bootstrap. * * "Already applied" is judged on the axios defaults themselves, not a * process-wide flag: the proxy state is shared through globalThis, but each * bundle's copy of axios has its own defaults, and a global flag set by the * first copy would leave every other one dialling direct. */ -export function applyBrokerProxy(): void { +export function applyBrokerProxy(defaults: AxiosDefaults): void { const s = state(); - if (!s.url || axios.defaults.httpsAgent === s.httpsAgent) return; - axios.defaults.proxy = false; // agents own the routing; ignore env HTTP(S)_PROXY handling - axios.defaults.httpAgent = s.httpAgent!; - axios.defaults.httpsAgent = s.httpsAgent!; + if (!s.url || defaults.httpsAgent === s.httpsAgent) return; + defaults.proxy = false; // agents own the routing; ignore env HTTP(S)_PROXY handling + defaults.httpAgent = s.httpAgent!; + defaults.httpsAgent = s.httpsAgent!; } /** Idle limit for one broker HTTP request. axios applies it as a socket @@ -147,53 +147,49 @@ export function applyBrokerProxy(): void { export const BROKER_HTTP_TIMEOUT_MS = 60_000; /** Gives axios a timeout unless one is already set (axios's own default, 0, - * means wait forever). Idempotent; runs when this module loads and at the + * means wait forever). Idempotent; runs when the shared HTTP client loads and at the * top of the getClients() bootstrap. Timeouts surface as ECONNABORTED, which * the Tradovate report fetch already retries as transient. */ -export function applyBrokerHttpDefaults(): void { - if (!axios.defaults.timeout) axios.defaults.timeout = BROKER_HTTP_TIMEOUT_MS; +export function applyBrokerHttpDefaults(defaults: AxiosDefaults): void { + if (!defaults.timeout) defaults.timeout = BROKER_HTTP_TIMEOUT_MS; } -// At load, so no broker request from this bundle can go out before them. The -// proxy is skipped during `next build`: the build imports route modules to -// collect page data, and it dials nothing that would need one. Nothing may -// throw out of here: a module that fails to load takes every route that -// imports it (the whole trading side) down with it. -applyBrokerHttpDefaults(); -try { - if (process.env.NEXT_PHASE !== 'phase-production-build') applyBrokerProxy(); -} catch (err) { - console.error(`[proxy] could not apply proxy_url at load: ${err instanceof Error ? err.message : String(err)}`); +/** Direct control-channel socket: same browser identity, no broker proxy. */ +export function directWebSocket(wsUrl: string): WebSocket { + return new UndiciWebSocket(wsUrl, { + headers: { 'User-Agent': getRequestUserAgent() }, + }) as unknown as WebSocket; } -/** A websocket to a BROKER endpoint: the built-in (undici) WebSocket, dialled - * through the proxy when one is set. The master link keeps using plain - * `new WebSocket(...)` — do not use this for it. */ +/** Broker socket with the same browser identity and optional broker proxy. + * Use the installed Undici: Node's built-in WebSocket may ignore headers. */ export function brokerWebSocket(wsUrl: string): WebSocket { const { dispatcher } = state(); - if (!dispatcher) return new WebSocket(wsUrl); - // The runtime accepts { dispatcher } fine; only the DOM lib type doesn't. - return new (WebSocket as unknown as new (u: string, o: object) => WebSocket)(wsUrl, { dispatcher }); + return new UndiciWebSocket(wsUrl, { + ...(dispatcher ? { dispatcher } : {}), + headers: { 'User-Agent': getRequestUserAgent() }, + }) as unknown as WebSocket; } -/** A `ws`-package constructor with the proxy agent baked in, for SignalR's - * `withUrl({ WebSocket })` option. undefined = no proxy, let SignalR use its - * own require('ws'). */ +/** SignalR always uses this constructor, even without a proxy, because its + * own transport supplies a library User-Agent that we must replace. */ export function signalrWebSocket(): unknown { const s = state(); - if (!s.httpsAgent) return undefined; - const agent = s.httpsAgent; - // eslint-disable-next-line @typescript-eslint/no-require-imports - const WS = require('ws') as new (address: string, protocols?: unknown, options?: object) => unknown; - return class ProxiedWebSocket extends (WS as new (...args: never[]) => object) { + return class ConfiguredWebSocket extends NodeWebSocket { constructor(address: string, protocols?: unknown, options?: Record) { // ws also allows (address, options) — normalize before merging. if (protocols && typeof protocols === 'object' && !Array.isArray(protocols)) { options = protocols as Record; protocols = undefined; } - super(...[address, protocols, { ...(options ?? {}), agent }] as never[]); + const headers = { ...(options?.headers as Record | undefined) }; + for (const name of Object.keys(headers)) { + if (name.toLowerCase() === 'user-agent') delete headers[name]; + } + headers['User-Agent'] = getRequestUserAgent(); + const agent = address.startsWith('ws:') ? s.httpAgent : s.httpsAgent; + super(address, protocols, { ...(options ?? {}), ...(agent ? { agent } : {}), headers }); } }; } diff --git a/lib/request-user-agent.ts b/lib/request-user-agent.ts new file mode 100644 index 0000000..366afc0 --- /dev/null +++ b/lib/request-user-agent.ts @@ -0,0 +1,16 @@ +import { randomInt } from 'node:crypto'; +import { getOrCreateSetting } from './db'; + +// Desktop Windows browser profiles. Choose a browser once per instance, not +// once per request, so HTTP calls and socket reconnects keep the same identity. +const USER_AGENTS = [ + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/144.0.0.0 Safari/537.36', + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:147.0) Gecko/20100101 Firefox/147.0', + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/144.0.0.0 Safari/537.36 Edg/144.0.0.0', +] as const; + +const g = globalThis as typeof globalThis & { __requestUserAgent?: string }; + +export function getRequestUserAgent(): string { + return g.__requestUserAgent ??= getOrCreateSetting('user_agent', () => USER_AGENTS[randomInt(USER_AGENTS.length)]); +} diff --git a/lib/topstepx-class.ts b/lib/topstepx-class.ts index 0b28b5a..8fc4e23 100644 --- a/lib/topstepx-class.ts +++ b/lib/topstepx-class.ts @@ -22,7 +22,7 @@ * all zeros; a blown account shows up as canTrade=false → active=false. */ -import axios from 'axios'; +import axios from './http-client'; import * as signalR from '@microsoft/signalr'; import { currentTradeDateCT, FUND_EVENT_MIN, fundReconcileStep, type FundSighting } from './trading-logic'; import { getCachedContract } from './contract-resolver'; @@ -676,14 +676,13 @@ export class TopstepXClient implements PlatformClient { // everything the hub does, just slower). // With skipNegotiation the hub is a single websocket to a broker // endpoint, so it follows the operator's proxy like the REST calls do. - const proxiedWs = signalrWebSocket(); + const configuredWs = signalrWebSocket(); const hub = new signalR.HubConnectionBuilder() .withUrl(USER_HUB, { skipNegotiation: true, transport: signalR.HttpTransportType.WebSockets, accessTokenFactory: () => this.token ?? '', - // eslint-disable-next-line @typescript-eslint/no-explicit-any - ...(proxiedWs ? { WebSocket: proxiedWs as any } : {}), + ...{ WebSocket: configuredWs }, }) .withAutomaticReconnect() .configureLogging(signalR.LogLevel.Warning) diff --git a/lib/tradovate-class.ts b/lib/tradovate-class.ts index c1c2fe2..a6a21c3 100644 --- a/lib/tradovate-class.ts +++ b/lib/tradovate-class.ts @@ -1,6 +1,6 @@ // Create a class with a constructor that takes a username and password and sets the credentials -import axios from 'axios'; +import axios from './http-client'; import type { AccountItem, AuthLoginResponse, Contract } from './tradovate-helpers'; import { computeSec, randomUUIDV4 } from './tradovate-helpers'; import { currentTradeDateCT } from './trading-logic'; diff --git a/package.json b/package.json index aaef155..dedf19a 100644 --- a/package.json +++ b/package.json @@ -7,7 +7,7 @@ "build": "next build", "start": "next start", "lint": "eslint", - "check": "node --experimental-strip-types --disable-warning=MODULE_TYPELESS_PACKAGE_JSON --test scripts/check-trading-logic.mjs scripts/check-instruments.mjs scripts/check-account-status.mjs scripts/check-kept-rules.mjs" + "check": "node --experimental-strip-types --disable-warning=MODULE_TYPELESS_PACKAGE_JSON --test scripts/check-trading-logic.mjs scripts/check-instruments.mjs scripts/check-account-status.mjs scripts/check-kept-rules.mjs scripts/check-request-user-agent.mjs" }, "dependencies": { "@microsoft/signalr": "^10.0.11", diff --git a/scripts/check-request-user-agent.mjs b/scripts/check-request-user-agent.mjs new file mode 100644 index 0000000..bcb4bde --- /dev/null +++ b/scripts/check-request-user-agent.mjs @@ -0,0 +1,201 @@ +// Real local HTTP requests and WebSocket handshakes. Never use the live DB +// or contact a broker. The existing check suite uses Node's type stripping. +import { after, test } from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { registerHooks } from 'node:module'; +import { execFileSync } from 'node:child_process'; +import { createServer } from 'node:http'; +import { connect } from 'node:net'; +import { once } from 'node:events'; +import { WebSocketServer } from 'ws'; +import { HubConnectionBuilder, HttpTransportType, LogLevel } from '@microsoft/signalr'; + +// Resolve the extensionless local imports used by Next, without a build. +function resolveTypes(specifier, context, nextResolve) { + try { + return nextResolve(specifier, context); + } catch (err) { + if (err.code !== 'ERR_MODULE_NOT_FOUND' || !specifier.startsWith('.')) throw err; + return nextResolve(`${specifier}.ts`, context); + } +} +registerHooks({ resolve: resolveTypes }); + +const directory = mkdtempSync(join(tmpdir(), 'autofirmer-user-agent-')); +process.env.AUTOFIRMER_DB = join(directory, 'test.sqlite'); +delete process.env.NEXT_PHASE; +after(() => rmSync(directory, { recursive: true, force: true })); + +const { getSetting, setSetting } = await import('../lib/db.ts'); +// Do not let inherited environment proxy settings route local test traffic. +process.env.NO_PROXY = '*'; +setSetting('proxy_url', ''); +const { getRequestUserAgent } = await import('../lib/request-user-agent.ts'); +const { default: axios } = await import('../lib/http-client.ts'); +const { brokerWebSocket, directWebSocket, signalrWebSocket } = await import('../lib/outbound-proxy.ts'); +const userAgent = getRequestUserAgent(); + +async function listen(server) { + server.listen(0, '127.0.0.1'); + await once(server, 'listening'); + return `http://127.0.0.1:${server.address().port}`; +} + +async function socketHeaders(makeSocket, server, url, path = '/') { + const accepted = once(server, 'connection'); + const client = makeSocket(url.replace('http:', 'ws:') + path); + const opened = new Promise((resolve, reject) => { + client.addEventListener('open', resolve, { once: true }); + client.addEventListener('error', reject, { once: true }); + }); + const [peer, request] = await accepted; + try { + await opened; + return request.headers; + } finally { + client.close(); + peer.terminate(); + } +} + +test('first load saves a browser identity and another process reuses it', () => { + assert.match(userAgent, /^Mozilla\/5\.0 .*(Chrome\/|Firefox\/)/); + assert.equal(getSetting('user_agent'), userAgent); + const value = execFileSync(process.execPath, [ + '--experimental-strip-types', '--disable-warning=MODULE_TYPELESS_PACKAGE_JSON', + '--input-type=module', '-e', + `import { registerHooks } from 'node:module'; + registerHooks({ resolve: ${resolveTypes.toString()} }); + const { getRequestUserAgent } = await import(${JSON.stringify(new URL('../lib/request-user-agent.ts', import.meta.url).href)}); + process.stdout.write(getRequestUserAgent());`, + ], { env: process.env, encoding: 'utf8' }); + assert.equal(value, userAgent); +}); + +test('Axios calls, custom headers, and nested create clients use the saved identity', async () => { + const server = createServer((req, res) => { + res.setHeader('Content-Type', 'application/json'); + res.end(JSON.stringify(req.headers)); + }); + const url = await listen(server); + try { + const child = axios.create({ headers: { 'user-agent': 'old child identity' } }); + const grandchild = child.create({ headers: { 'User-Agent': false } }); + for (const client of [axios, child, grandchild]) { + const result = await client.post(url, { hello: 'world' }, { + headers: { 'uSeR-aGeNt': false, Authorization: 'Bearer local-test', 'X-Test': 'kept' }, + }); + assert.equal(result.data['user-agent'], userAgent); + assert.equal(result.data.authorization, 'Bearer local-test'); + assert.equal(result.data['x-test'], 'kept'); + assert.equal(client.defaults.timeout, 60_000); + } + assert.equal((await axios(url)).data['user-agent'], userAgent); + } finally { + await new Promise((resolve) => server.close(resolve)); + } +}); + +test('direct, broker and SignalR handshakes preserve the saved identity and auth', async () => { + const server = createServer(); + const wss = new WebSocketServer({ server }); + const url = await listen(server); + try { + for (const makeSocket of [directWebSocket, brokerWebSocket]) { + assert.equal((await socketHeaders(makeSocket, wss, url))['user-agent'], userAgent); + } + const WS = signalrWebSocket(); + for (const overload of [false, true]) { + const headers = await socketHeaders((address) => { + const options = { headers: { 'user-agent': 'SignalR default', Cookie: 'session=local', Authorization: 'Bearer local-test' } }; + return overload ? new WS(address, options) : new WS(address, undefined, options); + }, wss, url); + assert.equal(headers['user-agent'], userAgent); + assert.equal(headers.cookie, 'session=local'); + assert.equal(headers.authorization, 'Bearer local-test'); + } + + // Exercise SignalR itself, including its generated UA/auth headers. + wss.on('connection', (socket) => { + socket.on('message', () => socket.send('{}\u001e')); + }); + const hub = new HubConnectionBuilder().withUrl(url, { + skipNegotiation: true, + transport: HttpTransportType.WebSockets, + WebSocket: WS, + accessTokenFactory: () => 'signalr-test', + }).configureLogging(LogLevel.None).build(); + const accepted = once(wss, 'connection'); + await hub.start(); + try { + const [, request] = await accepted; + assert.equal(request.headers['user-agent'], userAgent); + assert.equal(request.headers.authorization, 'Bearer signalr-test'); + } finally { + await hub.stop(); + } + } finally { + for (const socket of wss.clients) socket.terminate(); + await new Promise((resolve) => wss.close(resolve)); + await new Promise((resolve) => server.close(resolve)); + } +}); + +test('HTTP proxy keeps the identity on broker sockets while control sockets stay direct', async () => { + const server = createServer(); + const wss = new WebSocketServer({ server }); + const url = await listen(server); + let tunnels = 0; + const connections = new Set(); + const proxy = createServer((req, res) => { + // ws with HttpProxyAgent uses an absolute-form upgrade; handled below. + res.writeHead(500).end(); + }); + function tunnel(req, socket, head, isConnect) { + tunnels++; + const target = new URL(isConnect ? `http://${req.url}` : req.url); + const upstream = connect(Number(target.port), target.hostname, () => { + if (isConnect) socket.write('HTTP/1.1 200 Connection Established\r\n\r\n'); + else { + upstream.write(`${req.method} ${target.pathname} HTTP/1.1\r\n`); + for (let i = 0; i < req.rawHeaders.length; i += 2) upstream.write(`${req.rawHeaders[i]}: ${req.rawHeaders[i + 1]}\r\n`); + upstream.write('\r\n'); + } + if (head.length) upstream.write(head); + socket.pipe(upstream).pipe(socket); + }); + connections.add(socket); + connections.add(upstream); + socket.on('error', () => upstream.destroy()); + upstream.on('error', () => socket.destroy()); + } + proxy.on('connect', (req, socket, head) => tunnel(req, socket, head, true)); + proxy.on('upgrade', (req, socket, head) => tunnel(req, socket, head, false)); + const proxyUrl = await listen(proxy); + setSetting('proxy_url', proxyUrl); + delete globalThis.__outboundProxy; + try { + for (const makeSocket of [brokerWebSocket, (address) => new (signalrWebSocket())(address)]) { + const before = tunnels; + assert.equal((await socketHeaders(makeSocket, wss, url))['user-agent'], userAgent); + assert.equal(tunnels, before + 1); + } + const before = tunnels; + assert.equal((await socketHeaders(directWebSocket, wss, url))['user-agent'], userAgent); + assert.equal(tunnels, before); + } finally { + await globalThis.__outboundProxy.dispatcher.destroy(); + globalThis.__outboundProxy.httpAgent.destroy(); + globalThis.__outboundProxy.httpsAgent.destroy(); + delete globalThis.__outboundProxy; + setSetting('proxy_url', ''); + for (const socket of connections) socket.destroy(); + for (const socket of wss.clients) socket.terminate(); + await new Promise((resolve) => wss.close(resolve)); + await new Promise((resolve) => server.close(resolve)); + await new Promise((resolve) => proxy.close(resolve)); + } +});