diff --git a/.gitignore b/.gitignore index c1cc441..4abf398 100644 --- a/.gitignore +++ b/.gitignore @@ -71,3 +71,9 @@ scripts/.last-check deploy/publish.log deploy/.last-check deploy/.publish.lock +*.sqlite-shm +*.sqlite-wal +# per-instance databases (add-instance.sh) +.data/ +# per-device instance registry (add-instance.sh) +scripts/instances.json diff --git a/app/api/backup/restore/route.ts b/app/api/backup/restore/route.ts new file mode 100644 index 0000000..fb51a0c --- /dev/null +++ b/app/api/backup/restore/route.ts @@ -0,0 +1,68 @@ +import { NextRequest, NextResponse } from 'next/server'; +import { getSetting } from '@/lib/db'; +import { applyBackup, fetchBackup, fetchBackupList } from '@/lib/backup-restore'; + +/** + * Restore-from-master for the settings page. + * + * The browser talks to this route and this route talks to the master with the + * machine's own stored master_dashboard_url + master_backup_token, so the + * token never reaches the browser. + * + * GET — the available backups, for the picker. + * POST { id } — pull that backup and apply it, adopting its identity. + * The process must be restarted afterwards; the response says so. + */ + +function config(): { masterUrl: string; token: string } | NextResponse { + const masterUrl = getSetting('master_dashboard_url'); + const token = getSetting('master_backup_token'); + if (!masterUrl) return NextResponse.json({ error: 'master_dashboard_url is not set' }, { status: 400 }); + if (!token) return NextResponse.json({ error: 'master_backup_token is not set' }, { status: 400 }); + return { masterUrl, token }; +} + +export async function GET() { + const cfg = config(); + if (cfg instanceof NextResponse) return cfg; + + try { + const backups = await fetchBackupList(cfg.masterUrl, cfg.token); + return NextResponse.json({ backups }); + } catch (err) { + const error = err instanceof Error ? err.message : String(err); + return NextResponse.json({ error }, { status: 502 }); + } +} + +export async function POST(req: NextRequest) { + const cfg = config(); + if (cfg instanceof NextResponse) return cfg; + + let id = ''; + try { + const body = await req.json(); + id = typeof body?.id === 'string' ? body.id.trim() : ''; + } catch { /* falls through to the check below */ } + if (!id) return NextResponse.json({ error: 'id is required' }, { status: 400 }); + + try { + const payload = await fetchBackup(cfg.masterUrl, cfg.token, id); + const restored = applyBackup(payload); + return NextResponse.json({ + ok: true, + restored, + savedAt: payload.savedAt, + adopted: { + name: payload.settings.instance_name ?? '', + id: payload.settings.instance_id ?? '', + }, + // Live platform clients, the master link's hello and the proxy all + // predate the restore — a restart is the only honest way in. + restartRequired: true, + }); + } catch (err) { + const error = err instanceof Error ? err.message : String(err); + return NextResponse.json({ error }, { status: 502 }); + } +} diff --git a/app/api/firms/[id]/credentials/route.ts b/app/api/firms/[id]/credentials/route.ts index 60185b6..bebda60 100644 --- a/app/api/firms/[id]/credentials/route.ts +++ b/app/api/firms/[id]/credentials/route.ts @@ -2,12 +2,15 @@ import { NextRequest, NextResponse } from 'next/server'; import { getCachedFirm } from '@/lib/firm-catalog'; import { setCredentials, clearCredentials } from '@/lib/db'; import { reconcileClients, removeClient } from '@/lib/clients'; +import { scheduleBackupPush } from '@/lib/backup-sync'; type Params = { params: Promise<{ id: string }> }; /** The platform login for one firm. This is the only firm data this machine - * owns — everything else comes from the master catalog, and these credentials - * never go the other way. + * owns — everything else comes from the master catalog. The one path these + * credentials travel is the encrypted, bearer-gated backup to the master + * (lib/backup-sync.ts); they are still absent from the catalog and the + * control socket. * * Tradovate firms need username + password. TopstepX firms need the API key; * username is optional for now (the loginKey call still wants it, so the @@ -36,6 +39,7 @@ export async function PUT(req: NextRequest, { params }: Params) { setCredentials(id, firm.name, username, password, apiKey); // Picks up the new login immediately rather than at the next restart. reconcileClients(); + scheduleBackupPush(); return NextResponse.json({ ok: true }); } catch (err) { console.error('[PUT /api/firms/:id/credentials]', err); @@ -47,5 +51,6 @@ export async function DELETE(_req: NextRequest, { params }: Params) { const { id } = await params; const cleared = clearCredentials(id); removeClient(id); + scheduleBackupPush(); return NextResponse.json({ ok: cleared }); } diff --git a/app/api/settings/route.ts b/app/api/settings/route.ts index 376029d..52998d1 100644 --- a/app/api/settings/route.ts +++ b/app/api/settings/route.ts @@ -1,7 +1,8 @@ import { NextRequest, NextResponse } from 'next/server'; import { getSetting, setSetting } from '@/lib/db'; +import { scheduleBackupPush } from '@/lib/backup-sync'; -const VALID_KEYS = ['max_concurrent_accounts', 'tick_interval_seconds', 'master_dashboard_url', 'instance_name', 'trading_hours', 'twocaptcha_api_key', 'twocaptcha_proxy'] as const; +const VALID_KEYS = ['max_concurrent_accounts', 'tick_interval_seconds', 'master_dashboard_url', 'instance_name', 'trading_hours', 'twocaptcha_api_key', 'twocaptcha_proxy', 'master_backup_token', 'proxy_url'] as const; type SettingKey = typeof VALID_KEYS[number]; export async function GET() { @@ -12,6 +13,8 @@ export async function GET() { // Read-only on purpose (not in VALID_KEYS): the master issues this uuid, // and editing it by hand would detach the instance from its person. result.instance_id = getSetting('instance_id'); + result.backup_last_push_at = getSetting('backup_last_push_at'); + result.backup_last_error = getSetting('backup_last_error'); return NextResponse.json(result); } @@ -27,6 +30,10 @@ export async function PATCH(req: NextRequest) { } } + // The edit should reach the master's backup within seconds, not at the + // next poll. + scheduleBackupPush(); + return NextResponse.json({ ok: true }); } catch (err: any) { return NextResponse.json({ error: err?.message ?? 'Failed to save settings' }, { status: 500 }); diff --git a/app/settings/page.tsx b/app/settings/page.tsx index 91c7e6e..1d63df3 100644 --- a/app/settings/page.tsx +++ b/app/settings/page.tsx @@ -19,6 +19,17 @@ interface AppSettings { instance_name: string | null; instance_id: string | null; trading_hours: string | null; + master_backup_token: string | null; + proxy_url: string | null; + backup_last_push_at: string | null; + backup_last_error: string | null; +} + +interface BackupMeta { + id: string; + name: string; + savedAt: string; + sizeBytes: number; } export default function SettingsPage() { @@ -38,6 +49,12 @@ export default function SettingsPage() { const [instanceId, setInstanceId] = useState(''); const [saving, setSaving] = useState(false); const [saved, setSaved] = useState(false); + const [backupToken, setBackupToken] = useState(''); + const [proxyUrl, setProxyUrl] = useState(''); + const [backupStatus, setBackupStatus] = useState<{ at: string | null; error: string | null }>({ at: null, error: null }); + const [restoreList, setRestoreList] = useState(null); + const [restoreBusy, setRestoreBusy] = useState(false); + const [restoreMsg, setRestoreMsg] = useState(''); const fetchCatalog = () => { fetch('/api/firm-catalog') @@ -80,6 +97,13 @@ export default function SettingsPage() { if (s.instance_id != null) { setInstanceId(s.instance_id); } + if (s.master_backup_token != null) { + setBackupToken(s.master_backup_token); + } + if (s.proxy_url != null) { + setProxyUrl(s.proxy_url); + } + setBackupStatus({ at: s.backup_last_push_at, error: s.backup_last_error || null }); }); // Load cached contracts; if cache is empty, auto-resolve @@ -134,13 +158,61 @@ export default function SettingsPage() { await fetch('/api/settings', { method: 'PATCH', headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ max_concurrent_accounts: maxConcurrent, trading_hours: tradingHours, master_dashboard_url: masterUrl, instance_name: instanceName }), + body: JSON.stringify({ max_concurrent_accounts: maxConcurrent, trading_hours: tradingHours, master_dashboard_url: masterUrl, instance_name: instanceName, master_backup_token: backupToken, proxy_url: proxyUrl }), }); setSaving(false); setSaved(true); setTimeout(() => setSaved(false), 2000); } + async function loadRestoreList() { + setRestoreBusy(true); + setRestoreMsg(''); + try { + const res = await fetch('/api/backup/restore'); + const body = await res.json(); + if (!res.ok) throw new Error(body?.error ?? `HTTP ${res.status}`); + setRestoreList(body.backups ?? []); + } catch (err) { + setRestoreMsg(err instanceof Error ? err.message : String(err)); + } + setRestoreBusy(false); + } + + async function restoreBackup(b: BackupMeta) { + const warning = + `Restore "${b.name}" (saved ${new Date(b.savedAt).toLocaleString()})?\n\n` + + 'This OVERWRITES this machine\'s settings and firm credentials and adopts ' + + 'that instance\'s identity on the master.\n\n' + + 'Make sure the old machine is STOPPED first — two live instances with one ' + + 'identity will fight over the connection.'; + if (!window.confirm(warning)) return; + setRestoreBusy(true); + setRestoreMsg(''); + try { + const res = await fetch('/api/backup/restore', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ id: b.id }), + }); + const body = await res.json(); + if (!res.ok) throw new Error(body?.error ?? `HTTP ${res.status}`); + setRestoreMsg( + `Restored ${body.restored.settings} settings and ${body.restored.credentials} credential(s) as ` + + `"${body.adopted.name}". Restart the autotrader now (pm2 restart) to finish.`, + ); + setRestoreList(null); + } catch (err) { + setRestoreMsg(err instanceof Error ? err.message : String(err)); + } + setRestoreBusy(false); + } + + /** >24h old backups get flagged — a dead VPS's last push may be stale. */ + function isStale(savedAt: string): boolean { + return Date.now() - new Date(savedAt).getTime() > 24 * 60 * 60 * 1000; + } + return (
@@ -271,6 +343,103 @@ export default function SettingsPage() { {syncing ? 'Syncing…' : 'Sync now'}
+
+
+

Backup token

+

+ Enables encrypted settings + credentials backups to the master + (its INSTANCE_BACKUP_TOKEN). Blank = backups off. +

+

+ {backupStatus.at + ? `Last backed up ${new Date(backupStatus.at).toLocaleString()}` + : 'Never backed up'} +

+ {backupStatus.error && ( +

{backupStatus.error}

+ )} +
+ setBackupToken(e.target.value)} + className="w-72 rounded-lg border border-slate-200 bg-slate-50 px-3 py-1.5 text-sm font-mono text-slate-800 focus:outline-none focus:ring-2 focus:ring-blue-500" + /> +
+
+
+
+

Restore from master

+

+ Replace a dead VPS: pull its backup, overwrite this machine + and adopt its identity. Stop the old machine first. +

+
+ +
+ {restoreMsg && ( +

{restoreMsg}

+ )} + {restoreList && ( +
+ {restoreList.length === 0 ? ( +

No backups on the master yet

+ ) : restoreList.map((b) => ( +
+
+ {b.name} + + saved {new Date(b.savedAt).toLocaleString()} + {isStale(b.savedAt) ? ' — over a day old' : ''} + + {b.id.slice(0, 8)} +
+ +
+ ))} +
+ )} +
+
+ + {/* ── Network ── */} +

+ Network +

+
+
+
+

Broker proxy

+

+ Routes broker traffic (TopstepX, Tradovate, quotes) through a proxy — + http://, https:// or socks5:// with optional user:pass. Use socks5h:// + to resolve DNS on the proxy. The master connection stays direct. +

+

+ Takes effect after a restart (pm2 restart). +

+
+ setProxyUrl(e.target.value)} + className="w-72 rounded-lg border border-slate-200 bg-slate-50 px-3 py-1.5 text-sm font-mono text-slate-800 focus:outline-none focus:ring-2 focus:ring-blue-500" + /> +
{/* ── Instruments ── */} diff --git a/lib/backup-restore.ts b/lib/backup-restore.ts new file mode 100644 index 0000000..3e5abd9 --- /dev/null +++ b/lib/backup-restore.ts @@ -0,0 +1,126 @@ +/** + * Restores a backup pulled from the master onto THIS machine — the other half + * of lib/backup-sync.ts, for replacing a dead VPS. + * + * Applying a backup overwrites this machine's settings, broker credentials, + * instruments and assignments, and — the point of the exercise — adopts the + * backed-up instance's uuid and name, so the master's roster, payouts and card + * links carry straight on. The master already handles adoption: a hello + * presenting a known uuid lands on that person's roster row. + * + * Run it before first boot via scripts/seed-settings.js --restore-from (which + * duplicates this apply logic — keep them in sync), or from the settings UI + * via POST /api/backup/restore. Either way the old machine must be stopped + * first: two live instances presenting one uuid fight over the socket. + */ + +import { + setSetting, + getSetting, + setCredentials, + setInstrumentEnabled, + setAssignment, + runInTransaction, +} from './db'; +import type { BackupPayload } from './backup-sync'; + +/** Mirror of the master's listing rows (lib/backups.ts BackupMeta). */ +export interface BackupMeta { + id: string; + name: string; + savedAt: string; + sizeBytes: number; +} + +const FETCH_TIMEOUT_MS = 15_000; + +// Runtime bookkeeping describing the machine the backup was taken on, not +// this one. Kept in sync with the skip-set in scripts/seed-settings.js. +const RESTORE_SKIP_KEYS = new Set([ + 'runner_heartbeat', + 'catalog_last_sync_at', + 'catalog_last_error', + 'backup_last_push_at', + 'backup_last_error', +]); + +function authedFetch(masterUrl: string, token: string, path: string) { + const url = new URL(path, masterUrl); + return fetch(url, { + headers: { authorization: `Bearer ${token}` }, + signal: AbortSignal.timeout(FETCH_TIMEOUT_MS), + }); +} + +export async function fetchBackupList(masterUrl: string, token: string): Promise { + const res = await authedFetch(masterUrl, token, '/api/instances/backup'); + if (!res.ok) { + const detail = await res.json().then((b) => b?.error).catch(() => ''); + throw new Error(`master returned ${res.status}${detail ? ` — ${detail}` : ''}`); + } + const body = await res.json(); + if (!Array.isArray(body?.backups)) throw new Error('unrecognized backup list'); + return body.backups as BackupMeta[]; +} + +export async function fetchBackup(masterUrl: string, token: string, id: string): Promise { + const res = await authedFetch(masterUrl, token, `/api/instances/backup/${encodeURIComponent(id)}`); + if (!res.ok) { + const detail = await res.json().then((b) => b?.error).catch(() => ''); + throw new Error(`master returned ${res.status}${detail ? ` — ${detail}` : ''}`); + } + const payload = await res.json() as BackupPayload; + if (payload?.v !== 1 || typeof payload.settings !== 'object' || payload.settings === null) { + throw new Error('unrecognized backup payload'); + } + return payload; +} + +export interface RestoreCounts { + settings: number; + credentials: number; + instruments: number; + assignments: number; +} + +/** + * Overwrites this machine's configuration with the backup, including + * instance_id and instance_name — the identity adoption. Callers restart the + * process afterwards: live platform clients, the master link's hello and any + * cached settings all predate the restore. + */ +export function applyBackup(payload: BackupPayload, opts: { keepMasterUrl?: boolean } = {}): RestoreCounts { + // Defaults to keeping this machine's master url: the machine is talking to + // that master right now, while an old backup may point at a dead host. + const keepMasterUrl = opts.keepMasterUrl ?? true; + const ownMasterUrl = getSetting('master_dashboard_url'); + const ownToken = getSetting('master_backup_token'); + + const counts: RestoreCounts = { settings: 0, credentials: 0, instruments: 0, assignments: 0 }; + + runInTransaction(() => { + for (const [key, value] of Object.entries(payload.settings)) { + if (RESTORE_SKIP_KEYS.has(key)) continue; + if (typeof value !== 'string') continue; + setSetting(key, value); + counts.settings++; + } + if (keepMasterUrl && ownMasterUrl) setSetting('master_dashboard_url', ownMasterUrl); + if (ownToken) setSetting('master_backup_token', ownToken); + + for (const c of payload.credentials ?? []) { + setCredentials(c.firm_id, c.firm_name, c.username, c.password, c.api_key ?? ''); + counts.credentials++; + } + for (const i of payload.instruments ?? []) { + setInstrumentEnabled(i.symbol, i.enabled); + counts.instruments++; + } + for (const a of payload.assignments ?? []) { + setAssignment(a.account_id, a.firm_id, a.account_type_id ?? null, a.manual === 1); + counts.assignments++; + } + }); + + return counts; +} diff --git a/lib/backup-sync.ts b/lib/backup-sync.ts new file mode 100644 index 0000000..07cb1d3 --- /dev/null +++ b/lib/backup-sync.ts @@ -0,0 +1,166 @@ +/** + * Pushes this machine's full configuration — settings, broker credentials, + * instruments, manual assignments — to the master, so a dead VPS can be + * replaced by restoring the backup (lib/backup-restore.ts) and adopting this + * instance's uuid. + * + * This is the deliberate exception to "credentials never leave the machine": + * the payload goes over HTTPS to the bearer-gated /api/instances/backup and is + * AES-encrypted at rest on the master. It never rides the unauthenticated + * control socket. + * + * Push policy: on bootstrap, then every POLL_INTERVAL, but only when the + * content actually changed (hash comparison) — plus an immediate debounced + * push whenever settings or credentials are edited (scheduleBackupPush, called + * from the mutation routes). + */ + +import { createHash } from 'node:crypto'; +import { + getSetting, + setSetting, + listSettings, + listCredentials, + getInstruments, + listAllAssignments, + type FirmCredentialRow, + type AssignmentRow, +} from './db'; + +export interface BackupPayload { + v: 1; + savedAt: string; + settings: Record; + credentials: FirmCredentialRow[]; + instruments: { symbol: string; enabled: boolean }[]; + assignments: AssignmentRow[]; +} + +const BACKUP_PATH = '/api/instances/backup'; +const POLL_INTERVAL_MS = 5 * 60_000; +const DEBOUNCE_MS = 5_000; +const FETCH_TIMEOUT_MS = 10_000; + +const g = globalThis as typeof globalThis & { + __backupSyncTimer?: ReturnType; + __backupDebounceTimer?: ReturnType; + __backupLastHash?: string; +}; + +// Bookkeeping that describes this process run, not the configuration worth +// restoring. Excluded from the hash so a heartbeat doesn't look like a change +// worth pushing. (The restore side has its own skip-set; these still travel in +// the payload so a restore debugger can see them.) +const VOLATILE_KEYS = new Set([ + 'runner_heartbeat', + 'catalog_last_sync_at', + 'catalog_last_error', + 'backup_last_push_at', + 'backup_last_error', +]); + +export function collectBackup(): BackupPayload { + return { + v: 1, + savedAt: new Date().toISOString(), + settings: listSettings(), + credentials: listCredentials(), + instruments: getInstruments().map((i) => ({ symbol: i.symbol, enabled: Boolean(i.enabled) })), + assignments: listAllAssignments(), + }; +} + +/** Stable digest of everything restorable, so unchanged config never pushes. */ +function contentHash(payload: BackupPayload): string { + const settings = Object.entries(payload.settings) + .filter(([k]) => !VOLATILE_KEYS.has(k)) + .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)); + return createHash('sha256') + .update(JSON.stringify({ + settings, + credentials: payload.credentials, + instruments: payload.instruments, + assignments: payload.assignments, + })) + .digest('hex'); +} + +function preconditions(): { masterUrl: string; id: string; token: string } | null { + const masterUrl = getSetting('master_dashboard_url'); + const token = getSetting('master_backup_token'); + // No uuid yet = the master has never said welcome; there is nothing stable + // to key the backup by, so wait for the first hello to complete. + const id = getSetting('instance_id'); + if (!masterUrl || !token || !id) return null; + return { masterUrl, id, token }; +} + +export async function pushBackup(force = false): Promise<{ ok: boolean; skipped?: boolean; error?: string }> { + const pre = preconditions(); + if (!pre) return { ok: false, skipped: true, error: 'master url, backup token or instance uuid not set' }; + + const payload = collectBackup(); + const hash = contentHash(payload); + if (!force && hash === g.__backupLastHash) return { ok: true, skipped: true }; + + let url: URL; + try { + url = new URL(BACKUP_PATH, pre.masterUrl); + } catch { + return { ok: false, error: `master_dashboard_url is not a URL: ${pre.masterUrl}` }; + } + + try { + const res = await fetch(url, { + method: 'PUT', + headers: { + 'content-type': 'application/json', + authorization: `Bearer ${pre.token}`, + }, + body: JSON.stringify({ + id: pre.id, + name: getSetting('instance_name') ?? '', + payload, + }), + signal: AbortSignal.timeout(FETCH_TIMEOUT_MS), + }); + if (!res.ok) { + const detail = await res.json().then((b) => b?.error).catch(() => ''); + throw new Error(`master returned ${res.status}${detail ? ` — ${detail}` : ''}`); + } + g.__backupLastHash = hash; + setSetting('backup_last_push_at', new Date().toISOString()); + setSetting('backup_last_error', ''); + console.log('[backup] pushed to master'); + return { ok: true }; + } catch (err) { + const error = err instanceof Error ? err.message : String(err); + console.error('[backup] push failed:', error); + setSetting('backup_last_error', error); + return { ok: false, error }; + } +} + +/** Debounced push for the mutation routes: a settings PATCH or credential PUT + * lands on the master seconds later instead of at the next poll. */ +export function scheduleBackupPush(): void { + if (g.__backupDebounceTimer) clearTimeout(g.__backupDebounceTimer); + g.__backupDebounceTimer = setTimeout(() => { void pushBackup(); }, DEBOUNCE_MS); +} + +export function startBackupSync(): void { + stopBackupSync(); + const tick = async () => { + await pushBackup(); + // Jitter so a fleet restarting together doesn't push in lockstep. + g.__backupSyncTimer = setTimeout(tick, POLL_INTERVAL_MS + Math.floor(Math.random() * 30_000)); + }; + void tick(); +} + +export function stopBackupSync(): void { + if (g.__backupSyncTimer) { + clearTimeout(g.__backupSyncTimer); + g.__backupSyncTimer = undefined; + } +} diff --git a/lib/clients.ts b/lib/clients.ts index 9c9da48..8c3211b 100644 --- a/lib/clients.ts +++ b/lib/clients.ts @@ -6,6 +6,8 @@ import { getTradableFirms, startCatalogSync } from './firm-catalog'; import { resolveFirmAssignments } from './account-type-resolver'; import { resolveContracts } from './contract-resolver'; import { startReporter, sendReport } from './reporter'; +import { startBackupSync } from './backup-sync'; +import { applyBrokerProxy } from './outbound-proxy'; import { startMasterLink } from './master-link'; import { resumeSchedulerIfPersisted } from './auto-trade'; @@ -128,6 +130,10 @@ export function getClients(): Map { if (!g.__tradovateClientsInitialized) { g.__tradovateClientsInitialized = true; try { + // Before anything dials out: put the operator's proxy (if any) + // under the broker clients. Master traffic stays direct by design. + applyBrokerProxy(); + // Pull the catalog before anything else — without it there are no // firms to start clients for. Subsequent changes re-reconcile and // re-resolve, so a new account type picks up accounts that were @@ -155,6 +161,10 @@ export function getClients(): Map { // Start master dashboard reporter startReporter(); + // Push settings+credentials backups to the master, so a dead VPS + // can be replaced and this identity adopted. + startBackupSync(); + // ...and the socket it prefers to report over, which is also how // the master triggers trades and AutoBuyer runs here. startMasterLink(() => { void sendReport(); }); diff --git a/lib/db.ts b/lib/db.ts index bc2eec5..c81a5f8 100644 --- a/lib/db.ts +++ b/lib/db.ts @@ -1,9 +1,22 @@ import Database from 'better-sqlite3'; +import { mkdirSync } from 'fs'; import path from 'path'; -const db = new Database(path.join(process.cwd(), 'autotrader.sqlite')); +// AUTOFIRMER_DB lets several instances share one checkout, each with its own +// database. It is read once, at module load — schema creation and seeding are +// import side effects — so it must be in the environment at process start. +const DB_PATH = process.env.AUTOFIRMER_DB + ? path.resolve(process.env.AUTOFIRMER_DB) + : path.join(process.cwd(), 'autotrader.sqlite'); + +mkdirSync(path.dirname(DB_PATH), { recursive: true }); +const db = new Database(DB_PATH); db.pragma('foreign_keys = ON'); +// If two processes are ever misconfigured onto the same file, WAL + a busy +// timeout turns SQLITE_BUSY crashes into brief contention. +db.pragma('journal_mode = WAL'); +db.pragma('busy_timeout = 5000'); db.exec(` CREATE TABLE IF NOT EXISTS instruments ( @@ -123,6 +136,11 @@ export function listAssignments(firmId: string): AssignmentRow[] { return db.prepare('SELECT * FROM account_assignments WHERE firm_id = ?').all(firmId) as AssignmentRow[]; } +/** Every assignment, for the backup pushed to the master. */ +export function listAllAssignments(): AssignmentRow[] { + return db.prepare('SELECT * FROM account_assignments').all() as AssignmentRow[]; +} + export function setAssignment( accountId: number, firmId: string, @@ -179,12 +197,31 @@ seedSetting.run('twocaptcha_api_key', ''); // Blank falls back to the proxy hardcoded in app/api/autobuyer/captcha; `none` // solves without one. seedSetting.run('twocaptcha_proxy', ''); +// Bearer for pushing/pulling the encrypted settings+credentials backup on the +// master (its INSTANCE_BACKUP_TOKEN). Blank = backups off. +seedSetting.run('master_backup_token', ''); +// Routes broker traffic (TopstepX, Tradovate, Yahoo quotes) through a proxy: +// http(s):// or socks5(h)://, user:pass ok. The master link stays direct on +// purpose. Blank = direct. Takes effect on restart. +seedSetting.run('proxy_url', ''); // Mirrors the in-memory scheduler state so a restart can resume it. seedSetting.run('scheduler_running', '0'); seedSetting.run('scheduler_action', 'Buy'); seedSetting.run('scheduler_symbol', 'NQ'); seedSetting.run('scheduler_stop_after_all', '0'); +/** Wraps several writes in one transaction (used by the backup restore, so a + * half-applied backup cannot survive a crash). */ +export function runInTransaction(fn: () => T): T { + return db.transaction(fn)(); +} + +/** The whole table, for the backup pushed to the master. */ +export function listSettings(): Record { + const rows = db.prepare('SELECT key, value FROM settings').all() as { key: string; value: string }[]; + return Object.fromEntries(rows.map((r) => [r.key, r.value])); +} + export function getSetting(key: string): string | null { const row = db.prepare('SELECT value FROM settings WHERE key = ?').get(key) as { value: string } | undefined; return row?.value ?? null; diff --git a/lib/master-link.ts b/lib/master-link.ts index 4a3d4bb..b8f92c7 100644 --- a/lib/master-link.ts +++ b/lib/master-link.ts @@ -20,6 +20,7 @@ import { runTrade, startScheduler, stopScheduler, getSchedulerStatus } from './a import { FIRMS, automationKey } from './automations'; import { POINT_VALUES } from './trading-logic'; import { queueRun } from './queue-run'; +import { pushBackup } from './backup-sync'; import { getBuildVersion } from './version'; import type { Command, @@ -119,6 +120,9 @@ async function execute(command: Command): Promise { case 'report-now': return { reported: true }; + case 'backup-now': + return pushBackup(true); + default: { // A newer master may know commands this build does not. Say so // rather than failing silently. diff --git a/lib/master-protocol.ts b/lib/master-protocol.ts index 5777b84..6e51340 100644 --- a/lib/master-protocol.ts +++ b/lib/master-protocol.ts @@ -198,7 +198,9 @@ export type Command = | { kind: 'trade'; action: 'Buy' | 'Sell' | 'Auto'; symbol: string; stopAfterAll?: boolean } | { kind: 'stop-trading' } | { kind: 'autobuyer-run'; automationId: string; inputs?: Record } - | { kind: 'report-now' }; + | { kind: 'report-now' } + /** Push a fresh settings/credentials backup right now. */ + | { kind: 'backup-now' }; export type MasterMessage = /** personId is the person's stable uuid; the instance stores it and diff --git a/lib/outbound-proxy.ts b/lib/outbound-proxy.ts new file mode 100644 index 0000000..bbc4cb7 --- /dev/null +++ b/lib/outbound-proxy.ts @@ -0,0 +1,130 @@ +/** + * Routes BROKER traffic through the operator's proxy — and nothing else. + * + * Scope is deliberate: TopstepX REST + its SignalR hub, Tradovate REST + its + * websocket, and the Yahoo quote lookups go through the proxy so each instance + * can present its own IP to the firms. The master link, reporter, catalog sync + * and backup push stay direct — the control channel must keep working when the + * proxy dies, and there is no setGlobalDispatcher here for exactly that + * reason. (Corollary: never call the master through axios — axios defaults + * carry the proxy agents. Master calls use fetch, which stays untouched.) + * + * Configured by the `proxy_url` setting: http://, https://, socks5:// or + * socks5h:// with optional user:pass. Read once per process — changing it + * requires a restart, which the settings UI says out loud. Blank = direct. + */ + +import axios from 'axios'; +import { ProxyAgent, type Dispatcher } from 'undici'; +import { socksDispatcher } from 'fetch-socks'; +import { HttpProxyAgent } from 'http-proxy-agent'; +import { HttpsProxyAgent } from 'https-proxy-agent'; +import { SocksProxyAgent } from 'socks-proxy-agent'; +import type { Agent } from 'node:http'; +import { getSetting } from './db'; + +const SCHEMES = ['http:', 'https:', 'socks5:', 'socks5h:'] as const; + +interface ProxyState { + url: URL | null; + dispatcher: Dispatcher | null; + httpAgent: Agent | null; + httpsAgent: Agent | null; + applied: boolean; +} + +// HMR-safe, and the "read once per process" contract lives here. +const g = globalThis as typeof globalThis & { __outboundProxy?: ProxyState }; + +function state(): ProxyState { + if (g.__outboundProxy) return g.__outboundProxy; + + const s: ProxyState = { url: null, dispatcher: null, httpAgent: null, httpsAgent: null, applied: false }; + g.__outboundProxy = s; + + const raw = getSetting('proxy_url')?.trim(); + if (!raw) return s; + + let url: URL; + try { + url = new URL(raw); + } catch { + console.error(`[proxy] proxy_url is not a URL — trading DIRECT: ${raw}`); + return s; + } + if (!SCHEMES.includes(url.protocol as (typeof SCHEMES)[number])) { + console.error(`[proxy] proxy_url scheme must be one of ${SCHEMES.join(' ')} — trading DIRECT: ${url.protocol}`); + return s; + } + + const socks = url.protocol.startsWith('socks'); + if (socks) { + s.dispatcher = socksDispatcher({ + type: 5, + host: url.hostname, + port: Number(url.port || 1080), + userId: url.username ? decodeURIComponent(url.username) : undefined, + password: url.password ? decodeURIComponent(url.password) : undefined, + }); + const agent = new SocksProxyAgent(url); + s.httpAgent = agent; + s.httpsAgent = agent; + } else { + s.dispatcher = new ProxyAgent(url.href); + s.httpAgent = new HttpProxyAgent(url); + s.httpsAgent = new HttpsProxyAgent(url); + } + s.url = url; + console.log(`[proxy] broker traffic will use ${url.protocol}//${url.host} (master link stays direct)`); + return s; +} + +export function getProxyUrl(): string | null { + const { url } = state(); + return url ? url.href : null; +} + +/** Puts the proxy under every axios call. All axios usage in this app is + * broker/quote traffic (TopstepX, Tradovate REST, contract resolver), so the + * process-wide default is the scoped choice, not a global one. Idempotent; + * called first in the getClients() bootstrap. */ +export function applyBrokerProxy(): void { + const s = state(); + if (s.applied || !s.url) return; + s.applied = true; + axios.defaults.proxy = false; // agents own the routing; ignore env HTTP(S)_PROXY handling + axios.defaults.httpAgent = s.httpAgent!; + axios.defaults.httpsAgent = s.httpsAgent!; +} + +/** A websocket to a BROKER endpoint: the built-in (undici) WebSocket, dialled + * through the proxy when one is set. The master link keeps using plain + * `new WebSocket(...)` — do not use this for it. */ +export function brokerWebSocket(wsUrl: string): WebSocket { + const { dispatcher } = state(); + if (!dispatcher) return new WebSocket(wsUrl); + // The runtime accepts { dispatcher } fine; only the DOM lib type doesn't. + return new (WebSocket as unknown as new (u: string, o: object) => WebSocket)(wsUrl, { dispatcher }); +} + +/** A `ws`-package constructor with the proxy agent baked in, for SignalR's + * `withUrl({ WebSocket })` option. undefined = no proxy, let SignalR use its + * own require('ws'). */ +export function signalrWebSocket(): unknown { + const s = state(); + if (!s.httpsAgent) return undefined; + const agent = s.httpsAgent; + // eslint-disable-next-line @typescript-eslint/no-require-imports + const WS = require('ws') as new (address: string, protocols?: unknown, options?: object) => unknown; + + return class ProxiedWebSocket extends (WS as new (...args: never[]) => object) { + constructor(address: string, protocols?: unknown, options?: Record) { + // ws also allows (address, options) — normalize before merging. + if (protocols && typeof protocols === 'object' && !Array.isArray(protocols)) { + options = protocols as Record; + protocols = undefined; + } + super(...[address, protocols, { ...(options ?? {}), agent }] as never[]); + } + }; +} diff --git a/lib/topstepx-class.ts b/lib/topstepx-class.ts index f8e9b18..be0444d 100644 --- a/lib/topstepx-class.ts +++ b/lib/topstepx-class.ts @@ -24,6 +24,7 @@ import axios from 'axios'; import * as signalR from '@microsoft/signalr'; import { currentTradeDateCT } from './trading-logic'; import { getCachedContract } from './contract-resolver'; +import { signalrWebSocket } from './outbound-proxy'; import { saveDailyPnL, loadDailyPnL, loadAccountMeta, saveAccountMeta, loadFundTransactions, saveFundTransactions } from './db'; import { getAccountConfig } from './account-type-resolver'; import type { PlatformClient, PlatformAccount, PlatformContract } from './platform-client'; @@ -258,11 +259,16 @@ export class TopstepXClient implements PlatformClient { // the package is listed in next.config serverExternalPackages and any // residual failure here is caught by the caller (REST polling covers // everything the hub does, just slower). + // With skipNegotiation the hub is a single websocket to a broker + // endpoint, so it follows the operator's proxy like the REST calls do. + const proxiedWs = signalrWebSocket(); const hub = new signalR.HubConnectionBuilder() .withUrl(USER_HUB, { skipNegotiation: true, transport: signalR.HttpTransportType.WebSockets, accessTokenFactory: () => this.token ?? '', + // eslint-disable-next-line @typescript-eslint/no-explicit-any + ...(proxiedWs ? { WebSocket: proxiedWs as any } : {}), }) .withAutomaticReconnect() .configureLogging(signalR.LogLevel.Warning) diff --git a/lib/tradovate-class.ts b/lib/tradovate-class.ts index f87b08c..d03f352 100644 --- a/lib/tradovate-class.ts +++ b/lib/tradovate-class.ts @@ -6,6 +6,7 @@ import { computeSec, randomUUIDV4 } from './tradovate-helpers'; import { POINT_VALUES } from './trading-logic'; import { getCachedContract, resolveContracts } from './contract-resolver'; import { saveDailyPnL, loadDailyPnL, saveAccountMeta, loadAccountMeta, saveFundTransactions, loadFundTransactions } from './db'; +import { brokerWebSocket } from './outbound-proxy'; /** * Entity types Tradovate sends over the user-sync socket. Keep this as the @@ -147,7 +148,9 @@ export class TradovateClient { this.syncComplete = false; const randomnumber = Math.random().toString(36).substring(2, 15); - this.ws = new WebSocket(`wss://demo.tradovateapi.com/v1/websocket?r=${randomnumber}`); + // Broker endpoint — dialled through the operator's proxy when one + // is configured (lib/outbound-proxy.ts). + this.ws = brokerWebSocket(`wss://demo.tradovateapi.com/v1/websocket?r=${randomnumber}`); this.ws.onopen = () => { console.log('Connected to websocket'); diff --git a/next.config.ts b/next.config.ts index baf59c1..933a913 100644 --- a/next.config.ts +++ b/next.config.ts @@ -4,7 +4,9 @@ const nextConfig: NextConfig = { // @microsoft/signalr does dynamic require('ws')/require('eventsource') for // its Node transports — bundling it (turbopack production builds) throws // "dynamic usage of require is not supported" the moment a hub is built. - serverExternalPackages: ['better-sqlite3', '@microsoft/signalr'], + // undici and ws stay external for the same reason as signalr: the proxy + // module (lib/outbound-proxy.ts) needs the real Node packages, not a bundle. + serverExternalPackages: ['better-sqlite3', '@microsoft/signalr', 'undici', 'ws'], }; export default nextConfig; diff --git a/package-lock.json b/package-lock.json index ce026ae..0a0af2b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11,10 +11,16 @@ "@microsoft/signalr": "^10.0.11", "axios": "^1.13.6", "better-sqlite3": "^12.6.2", + "fetch-socks": "^1.3.3", + "http-proxy-agent": "^9.1.0", + "https-proxy-agent": "^9.1.0", "next": "16.1.6", "react": "19.2.3", "react-dom": "19.2.3", - "recharts": "^3.8.0" + "recharts": "^3.8.0", + "socks-proxy-agent": "^10.1.0", + "undici": "^7.29.1", + "ws": "^8.21.3" }, "devDependencies": { "@tailwindcss/postcss": "^4", @@ -1039,6 +1045,27 @@ "ws": "^7.5.10" } }, + "node_modules/@microsoft/signalr/node_modules/ws": { + "version": "7.5.13", + "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.13.tgz", + "integrity": "sha512-rsKI6xDBFVf4r/x8XyChGK04QR/XHroxs/jUcoWvtEZM8TPU/X/uIY9B1CsSzYws9ZJb/6bbBu7dPhFW00CAoA==", + "license": "MIT", + "engines": { + "node": ">=8.3.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": "^5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, "node_modules/@napi-rs/wasm-runtime": { "version": "0.2.12", "resolved": "https://registry.npmjs.org/@napi-rs/wasm-runtime/-/wasm-runtime-0.2.12.tgz", @@ -2319,6 +2346,15 @@ "acorn": "^6.0.0 || ^7.0.0 || ^8.0.0" } }, + "node_modules/agent-base": { + "version": "9.0.0", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-9.0.0.tgz", + "integrity": "sha512-TQf59BsZnytt8GdJKLPfUZ54g/iaUL2OWDSFCCvMOhsHduDQxO8xC4PNeyIkVcA5KwL2phPSv0douC0fgWzmnA==", + "license": "MIT", + "engines": { + "node": ">= 20" + } + }, "node_modules/ajv": { "version": "6.14.0", "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.14.0.tgz", @@ -3125,7 +3161,6 @@ "version": "4.4.3", "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", - "dev": true, "license": "MIT", "dependencies": { "ms": "^2.1.3" @@ -4034,6 +4069,16 @@ "tough-cookie": "^4.0.0" } }, + "node_modules/fetch-socks": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/fetch-socks/-/fetch-socks-1.3.3.tgz", + "integrity": "sha512-jI22jzTf7x3GAcUoZARwuxKBPotb3PFkemVWqxlF2qV55FhRytRJrn95HeLoUMq2vkIbY3aQqNYytvZKqUoMaQ==", + "license": "MIT", + "dependencies": { + "socks": "^2.8.7", + "undici": ">=7" + } + }, "node_modules/file-entry-cache": { "version": "8.0.0", "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-8.0.0.tgz", @@ -4466,6 +4511,34 @@ "hermes-estree": "0.25.1" } }, + "node_modules/http-proxy-agent": { + "version": "9.1.0", + "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-9.1.0.tgz", + "integrity": "sha512-2NxoveTT58mjYT4n3RPTEfCZGLMbidoO8XEieXfpSYxu+PQJ1qpx4ypwH6N+uF9twBPIvRRgvkvW5HUTYWENig==", + "license": "MIT", + "dependencies": { + "agent-base": "9.0.0", + "debug": "^4.3.4", + "proxy-agent-negotiate": "1.1.0" + }, + "engines": { + "node": ">= 20" + } + }, + "node_modules/https-proxy-agent": { + "version": "9.1.0", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-9.1.0.tgz", + "integrity": "sha512-ag87y7cJJ9/3+GxFr8Oy4O5faDsGRGnBGsJj/YjOSsSx/5eadKLYTMPlzuR6obgoCDDm0abAAZitXXQkMOPSpA==", + "license": "MIT", + "dependencies": { + "agent-base": "9.0.0", + "debug": "^4.3.4", + "proxy-agent-negotiate": "1.1.0" + }, + "engines": { + "node": ">= 20" + } + }, "node_modules/ieee754": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", @@ -4569,6 +4642,15 @@ "node": ">=12" } }, + "node_modules/ip-address": { + "version": "10.7.2", + "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-10.7.2.tgz", + "integrity": "sha512-7H/2gFSIitxc0hG3nOI1glS8QLo/EHBFFLk8vEUjXY/xu0AdL8jZ9U1IzO2PUm0d2D/ofQcAifb0g6OBkt8U7w==", + "license": "MIT", + "engines": { + "node": ">= 12" + } + }, "node_modules/is-array-buffer": { "version": "3.0.5", "resolved": "https://registry.npmjs.org/is-array-buffer/-/is-array-buffer-3.0.5.tgz", @@ -5568,7 +5650,6 @@ "version": "2.1.3", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", - "dev": true, "license": "MIT" }, "node_modules/nanoid": { @@ -6116,6 +6197,23 @@ "react-is": "^16.13.1" } }, + "node_modules/proxy-agent-negotiate": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/proxy-agent-negotiate/-/proxy-agent-negotiate-1.1.0.tgz", + "integrity": "sha512-N8IBcM3UgCVzz2L2Lqv8DVntDnnC8/hiV4nEDUPkqq72TPUgYWjQc+bdZlBPZK9LzPAvOY//gAt0S0DApoOXWQ==", + "license": "MIT", + "engines": { + "node": ">= 20" + }, + "peerDependencies": { + "kerberos": "^2.0.0" + }, + "peerDependenciesMeta": { + "kerberos": { + "optional": true + } + } + }, "node_modules/proxy-from-env": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", @@ -6793,6 +6891,44 @@ "simple-concat": "^1.0.0" } }, + "node_modules/smart-buffer": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/smart-buffer/-/smart-buffer-4.2.0.tgz", + "integrity": "sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg==", + "license": "MIT", + "engines": { + "node": ">= 6.0.0", + "npm": ">= 3.0.0" + } + }, + "node_modules/socks": { + "version": "2.8.10", + "resolved": "https://registry.npmjs.org/socks/-/socks-2.8.10.tgz", + "integrity": "sha512-e0VyvkVTwVYViNovRkZ9aodhxVlyoMn7eJhVUPxZ+eK9P/7CBkxvvsBOHqFPEH416726W8tLXXXjKwqgTErrCQ==", + "license": "MIT", + "dependencies": { + "ip-address": "^10.1.1", + "smart-buffer": "^4.2.0" + }, + "engines": { + "node": ">= 10.0.0", + "npm": ">= 3.0.0" + } + }, + "node_modules/socks-proxy-agent": { + "version": "10.1.0", + "resolved": "https://registry.npmjs.org/socks-proxy-agent/-/socks-proxy-agent-10.1.0.tgz", + "integrity": "sha512-WlMj/67cEJ6MDI1OcsnjuYKDNDoyPCCYZ249kuuXPiMDw9F8PXkVaQ7YWu3siTydfQ/4BEZcvGzu+aYvz7dDCQ==", + "license": "MIT", + "dependencies": { + "agent-base": "9.0.0", + "debug": "^4.3.4", + "socks": "^2.8.3" + }, + "engines": { + "node": ">= 20" + } + }, "node_modules/source-map-js": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz", @@ -7359,6 +7495,15 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/undici": { + "version": "7.29.1", + "resolved": "https://registry.npmjs.org/undici/-/undici-7.29.1.tgz", + "integrity": "sha512-RYONW2MeafgYlkVOKYKkA/Ag7BmXqgIWCa8t1m0JcxrQg9pI9lEqRhAOruOBCbAohOa/gkCF+iPi9hrgvTzu6Q==", + "license": "MIT", + "engines": { + "node": ">=20.18.1" + } + }, "node_modules/undici-types": { "version": "6.21.0", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", @@ -7636,16 +7781,16 @@ "license": "ISC" }, "node_modules/ws": { - "version": "7.5.13", - "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.13.tgz", - "integrity": "sha512-rsKI6xDBFVf4r/x8XyChGK04QR/XHroxs/jUcoWvtEZM8TPU/X/uIY9B1CsSzYws9ZJb/6bbBu7dPhFW00CAoA==", + "version": "8.21.3", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.21.3.tgz", + "integrity": "sha512-201TZ/kPWxoPr/OKWjquZR1SWKXcvxdH+e1xrx89b3YbmzLMFCLfnaG1HFIgWzJOEWZ7MvpK++odZufgYR50Rw==", "license": "MIT", "engines": { - "node": ">=8.3.0" + "node": ">=10.0.0" }, "peerDependencies": { "bufferutil": "^4.0.1", - "utf-8-validate": "^5.0.2" + "utf-8-validate": ">=5.0.2" }, "peerDependenciesMeta": { "bufferutil": { diff --git a/package.json b/package.json index 77458f3..5e4c2e0 100644 --- a/package.json +++ b/package.json @@ -12,10 +12,16 @@ "@microsoft/signalr": "^10.0.11", "axios": "^1.13.6", "better-sqlite3": "^12.6.2", + "fetch-socks": "^1.3.3", + "http-proxy-agent": "^9.1.0", + "https-proxy-agent": "^9.1.0", "next": "16.1.6", "react": "19.2.3", "react-dom": "19.2.3", - "recharts": "^3.8.0" + "recharts": "^3.8.0", + "socks-proxy-agent": "^10.1.0", + "undici": "^7.29.1", + "ws": "^8.21.3" }, "devDependencies": { "@tailwindcss/postcss": "^4", diff --git a/scripts/add-instance.sh b/scripts/add-instance.sh new file mode 100755 index 0000000..90b8e0e --- /dev/null +++ b/scripts/add-instance.sh @@ -0,0 +1,106 @@ +#!/usr/bin/env bash +# +# Register an additional autofirmer instance on this device: same checkout, +# same build, its own PM2 process, port and database. +# +# ./scripts/add-instance.sh [--name ] [--master ] +# [--port ] [--token ] +# [--restore-from ] +# +# N instance number, 2 or higher (1 is the legacy install: +# pm2 name `autofirmer`, port 3000, ./autotrader.sqlite — +# registered by install-autostart.sh, never by this script) +# --name instance_name to seed (default: -N) +# --master master dashboard url to seed (default: leave the app's seed) +# --port listen port (default: 3000 + N - 1; NOTE the master dev +# server uses 3001 — pick another port on a dev machine) +# --token master_backup_token to seed +# --restore-from pull that instance's backup from the master before first +# boot and adopt its identity (requires --master and --token) +# +# The instance lands in scripts/instances.json, which start-all.mjs and +# update-check.mjs iterate. The first run of this script also records the +# legacy instance 1 there so updates keep restarting it. +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +REGISTRY="$ROOT/scripts/instances.json" + +info() { printf ' %s\n' "$1"; } +die() { printf ' [X] %s\n' "$1" >&2; exit 1; } + +[ $# -ge 1 ] || die "usage: $0 [--name ...] [--master ...] [--port ...] [--token ...] [--restore-from ...]" +N="$1"; shift +case "$N" in (*[!0-9]*|'') die "N must be a number, got: $N" ;; esac +[ "$N" -ge 2 ] || die "N starts at 2 — instance 1 is the legacy install (install-autostart.sh)" + +NAME="$(hostname)-$N" +MASTER="" +PORT=$((3000 + N - 1)) +TOKEN="" +RESTORE_FROM="" + +while [ $# -gt 0 ]; do + case "$1" in + --name) NAME="$2"; shift 2 ;; + --master) MASTER="$2"; shift 2 ;; + --port) PORT="$2"; shift 2 ;; + --token) TOKEN="$2"; shift 2 ;; + --restore-from) RESTORE_FROM="$2"; shift 2 ;; + *) die "unknown option: $1" ;; + esac +done + +PM2_NAME="autofirmer-$N" +DB_REL=".data/instance-$N/autotrader.sqlite" +DB_ABS="$ROOT/$DB_REL" + +if [ -n "$RESTORE_FROM" ]; then + [ -n "$MASTER" ] || die "--restore-from needs --master" + [ -n "$TOKEN" ] || die "--restore-from needs --token" +fi + +NEXT_BIN="$ROOT/node_modules/next/dist/bin/next" +[ -f "$NEXT_BIN" ] || die "next not found at $NEXT_BIN - run npm install first" +[ -d "$ROOT/.next" ] || die "no production build - run npm run build first" +command -v pm2 >/dev/null 2>&1 || die "PM2 not installed - run install-autostart.sh first" + +# ── registry: create with the legacy entry, refuse duplicates, append ─────── +node - "$REGISTRY" "$PM2_NAME" "$PORT" "$DB_REL" <<'NODE' +const { existsSync, readFileSync, writeFileSync } = require('node:fs'); +const [, file, name, port, db] = process.argv; +const entries = existsSync(file) + ? JSON.parse(readFileSync(file, 'utf8')) + : [{ name: 'autofirmer', port: 3000, db: 'autotrader.sqlite' }]; +for (const e of entries) { + if (e.name === name) { console.error(` [X] ${name} is already registered`); process.exit(1); } + if (e.port === Number(port)) { console.error(` [X] port ${port} is already used by ${e.name}`); process.exit(1); } + if (e.db === db) { console.error(` [X] db ${db} is already used by ${e.name}`); process.exit(1); } +} +entries.push({ name, port: Number(port), db }); +writeFileSync(file, JSON.stringify(entries, null, 4) + '\n'); +console.log(` registry: ${name} on :${port} -> ${db}`); +NODE + +mkdir -p "$(dirname "$DB_ABS")" + +# ── seed (and optionally restore) the new database ────────────────────────── +cd "$ROOT" +SEED_ARGS=() +[ -n "$TOKEN" ] && SEED_ARGS+=(--token "$TOKEN") +[ -n "$RESTORE_FROM" ] && SEED_ARGS+=(--restore-from "$RESTORE_FROM") +AUTOFIRMER_DB="$DB_ABS" node scripts/seed-settings.js "$MASTER" "$NAME" ${SEED_ARGS[@]+"${SEED_ARGS[@]}"} + +# ── PM2 ───────────────────────────────────────────────────────────────────── +pm2 delete "$PM2_NAME" >/dev/null 2>&1 || true +PORT="$PORT" AUTOFIRMER_DB="$DB_ABS" pm2 start "$NEXT_BIN" --name "$PM2_NAME" --interpreter node --update-env -- start +pm2 save >/dev/null +info "pm2: $PM2_NAME registered and saved" + +echo +info "instance $N is up: http://localhost:$PORT" +info "database: $DB_REL" +if [ -n "$RESTORE_FROM" ]; then + info "restored from backup $RESTORE_FROM - stop the old machine if it is still running" +fi +info "remove later with: ./scripts/remove-instance.sh $N" diff --git a/scripts/instances.mjs b/scripts/instances.mjs new file mode 100644 index 0000000..3035c72 --- /dev/null +++ b/scripts/instances.mjs @@ -0,0 +1,44 @@ +/** + * The per-device instance registry: which autofirmer processes run on this + * machine, on which ports, against which databases. + * + * scripts/instances.json is created and edited by add-instance.sh / + * remove-instance.sh. When the file is absent the device is a legacy + * single-instance install and everything behaves exactly as it always has: + * one PM2 process named `autofirmer` on port 3000 using ./autotrader.sqlite. + * + * Entry shape: { "name": "autofirmer-2", "port": 3100, "db": ".data/instance-2/autotrader.sqlite" } + * `db` is relative to the project root. Instance 1 keeps the legacy + * name/port/db so the clicker and extension (hardwired to :3000) still work. + */ +import { existsSync, readFileSync } from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +export const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..'); +export const REGISTRY_FILE = path.join(ROOT, 'scripts', 'instances.json'); + +const LEGACY = [{ + name: 'autofirmer', + port: Number(process.env.AUTOFIRMER_PORT ?? '3000'), + db: 'autotrader.sqlite', +}]; + +export function loadInstances() { + if (!existsSync(REGISTRY_FILE)) return LEGACY; + let parsed; + try { + parsed = JSON.parse(readFileSync(REGISTRY_FILE, 'utf8')); + } catch (err) { + throw new Error(`scripts/instances.json is not valid JSON: ${err.message}`); + } + if (!Array.isArray(parsed) || parsed.length === 0) return LEGACY; + for (const e of parsed) { + if (typeof e?.name !== 'string' || !e.name + || !Number.isInteger(e?.port) + || typeof e?.db !== 'string' || !e.db) { + throw new Error(`scripts/instances.json entry is malformed: ${JSON.stringify(e)}`); + } + } + return parsed; +} diff --git a/scripts/remove-instance.sh b/scripts/remove-instance.sh new file mode 100755 index 0000000..b8e62ae --- /dev/null +++ b/scripts/remove-instance.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +# +# Unregister an instance added by add-instance.sh: stops its PM2 process and +# drops it from scripts/instances.json. The database directory is left on +# disk — deleting credentials is a decision for a human with rm. +# +# ./scripts/remove-instance.sh +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +REGISTRY="$ROOT/scripts/instances.json" + +info() { printf ' %s\n' "$1"; } +die() { printf ' [X] %s\n' "$1" >&2; exit 1; } + +[ $# -eq 1 ] || die "usage: $0 " +N="$1" +case "$N" in (*[!0-9]*|'') die "N must be a number, got: $N" ;; esac +[ "$N" -ge 2 ] || die "instance 1 is the legacy install - remove it with pm2 delete autofirmer" +[ -f "$REGISTRY" ] || die "no scripts/instances.json - nothing to remove" + +PM2_NAME="autofirmer-$N" + +node - "$REGISTRY" "$PM2_NAME" <<'NODE' +const { readFileSync, writeFileSync } = require('node:fs'); +const [, file, name] = process.argv; +const entries = JSON.parse(readFileSync(file, 'utf8')); +if (!entries.some((e) => e.name === name)) { console.error(` [X] ${name} is not registered`); process.exit(1); } +writeFileSync(file, JSON.stringify(entries.filter((e) => e.name !== name), null, 4) + '\n'); +console.log(` registry: ${name} removed`); +NODE + +pm2 delete "$PM2_NAME" >/dev/null 2>&1 || info "pm2: $PM2_NAME was not running" +pm2 save >/dev/null +info "pm2: $PM2_NAME deleted and process list saved" +info "database left at .data/instance-$N/ - delete by hand if you mean it" diff --git a/scripts/seed-settings.js b/scripts/seed-settings.js index 2f11357..46c7af0 100644 --- a/scripts/seed-settings.js +++ b/scripts/seed-settings.js @@ -1,31 +1,174 @@ /** - * Writes the reporter settings into autotrader.sqlite before first launch. + * Writes the reporter settings into the instance database before first launch, + * and optionally restores a full backup from the master. * - * lib/db.ts seeds these keys with INSERT OR IGNORE, so values written here - * survive the app's own startup seeding. Must be run from the project root — - * lib/db.ts opens the database at process.cwd(). + * lib/db.ts seeds its keys with INSERT OR IGNORE, so values written here + * survive the app's own startup seeding. Must be run from the project root + * unless AUTOFIRMER_DB points elsewhere — lib/db.ts opens the database at + * process.cwd() by default, and this script must hit the same file. * - * node scripts/seed-settings.js + * node scripts/seed-settings.js \ + * [--token ] [--restore-from ] + * + * --restore-from pulls that instance's encrypted backup from the master + * (authenticated by --token) and applies it — settings, firm credentials, + * instruments, account assignments — before the app has ever booted, so the + * very first hello to the master already presents the adopted uuid. The + * backup's instance_name wins over the CLI argument. */ const path = require('path'); +const { mkdirSync } = require('fs'); const Database = require('better-sqlite3'); -const [, , url, name] = process.argv; +const args = process.argv.slice(2); +const positional = []; +let token = ''; +let restoreFrom = ''; +for (let i = 0; i < args.length; i++) { + if (args[i] === '--token') token = args[++i] ?? ''; + else if (args[i] === '--restore-from') restoreFrom = args[++i] ?? ''; + else positional.push(args[i]); +} +const [url, name] = positional; -const db = new Database(path.join(process.cwd(), 'autotrader.sqlite')); +const DB_PATH = process.env.AUTOFIRMER_DB + ? path.resolve(process.env.AUTOFIRMER_DB) + : path.join(process.cwd(), 'autotrader.sqlite'); + +mkdirSync(path.dirname(DB_PATH), { recursive: true }); +const db = new Database(DB_PATH); +db.pragma('journal_mode = WAL'); +db.pragma('busy_timeout = 5000'); + +// Mirrors the schema in lib/db.ts for every table a restore touches. The app +// has usually never run at this point, so the tables may not exist yet; the +// shapes must be kept in sync with lib/db.ts by hand. db.exec(` CREATE TABLE IF NOT EXISTS settings ( key TEXT PRIMARY KEY, value TEXT NOT NULL ); + + CREATE TABLE IF NOT EXISTS firm_credentials ( + firm_id TEXT PRIMARY KEY, + firm_name TEXT NOT NULL, + username TEXT NOT NULL, + password TEXT NOT NULL, + updated_at TEXT NOT NULL + ); + + CREATE TABLE IF NOT EXISTS instruments ( + symbol TEXT PRIMARY KEY, + enabled INTEGER NOT NULL DEFAULT 1 + ); + + CREATE TABLE IF NOT EXISTS account_assignments ( + account_id INTEGER PRIMARY KEY, + firm_id TEXT NOT NULL, + account_type_id TEXT, + manual INTEGER NOT NULL DEFAULT 0, + resolved_at TEXT NOT NULL + ); `); +try { + db.exec("ALTER TABLE firm_credentials ADD COLUMN api_key TEXT NOT NULL DEFAULT ''"); +} catch { + // Column already exists +} const upsert = db.prepare( 'INSERT INTO settings (key, value) VALUES (?, ?) ' + 'ON CONFLICT(key) DO UPDATE SET value = excluded.value' ); -if (url) { upsert.run('master_dashboard_url', url); console.log(' master_dashboard_url = ' + url); } -if (name) { upsert.run('instance_name', name); console.log(' instance_name = ' + name); } +if (url) { upsert.run('master_dashboard_url', url); console.log(' master_dashboard_url = ' + url); } +if (name) { upsert.run('instance_name', name); console.log(' instance_name = ' + name); } +if (token) { upsert.run('master_backup_token', token); console.log(' master_backup_token = (set)'); } -db.close(); +// Runtime bookkeeping keys that describe the machine the backup was taken on, +// not the one being provisioned. Kept in sync with lib/backup-restore.ts. +const RESTORE_SKIP_KEYS = new Set([ + 'runner_heartbeat', + 'catalog_last_sync_at', + 'catalog_last_error', + 'backup_last_push_at', + 'backup_last_error', +]); + +async function restore() { + if (!restoreFrom) return; + if (!url) { console.error('--restore-from needs the master url argument'); process.exit(1); } + if (!token) { console.error('--restore-from needs --token'); process.exit(1); } + + const endpoint = new URL(`/api/instances/backup/${restoreFrom}`, url); + console.log(` restoring from ${endpoint}`); + const res = await fetch(endpoint, { + headers: { Authorization: `Bearer ${token}` }, + signal: AbortSignal.timeout(15_000), + }); + if (!res.ok) { + console.error(` restore failed: master answered ${res.status} ${await res.text().catch(() => '')}`); + process.exit(1); + } + const payload = await res.json(); + if (payload?.v !== 1 || typeof payload.settings !== 'object') { + console.error(' restore failed: unrecognized backup payload'); + process.exit(1); + } + + const setCred = db.prepare(` + INSERT INTO firm_credentials (firm_id, firm_name, username, password, api_key, updated_at) + VALUES (?, ?, ?, ?, ?, ?) + ON CONFLICT(firm_id) DO UPDATE SET + firm_name = excluded.firm_name, + username = excluded.username, + password = excluded.password, + api_key = excluded.api_key, + updated_at = excluded.updated_at + `); + const setInstr = db.prepare( + 'INSERT INTO instruments (symbol, enabled) VALUES (?, ?) ' + + 'ON CONFLICT(symbol) DO UPDATE SET enabled = excluded.enabled' + ); + const setAssign = db.prepare(` + INSERT INTO account_assignments (account_id, firm_id, account_type_id, manual, resolved_at) + VALUES (?, ?, ?, ?, ?) + ON CONFLICT(account_id) DO UPDATE SET + firm_id = excluded.firm_id, + account_type_id = excluded.account_type_id, + manual = excluded.manual, + resolved_at = excluded.resolved_at + `); + + db.transaction(() => { + for (const [key, value] of Object.entries(payload.settings)) { + if (RESTORE_SKIP_KEYS.has(key)) continue; + if (typeof value !== 'string') continue; + upsert.run(key, value); + } + // The freshly seeded master url wins over whatever the backup recorded: + // the operator just typed it, and an old backup may point at a dead host. + if (url) upsert.run('master_dashboard_url', url); + if (token) upsert.run('master_backup_token', token); + + for (const c of payload.credentials ?? []) { + setCred.run(c.firm_id, c.firm_name, c.username, c.password, c.api_key ?? '', c.updated_at ?? new Date().toISOString()); + } + for (const i of payload.instruments ?? []) { + setInstr.run(i.symbol, i.enabled ? 1 : 0); + } + for (const a of payload.assignments ?? []) { + setAssign.run(a.account_id, a.firm_id, a.account_type_id ?? null, a.manual ? 1 : 0, a.resolved_at ?? new Date().toISOString()); + } + })(); + + console.log(` restored: ${Object.keys(payload.settings).length} settings, ` + + `${(payload.credentials ?? []).length} credentials, ` + + `${(payload.instruments ?? []).length} instruments, ` + + `${(payload.assignments ?? []).length} assignments`); + console.log(` adopted identity: "${payload.settings.instance_name ?? ''}" (${payload.settings.instance_id ?? 'no uuid'})`); +} + +restore() + .catch((err) => { console.error(` restore failed: ${err.message}`); process.exit(1); }) + .finally(() => db.close()); diff --git a/scripts/start-all.mjs b/scripts/start-all.mjs index 32e43a4..70bba17 100644 --- a/scripts/start-all.mjs +++ b/scripts/start-all.mjs @@ -1,5 +1,6 @@ /** - * Bring an instance up: clicker dependencies, PM2 processes, then a warm-up. + * Bring the device up: clicker dependencies, PM2 processes, then a warm-up of + * every registered instance. * * Run by the "at log on" scheduled task. Safe to run by hand at any time. * @@ -9,13 +10,8 @@ * HTTP request the process sits idle and none of that happens. */ import { spawnSync } from 'node:child_process'; -import path from 'node:path'; -import { fileURLToPath } from 'node:url'; import { ensurePythonDeps } from './ensure-python-deps.mjs'; - -const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..'); -const PORT = process.env.AUTOFIRMER_PORT ?? '3000'; -const BASE = `http://127.0.0.1:${PORT}`; +import { ROOT, loadInstances } from './instances.mjs'; const log = (msg) => console.log(`[start-all] ${msg}`); @@ -28,11 +24,11 @@ function run(cmd, args) { }); } -async function waitForApp(timeoutMs = 120_000) { +async function waitForApp(base, timeoutMs = 120_000) { const deadline = Date.now() + timeoutMs; while (Date.now() < deadline) { try { - const res = await fetch(`${BASE}/api/auto-trade`, { signal: AbortSignal.timeout(5_000) }); + const res = await fetch(`${base}/api/auto-trade`, { signal: AbortSignal.timeout(5_000) }); if (res.ok) return true; } catch { /* not up yet */ } await new Promise((r) => setTimeout(r, 2_000)); @@ -50,25 +46,32 @@ if (run('pm2', ['resurrect']).status !== 0) { process.exit(1); } -// ── warm-up ───────────────────────────────────────────────────────────────── -log('waiting for the dashboard to answer'); -if (!(await waitForApp())) { - log(`dashboard did not come up on ${BASE} within 120s — check \`pm2 logs autofirmer\``); - process.exit(1); +// ── warm-up, one instance at a time ───────────────────────────────────────── +let failed = false; +for (const instance of loadInstances()) { + const base = `http://127.0.0.1:${instance.port}`; + + log(`waiting for ${instance.name} to answer on ${base}`); + if (!(await waitForApp(base))) { + log(`${instance.name} did not come up on ${base} within 120s — check \`pm2 logs ${instance.name}\``); + failed = true; + continue; + } + + log(`warming up ${instance.name} (this is what triggers the client bootstrap)`); + try { + await fetch(`${base}/api/state`, { signal: AbortSignal.timeout(60_000) }); + } catch (err) { + log(`warm-up request failed: ${err.message}`); + } + + try { + const status = await (await fetch(`${base}/api/auto-trade`)).json(); + log(status.running + ? `${instance.name}: scheduler resumed — ${status.action} ${status.symbol}` + : `${instance.name}: scheduler is stopped`); + } catch { /* non-critical */ } } -log('warming up (this is what triggers the client bootstrap)'); -try { - await fetch(`${BASE}/api/state`, { signal: AbortSignal.timeout(60_000) }); -} catch (err) { - log(`warm-up request failed: ${err.message}`); -} - -try { - const status = await (await fetch(`${BASE}/api/auto-trade`)).json(); - log(status.running - ? `scheduler resumed — ${status.action} ${status.symbol}` - : 'scheduler is stopped'); -} catch { /* non-critical */ } - +if (failed) process.exit(1); log('up'); diff --git a/scripts/update-check.mjs b/scripts/update-check.mjs index 641a863..2c3ebc4 100644 --- a/scripts/update-check.mjs +++ b/scripts/update-check.mjs @@ -16,14 +16,16 @@ import { appendFileSync, existsSync, mkdirSync, readFileSync, rmSync, statSync, import path from 'node:path'; import { fileURLToPath } from 'node:url'; import { ensurePythonDeps } from './ensure-python-deps.mjs'; +import { loadInstances } from './instances.mjs'; const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..'); const LOG_FILE = path.join(ROOT, 'scripts', 'update.log'); const LOCK_FILE = path.join(ROOT, 'scripts', '.update.lock'); const LAST_CHECK_FILE = path.join(ROOT, 'scripts', '.last-check'); const BRANCH = process.env.AUTOFIRMER_BRANCH ?? 'master'; -const PORT = process.env.AUTOFIRMER_PORT ?? '3000'; -const BASE = `http://127.0.0.1:${PORT}`; +// One checkout, one build — but possibly several PM2 processes to restart and +// warm afterwards (see scripts/instances.mjs). +const INSTANCES = loadInstances(); const DRY_RUN = process.argv.includes('--dry-run'); const STALE_LOCK_MS = 60 * 60 * 1000; const LOG_MAX_BYTES = 256 * 1024; @@ -126,7 +128,7 @@ async function main() { changed.includes('package-lock.json') && 'npm install', changed.includes('clicker/requirements.txt') && 'pip install', 'npm run build', - 'pm2 restart autofirmer', + ...INSTANCES.map((i) => `pm2 restart ${i.name}`), changed.some((f) => f.startsWith('clicker/')) && 'pm2 restart clicker', ].filter(Boolean).join(', ')}`); return 0; @@ -153,8 +155,10 @@ async function main() { return 1; } - log('restarting autofirmer'); - if (run('pm2', ['restart', 'autofirmer']).status !== 0) { log('pm2 restart autofirmer failed'); outcome = 'pm2 restart failed'; return 1; } + for (const instance of INSTANCES) { + log(`restarting ${instance.name}`); + if (run('pm2', ['restart', instance.name]).status !== 0) { log(`pm2 restart ${instance.name} failed`); outcome = 'pm2 restart failed'; return 1; } + } if (changed.some((f) => f.startsWith('clicker/'))) { log('clicker changed — restarting it too'); @@ -178,21 +182,29 @@ async function main() { } async function warmUp() { - const deadline = Date.now() + 120_000; - while (Date.now() < deadline) { + for (const instance of INSTANCES) { + const base = `http://127.0.0.1:${instance.port}`; + const deadline = Date.now() + 120_000; + while (Date.now() < deadline) { + try { + if ((await fetch(`${base}/api/auto-trade`, { signal: AbortSignal.timeout(5_000) })).ok) break; + } catch { /* still restarting */ } + await new Promise((r) => setTimeout(r, 2_000)); + } try { - if ((await fetch(`${BASE}/api/auto-trade`, { signal: AbortSignal.timeout(5_000) })).ok) break; - } catch { /* still restarting */ } - await new Promise((r) => setTimeout(r, 2_000)); - } - try { - await fetch(`${BASE}/api/state`, { signal: AbortSignal.timeout(60_000) }); - const status = await (await fetch(`${BASE}/api/auto-trade`)).json(); - log(status.running ? `scheduler resumed — ${status.action} ${status.symbol}` : 'scheduler is stopped'); - const runner = await (await fetch(`${BASE}/api/autobuyer/runner`)).json(); - log(`runner ${runner.online ? 'online' : 'OFFLINE'}${runner.stale ? ' (version stale — reload the extension)' : ''}`); - } catch (err) { - log(`post-restart check failed: ${err.message}`); + await fetch(`${base}/api/state`, { signal: AbortSignal.timeout(60_000) }); + const status = await (await fetch(`${base}/api/auto-trade`)).json(); + log(status.running + ? `${instance.name}: scheduler resumed — ${status.action} ${status.symbol}` + : `${instance.name}: scheduler is stopped`); + // The clicker/extension runner only ever serves instance 1. + if (instance.port === 3000) { + const runner = await (await fetch(`${base}/api/autobuyer/runner`)).json(); + log(`runner ${runner.online ? 'online' : 'OFFLINE'}${runner.stale ? ' (version stale — reload the extension)' : ''}`); + } + } catch (err) { + log(`${instance.name}: post-restart check failed: ${err.message}`); + } } } diff --git a/setup-linux.sh b/setup-linux.sh index d8e3e16..b66d720 100755 --- a/setup-linux.sh +++ b/setup-linux.sh @@ -86,6 +86,11 @@ read -r -p " Master dashboard URL [$DEFAULT_MASTER]: " MASTER_URL MASTER_URL="${MASTER_URL:-$DEFAULT_MASTER}" read -r -p " Instance name [$(hostname)]: " INSTANCE INSTANCE="${INSTANCE:-$(hostname)}" +read -r -p " Backup token (blank = no master backups): " BACKUP_TOKEN +RESTORE_FROM="" +if [ -n "$BACKUP_TOKEN" ]; then + read -r -p " Restore a dead instance? Its uuid (blank = fresh install): " RESTORE_FROM +fi echo info "installing to: $TARGET" echo @@ -115,7 +120,13 @@ npm run build # ── 4. settings ───────────────────────────────────────────────────────────── info "[4/7] writing instance settings..." -node scripts/seed-settings.js "$MASTER_URL" "$INSTANCE" +SEED_ARGS=() +[ -n "$BACKUP_TOKEN" ] && SEED_ARGS+=(--token "$BACKUP_TOKEN") +# --restore-from pulls the dead instance's backup (settings, credentials, +# identity) before the app ever boots, so its very first hello to the master +# presents the adopted uuid. Stop the old machine first if it still runs. +[ -n "$RESTORE_FROM" ] && SEED_ARGS+=(--restore-from "$RESTORE_FROM") +node scripts/seed-settings.js "$MASTER_URL" "$INSTANCE" ${SEED_ARGS[@]+"${SEED_ARGS[@]}"} # ── 5. python ─────────────────────────────────────────────────────────────── # A virtualenv rather than a system pip install: Raspberry Pi OS Bookworm diff --git a/yarn.lock b/yarn.lock index 8719efb..41c9f95 100644 --- a/yarn.lock +++ b/yarn.lock @@ -667,6 +667,11 @@ acorn-jsx@^5.3.2: resolved "https://registry.npmjs.org/acorn/-/acorn-8.16.0.tgz" integrity sha512-UVJyE9MttOsBQIDKw1skb9nAwQuR5wuGD3+82K6JgJlm/Y+KI92oNsMNGZCYdDsVtRHSak0pcV5Dno5+4jh9sw== +agent-base@9.0.0: + version "9.0.0" + resolved "https://registry.npmjs.org/agent-base/-/agent-base-9.0.0.tgz" + integrity sha512-TQf59BsZnytt8GdJKLPfUZ54g/iaUL2OWDSFCCvMOhsHduDQxO8xC4PNeyIkVcA5KwL2phPSv0douC0fgWzmnA== + ajv@^6.14.0: version "6.14.0" resolved "https://registry.npmjs.org/ajv/-/ajv-6.14.0.tgz" @@ -1123,7 +1128,7 @@ debug@^3.2.7: dependencies: ms "^2.1.1" -debug@^4.1.0, debug@^4.3.1, debug@^4.3.2, debug@^4.4.0, debug@^4.4.3: +debug@^4.1.0, debug@^4.3.1, debug@^4.3.2, debug@^4.3.4, debug@^4.4.0, debug@^4.4.3: version "4.4.3" resolved "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz" integrity sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA== @@ -1648,6 +1653,14 @@ fetch-cookie@^2.0.3: set-cookie-parser "^2.4.8" tough-cookie "^4.0.0" +fetch-socks@^1.3.3: + version "1.3.3" + resolved "https://registry.npmjs.org/fetch-socks/-/fetch-socks-1.3.3.tgz" + integrity sha512-jI22jzTf7x3GAcUoZARwuxKBPotb3PFkemVWqxlF2qV55FhRytRJrn95HeLoUMq2vkIbY3aQqNYytvZKqUoMaQ== + dependencies: + socks "^2.8.7" + undici ">=7" + file-entry-cache@^8.0.0: version "8.0.0" resolved "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-8.0.0.tgz" @@ -1890,6 +1903,24 @@ hermes-parser@^0.25.1: dependencies: hermes-estree "0.25.1" +http-proxy-agent@^9.1.0: + version "9.1.0" + resolved "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-9.1.0.tgz" + integrity sha512-2NxoveTT58mjYT4n3RPTEfCZGLMbidoO8XEieXfpSYxu+PQJ1qpx4ypwH6N+uF9twBPIvRRgvkvW5HUTYWENig== + dependencies: + agent-base "9.0.0" + debug "^4.3.4" + proxy-agent-negotiate "1.1.0" + +https-proxy-agent@^9.1.0: + version "9.1.0" + resolved "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-9.1.0.tgz" + integrity sha512-ag87y7cJJ9/3+GxFr8Oy4O5faDsGRGnBGsJj/YjOSsSx/5eadKLYTMPlzuR6obgoCDDm0abAAZitXXQkMOPSpA== + dependencies: + agent-base "9.0.0" + debug "^4.3.4" + proxy-agent-negotiate "1.1.0" + ieee754@^1.1.13: version "1.2.1" resolved "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz" @@ -1952,6 +1983,11 @@ internal-slot@^1.1.0: resolved "https://registry.npmjs.org/internmap/-/internmap-2.0.3.tgz" integrity sha512-5Hh7Y1wQbvY5ooGgPbDaL5iYLAPzMTUrjMulskHLH6wnv/A+1q5rgEaiuqEjB+oxGXIVZs1FF+R/KPN3ZSQYYg== +ip-address@^10.1.1: + version "10.7.2" + resolved "https://registry.npmjs.org/ip-address/-/ip-address-10.7.2.tgz" + integrity sha512-7H/2gFSIitxc0hG3nOI1glS8QLo/EHBFFLk8vEUjXY/xu0AdL8jZ9U1IzO2PUm0d2D/ofQcAifb0g6OBkt8U7w== + is-array-buffer@^3.0.4, is-array-buffer@^3.0.5: version "3.0.5" resolved "https://registry.npmjs.org/is-array-buffer/-/is-array-buffer-3.0.5.tgz" @@ -2642,6 +2678,11 @@ prop-types@^15.8.1: object-assign "^4.1.1" react-is "^16.13.1" +proxy-agent-negotiate@1.1.0: + version "1.1.0" + resolved "https://registry.npmjs.org/proxy-agent-negotiate/-/proxy-agent-negotiate-1.1.0.tgz" + integrity sha512-N8IBcM3UgCVzz2L2Lqv8DVntDnnC8/hiV4nEDUPkqq72TPUgYWjQc+bdZlBPZK9LzPAvOY//gAt0S0DApoOXWQ== + proxy-from-env@^1.1.0: version "1.1.0" resolved "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz" @@ -3021,6 +3062,28 @@ simple-get@^4.0.0: once "^1.3.1" simple-concat "^1.0.0" +smart-buffer@^4.2.0: + version "4.2.0" + resolved "https://registry.npmjs.org/smart-buffer/-/smart-buffer-4.2.0.tgz" + integrity sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg== + +socks-proxy-agent@^10.1.0: + version "10.1.0" + resolved "https://registry.npmjs.org/socks-proxy-agent/-/socks-proxy-agent-10.1.0.tgz" + integrity sha512-WlMj/67cEJ6MDI1OcsnjuYKDNDoyPCCYZ249kuuXPiMDw9F8PXkVaQ7YWu3siTydfQ/4BEZcvGzu+aYvz7dDCQ== + dependencies: + agent-base "9.0.0" + debug "^4.3.4" + socks "^2.8.3" + +socks@^2.8.3, socks@^2.8.7: + version "2.8.10" + resolved "https://registry.npmjs.org/socks/-/socks-2.8.10.tgz" + integrity sha512-e0VyvkVTwVYViNovRkZ9aodhxVlyoMn7eJhVUPxZ+eK9P/7CBkxvvsBOHqFPEH416726W8tLXXXjKwqgTErrCQ== + dependencies: + ip-address "^10.1.1" + smart-buffer "^4.2.0" + source-map-js@^1.0.2, source-map-js@^1.2.1: version "1.2.1" resolved "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz" @@ -3323,6 +3386,11 @@ undici-types@~6.21.0: resolved "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz" integrity sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ== +undici@^7.29.1, undici@>=7: + version "7.29.1" + resolved "https://registry.npmjs.org/undici/-/undici-7.29.1.tgz" + integrity sha512-RYONW2MeafgYlkVOKYKkA/Ag7BmXqgIWCa8t1m0JcxrQg9pI9lEqRhAOruOBCbAohOa/gkCF+iPi9hrgvTzu6Q== + universalify@^0.2.0: version "0.2.0" resolved "https://registry.npmjs.org/universalify/-/universalify-0.2.0.tgz" @@ -3496,6 +3564,11 @@ ws@^7.5.10: resolved "https://registry.npmjs.org/ws/-/ws-7.5.13.tgz" integrity sha512-rsKI6xDBFVf4r/x8XyChGK04QR/XHroxs/jUcoWvtEZM8TPU/X/uIY9B1CsSzYws9ZJb/6bbBu7dPhFW00CAoA== +ws@^8.21.3: + version "8.21.3" + resolved "https://registry.npmjs.org/ws/-/ws-8.21.3.tgz" + integrity sha512-201TZ/kPWxoPr/OKWjquZR1SWKXcvxdH+e1xrx89b3YbmzLMFCLfnaG1HFIgWzJOEWZ7MvpK++odZufgYR50Rw== + yallist@^3.0.2: version "3.1.1" resolved "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz"