autofirmer-expanded/lib/updater-watchdog.ts

/**
 * Keeps this machine's auto-updater (scripts/update-check.mjs) alive.
 *
 * PM2 fires the update checker every minute, and nothing on the machine
 * notices when it stops. On the fleet it stops on its own: PM2's cron dies
 * inside a live daemon (six Windows machines in October 2026; the mechanism
 * is in update-check.mjs heal), and a `pm2 stop` or a reboot after an unsaved
 * process-list change does the same. The master's dashboard notices ("updater
 * not running"), but then someone has to get onto the machine. The app is a
 * long-running PM2 process, which PM2 keeps alive reliably, so it checks: once
 * the last check is older than the master's 10 minutes, it asks the script to
 * heal, i.e. get PM2 firing it again, and run a check.
 *
 * It only revives an updater that has run here before (scripts/.last-check
 * exists). A checkout that never had one is someone's choice, and
 * scripts/.no-auto-update switches a checkout off outright (a dev checkout).
 * Only the checkout's first instance watches — PM2 name `autofirmer`, which
 * scripts/instances.mjs keeps for instance 1 — since the others share its
 * updater.
 *
 * The master's "Update now" command comes through here as well.
 */
import { spawn } from 'node:child_process';
import { existsSync, readFileSync, statSync } from 'node:fs';
import path from 'node:path';

const EVERY_MS = 5 * 60_000;
/** The master's UPDATER_STALE_MS: past this its dashboard says "not running". */
const STALE_MS = 10 * 60_000;
/** A heal that didn't take (PM2 missing, say) is retried no sooner than this. */
const RETRY_MS = 30 * 60_000;

const scriptsDir = () => path.join(process.cwd(), 'scripts');
const optedOut = () => existsSync(path.join(scriptsDir(), '.no-auto-update'));

// Per process: instrumentation.ts and the route handlers each load this module.
const g = globalThis as typeof globalThis & { __updaterWatchdog?: { lastHealAt: number } };

/** When the updater last checked (epoch ms), or null if it never has here. */
function lastCheckAt(): number | null {
    const file = path.join(scriptsDir(), '.last-check');
    try {
        // "<ISO time>  <status>", rewritten on every check.
        const at = Date.parse(readFileSync(file, 'utf8').split(/\s/, 1)[0]);
        return Number.isFinite(at) ? at : statSync(file).mtimeMs;
    } catch {
        return null;
    }
}

/**
 * Start update-check.mjs on its own. Detached, because what it starts may
 * restart this very process; the script logs to scripts/update.log itself.
 */
function spawnUpdateCheck(args: string[]): number {
    const child = spawn(process.execPath, [path.join(scriptsDir(), 'update-check.mjs'), ...args], {
        cwd: process.cwd(),
        detached: true,
        // No console window popping up on a Windows trading desktop.
        windowsHide: true,
        stdio: 'ignore',
    });
    child.on('error', (err) => console.error(`[updater-watchdog] update-check.mjs failed to start: ${err.message}`));
    child.unref();
    if (!child.pid) throw new Error('could not start scripts/update-check.mjs');
    return child.pid;
}

function tick(): void {
    const state = g.__updaterWatchdog;
    if (!state || optedOut()) return;
    const last = lastCheckAt();
    if (last === null) return;                       // never ran here: not ours to start
    const now = Date.now();
    if (now - last <= STALE_MS || now - state.lastHealAt < RETRY_MS) return;
    state.lastHealAt = now;
    try {
        const pid = spawnUpdateCheck(['--heal']);
        console.log(`[updater-watchdog] last update check ${Math.round((now - last) / 60_000)} min ago — asked update-check.mjs to heal (pid ${pid})`);
    } catch (err) {
        console.error(`[updater-watchdog] ${err instanceof Error ? err.message : String(err)}`);
    }
}

export function startUpdaterWatchdog(): void {
    if (g.__updaterWatchdog) return;
    // Under PM2 only (pm_id is PM2's): `next dev`, a sandbox copy or a
    // by-hand `next start` has no updater to keep alive.
    if (process.env.pm_id === undefined || process.env.name !== 'autofirmer') return;
    g.__updaterWatchdog = { lastHealAt: 0 };
    setInterval(tick, EVERY_MS).unref();
}

/**
 * The master's "Update now": heal the updater if it stopped, then check at
 * once, skipping a failed update's retry wait. Answers before the update
 * runs — it may restart this process — so its outcome reaches the master
 * with the next report's updater status.
 */
export function runUpdateNow(): { started: true; pid: number } {
    if (optedOut()) throw new Error('updates are switched off on this machine (scripts/.no-auto-update)');
    return { started: true, pid: spawnUpdateCheck(['--heal', '--now']) };
}