/** * Keep this checkout on master: fetch, and when something needs doing, pull, * rebuild and restart, without ever leaving a deploy half-applied. * * PM2 fires this every minute (--no-autorestart --cron-restart, registered by * install-autostart.*), and that schedule dictates the shape of the script. * PM2's cron handler *restarts* the process, and a restart first stops a * process that is still running: it is tree-killed at the next minute boundary. * So anything that outlasts ~55 s (npm install, next build) used to die * half-way through, and a kill skips `finally`. The lock stayed behind, HEAD * had already moved at the pull (so the next run saw nothing to pull and never * retried), and next build had already wiped .next under the live server. * * Hence two halves: * * CHECK (no arguments; what the cron runs) takes seconds. It fetches, decides * whether anything needs doing, hands the work to a detached worker and * exits. PM2 sees the process stop on its own, so the next tick has nothing * to kill, and the worker, orphaned once this process exits, is outside the * process tree PM2 kills. * * APPLY (--apply) is the worker, and is also safe to run by hand in the * foreground. It takes the lock, re-checks under it, then pulls, installs, * builds, restarts and warms up. * * "Needs doing" is judged against scripts/.built-commit, the commit .next was * last built from, not against HEAD. HEAD moves at the pull, before the work * that makes it real, so HEAD === origin says nothing about whether the build * finished. * * A broken push still cannot take a trading PC down. The build runs before * anything is restarted, the previous build is set aside first and put back if * the build fails, and a failure is retried on a backoff (scripts/.update-failed) * rather than every minute. * * A machine whose checker stopped firing can't notice by itself, so the app * asks for a heal (lib/updater-watchdog.ts) once the last check goes stale, and * the master's "Update now" asks for a heal plus an immediate check. * * node scripts/update-check.mjs # check; start a worker if needed * node scripts/update-check.mjs --apply # update now, here, in the foreground * node scripts/update-check.mjs --dry-run # report only, change nothing * node scripts/update-check.mjs --heal # re-register a checker that stopped, then check * node scripts/update-check.mjs --now # check without waiting out a failure's backoff */ import { spawn, spawnSync } from 'node:child_process'; import { appendFileSync, closeSync, existsSync, lstatSync, mkdirSync, openSync, readdirSync, readFileSync, renameSync, rmdirSync, rmSync, statSync, unlinkSync, writeFileSync, writeSync, } from 'node:fs'; import os from 'node:os'; import path from 'node:path'; import { fileURLToPath } from 'node:url'; import { ensurePythonDeps } from './ensure-python-deps.mjs'; import { loadInstances } from './instances.mjs'; const SELF = fileURLToPath(import.meta.url); const ROOT = path.resolve(path.dirname(SELF), '..'); const LOG_FILE = path.join(ROOT, 'scripts', 'update.log'); const WORKER_LOG_FILE = path.join(ROOT, 'scripts', 'update-worker.log'); const LOCK_FILE = path.join(ROOT, 'scripts', '.update.lock'); const LAST_CHECK_FILE = path.join(ROOT, 'scripts', '.last-check'); const BUILT_FILE = path.join(ROOT, 'scripts', '.built-commit'); const FAILED_FILE = path.join(ROOT, 'scripts', '.update-failed'); const NEXT_DIR = path.join(ROOT, '.next'); const PREV_DIR = path.join(ROOT, '.next-prev'); const BUILD_ID_FILE = path.join(NEXT_DIR, 'BUILD_ID'); const BRANCH = process.env.AUTOFIRMER_BRANCH ?? 'master'; // One checkout, one build — but possibly several PM2 processes to restart and // warm afterwards (see scripts/instances.mjs). const INSTANCES = loadInstances(); const APPLY = process.argv.includes('--apply'); const DRY_RUN = process.argv.includes('--dry-run'); const HEAL = process.argv.includes('--heal'); const NOW = process.argv.includes('--now'); const OPT_OUT_FILE = path.join(ROOT, 'scripts', '.no-auto-update'); // The PM2 app install-autostart.* registers, and its schedule. const CHECKER = 'autofirmer-update'; const DEFAULT_CRON = '*/1 * * * *'; // The master's UPDATER_STALE_MS: past this its dashboard says "updater not running". const STALE_CHECK_MS = 10 * 60 * 1000; const STALE_LOCK_MS = 2 * 60 * 60 * 1000; const LEGACY_STALE_LOCK_MS = 10 * 60 * 1000; const LOCK_SETTLE_MS = 1000; const FIRST_RETRY_MS = 15 * 60 * 1000; const REPEAT_RETRY_MS = 60 * 60 * 1000; // Bounds on the steps that wait on something else (the network, the PM2 // daemon). Without them a stalled one holds the worker, and the lock, until the // lock goes stale by age, and the worker that takes over then runs alongside // it. npm and next build are left unbounded: on Windows a timeout would kill // only the cmd.exe in front of them, and an orphaned build still writing into // .next is worse than a slow one. // The fetch also stays under the cron's minute: a check still running at the next // fire has to be killed by PM2, and that kill is where PM2 loses the cron // (see heal). const FETCH_TIMEOUT_MS = 45 * 1000; const PM2_TIMEOUT_MS = 3 * 60 * 1000; const LOG_MAX_BYTES = 256 * 1024; const LOG_KEEP_LINES = 500; const short = (commit) => (commit ? commit.slice(0, 8) : 'unknown'); /** * Keep update.log from growing without bound. * * Trimmed once per run, before anything is written, rather than on every line: * the no-change path writes nothing at all, so this is the only moment the file * can have grown since last time. Housekeeping must never break an update, so * any failure here is swallowed. */ function trimLog() { try { if (!existsSync(LOG_FILE) || statSync(LOG_FILE).size <= LOG_MAX_BYTES) return; const kept = readFileSync(LOG_FILE, 'utf8').split('\n').filter(Boolean).slice(-LOG_KEEP_LINES); writeFileSync(LOG_FILE, kept.join('\n') + '\n'); } catch { /* ignore */ } } function log(msg) { const line = `${new Date().toISOString()} ${msg}`; console.log(line); try { appendFileSync(LOG_FILE, line + '\n'); } catch { /* logging must never throw */ } } /** * Record that a check happened, whether or not it found anything. * * Overwritten rather than appended, so it stays one line and needs no trimming. * Without it there is no way to tell "running, nothing to pull" from "not * running at all": the no-change path logs nothing by design, and PM2 reports a * cron-restart process as `stopped` with a restart count of 0 even while it is * firing on schedule. */ function recordCheck(status) { try { writeFileSync(LAST_CHECK_FILE, `${new Date().toISOString()} ${status}\n`); } catch { /* never let bookkeeping break an update */ } } /** * An argument for run() as cmd.exe will see it. On Windows run() goes through * the shell, which joins the arguments with plain spaces, so a cron string or * a path with a space in it would arrive as several. */ const shellArg = (arg) => (process.platform === 'win32' && /[\s"]/.test(arg) ? `"${arg.replace(/"/g, '""')}"` : arg); function run(cmd, args, { capture = false, timeout } = {}) { return spawnSync(cmd, args, { cwd: ROOT, encoding: 'utf8', stdio: capture ? 'pipe' : 'inherit', shell: process.platform === 'win32', // The worker is detached, so on Windows it has no console of its own // and every npm/git/pm2 child would otherwise open a visible console // window on the trading desktop, stealing focus from the clicker. windowsHide: true, maxBuffer: 16 * 1024 * 1024, timeout, }); } function git(...args) { const r = run('git', args, { capture: true }); if (r.status !== 0) throw new Error(`git ${args.join(' ')} failed: ${(r.stderr ?? '').trim()}`); return (r.stdout ?? '').trim(); } /** git fetch, bounded: git sets no network timeout of its own, so a stalled * connection would otherwise wait forever (see FETCH_TIMEOUT_MS). */ function fetchOrigin() { const r = run('git', ['fetch', 'origin', BRANCH], { capture: true, timeout: FETCH_TIMEOUT_MS }); if (r.status === 0) return; const why = r.error?.code === 'ETIMEDOUT' ? `timed out after ${FETCH_TIMEOUT_MS / 1000} s` : (r.stderr ?? '').trim(); throw new Error(`git fetch origin ${BRANCH} failed: ${why}`); } /** A complete build: BUILD_ID alone is not proof. next build writes it before * static generation, so a build killed late has one; export-marker.json is * among the last things it writes. */ const buildComplete = (dir) => existsSync(path.join(dir, 'BUILD_ID')) && existsSync(path.join(dir, 'export-marker.json')); /** Block for a moment without an event loop turn (retries on Windows only). */ function sleepSync(ms) { Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms); } /** * renameSync with the retries Windows needs. Defender or the search indexer * briefly holding a handle inside the directory turns a rename into * EPERM/EBUSY/EACCES, and unlike rmSync there is no maxRetries option. */ function renameWithRetries(from, to) { for (let attempt = 1; ; attempt++) { try { renameSync(from, to); return; } catch (err) { const transient = ['EPERM', 'EBUSY', 'EACCES'].includes(err.code); if (process.platform !== 'win32' || !transient || attempt >= 10) throw err; sleepSync(attempt * 200); } } } /** * Delete a build directory without ever following a link out of it. * * .next/node_modules holds links into the real node_modules for the external * packages (better-sqlite3, ws, undici, signalr) — symlinks on macOS/Linux, * possibly junctions on Windows. A recursive delete that followed one would * empty ROOT/node_modules/better-sqlite3, and the next restart would crash-loop * until someone ran npm install. So walk the tree with lstat and unlink every * link as a link (lstat reports Windows junctions as symbolic links too); only * real files and directories are removed by content. */ function removeDir(dir) { let st; try { st = lstatSync(dir); } catch (err) { if (err.code === 'ENOENT') return; throw err; } if (st.isSymbolicLink()) { removeLink(dir); return; } if (!st.isDirectory()) { rmSync(dir, { force: true, maxRetries: 10, retryDelay: 200 }); return; } for (const name of readdirSync(dir)) removeDir(path.join(dir, name)); // Now empty. Windows can briefly refuse (indexer/Defender handles). for (let attempt = 1; ; attempt++) { try { rmdirSync(dir); return; } catch (err) { if (err.code === 'ENOENT') return; const transient = ['EPERM', 'EBUSY', 'EACCES', 'ENOTEMPTY'].includes(err.code); if (process.platform !== 'win32' || !transient || attempt >= 10) throw err; sleepSync(attempt * 200); } } } /** A symlink unlinks; a Windows junction (a directory link) needs rmdir, which * removes the link itself and never touches its target. */ function removeLink(link) { try { unlinkSync(link); } catch (err) { if (process.platform === 'win32' && ['EPERM', 'EISDIR', 'EACCES'].includes(err.code)) rmdirSync(link); else if (err.code !== 'ENOENT') throw err; } } // ── lock ──────────────────────────────────────────────────────────────────── // A build outlasts the schedule interval, so overlapping runs are otherwise a // certainty rather than a risk. Only APPLY takes the lock; CHECK only looks. function pidAlive(pid) { try { process.kill(pid, 0); return true; } catch (err) { return err.code === 'EPERM'; // exists, just not ours to signal } } /** * Whether anything is left in the process group `pid` led. A worker is a group * leader (startWorker spawns it detached, and a shell gives a foreground run a * group of its own), and its npm and next build children are in that group, so * they outlive a worker that is killed on its own. POSIX only: Windows has no * process groups to ask, and says no. */ function groupAlive(pid) { if (process.platform === 'win32' || pid <= 1) return false; try { process.kill(-pid, 0); return true; } catch (err) { return err.code === 'EPERM'; } } /** * Who holds the lock, and whether they still can. null = no lock. * * The pid check is what matters: a worker killed outright (or a machine * rebooted mid-update) never reaches its `finally`, and waiting out an age * limit would hold every later update back for it. The age limit is the * backstop for a worker that is alive but wedged, and for a pid reused by an * unrelated process. * * A dead pid is not the whole story, though. A worker killed on its own leaves * its build running, still writing into .next, and a worker that took over * would pull .next out from under it and start a second build into the same * directory. So the lock stays held until the dead worker's process group is * empty too (orphaned: true). * * Age and "taken before the last boot" are measured on the uptime clock the * lock records, not on the wall clock. The wall clock can step by hours under * a live worker: a Pi has no RTC and jumps forward at its first NTP sync after * boot, and a VM resumed from a pause does the same. A jump must never make a * running worker's lock look abandoned, because the worker that took over * would put .next-prev back under the first one's build, and both would build * and restart. Uptime only goes backwards across a reboot. */ function readLock() { let raw; let mtime; try { raw = readFileSync(LOCK_FILE, 'utf8').trim(); mtime = statSync(LOCK_FILE).mtimeMs; } catch { return null; } let info = null; try { info = JSON.parse(raw); } catch { /* legacy, or caught between create and write */ } if (info && typeof info === 'object' && Number.isInteger(info.pid)) { const upNow = os.uptime(); const upThen = Number(info.uptime); const hasUptime = Number.isFinite(upThen) && upThen > 0; const rebooted = hasUptime && upNow < upThen; const age = hasUptime && !rebooted ? (upNow - upThen) * 1000 : Date.now() - (Number(info.startedAt) || mtime); const mins = Math.round(age / 60000); if (!pidAlive(info.pid)) { // Nothing outlives a reboot, and the age limit caps a wedged // orphaned build just as it caps a wedged worker. if (!rebooted && age <= STALE_LOCK_MS && groupAlive(info.pid)) return { stale: false, pid: info.pid, age, orphaned: true }; return { stale: true, pid: info.pid, age, why: `pid ${info.pid} is gone, ${mins} min old` }; } if (rebooted) return { stale: true, pid: info.pid, age, why: 'taken before the last boot' }; if (age > STALE_LOCK_MS) return { stale: true, pid: info.pid, age, why: `pid ${info.pid} has held it for ${mins} min` }; return { stale: false, pid: info.pid, age }; } // The pre-worker version of this script wrote a bare Date.now(). It ran // under the cron, which kills it at the next minute boundary, so one of // those locks older than a few minutes was left by a killed run, and the // box behind it may be sitting on a wiped .next. Waiting the old hour out // before repairing it would be the wrong trade. An empty file is a new lock // caught between create and write; its mtime is as good as a timestamp. const since = /^\d+$/.test(raw) ? Number(raw) : mtime; const age = Date.now() - since; if (age < LEGACY_STALE_LOCK_MS) return { stale: false, pid: null, age }; return { stale: true, pid: null, age, why: `left by an older updater, ${Math.round(age / 60000)} min old` }; } /** Whether the lock on disk names this process. */ function ownsLock() { try { return JSON.parse(readFileSync(LOCK_FILE, 'utf8')).pid === process.pid; } catch { return false; // absent or unreadable: not ours } } /** * Atomic create ('wx'), so two workers can never both believe they won. * * Breaking a stale lock is not atomic, though: it is a read, then a delete. * Workers racing over the same stale lock (two checks at once, or a check and * a by-hand --apply) each judge it stale, and a later one's delete then removes * the lock an earlier one has just created, so several carry on at once. The * delete follows its read at once (nothing, not even the log line, sits * between them), so after a pause the file names the one worker that really * holds the lock, and the rest back off. */ function acquireLock() { for (let attempt = 0; attempt < 2; attempt++) { try { const fd = openSync(LOCK_FILE, 'wx'); try { writeSync(fd, JSON.stringify({ pid: process.pid, startedAt: Date.now(), uptime: os.uptime() })); } finally { closeSync(fd); } sleepSync(LOCK_SETTLE_MS); return ownsLock(); } catch (err) { if (err.code !== 'EEXIST') throw err; } const lock = readLock(); if (lock && !lock.stale) return false; // No lock by now means its holder just released it: nothing to delete, // and a delete here could only hit a lock someone made since. if (lock) { rmSync(LOCK_FILE, { force: true }); log(`clearing a stale lock (${lock.why})`); } } return false; } /** Remove the lock only if it is still ours. One taken over as stale belongs to its new holder. */ function releaseLock() { if (!ownsLock()) return; try { rmSync(LOCK_FILE, { force: true }); } catch { /* already gone */ } } // ── failure record ────────────────────────────────────────────────────────── // scripts/.update-failed = { target, step, at, count, restart, clicker, detail? }. // // Written when an attempt STARTS and deleted when it succeeds, so a worker // that dies without reporting (killed, rebooted) still counts as a failed // attempt. Without that a crash would be retried every minute (the first one // is retried at once, see retryAt, but only the first). `restart` // marks a target whose build finished but whose restarts did not; the check // has no other way to see that, because .built-commit already matches. function readFailure() { try { const f = JSON.parse(readFileSync(FAILED_FILE, 'utf8')); return f && typeof f === 'object' && typeof f.target === 'string' ? f : null; } catch { return null; } } function writeFailure(f) { try { writeFileSync(FAILED_FILE, JSON.stringify(f) + '\n'); } catch { /* bookkeeping */ } } function clearFailure() { try { rmSync(FAILED_FILE, { force: true }); } catch { /* bookkeeping */ } } /** Size of update-worker.log right now, so a step's own output can be told apart. */ function workerLogSize() { try { return statSync(WORKER_LOG_FILE).size; } catch { return 0; } } /** What update-worker.log gained since `from` ('' when a by-hand --apply * wrote to the terminal instead). */ function workerLogSince(from) { try { const buf = readFileSync(WORKER_LOG_FILE); return buf.length > from ? buf.subarray(from).toString('utf8') : ''; } catch { return ''; } } /** * Why a step failed, from its output since `from`: the file this user can't * replace when a permission is what stopped it, else one line of the output. */ function explainFailure(from) { const text = workerLogSince(from); return (PERMISSION_DENIED.test(text) && ownershipProblem()) || failureReason(text); } /** * One line on why a step failed, from its output, or null. * * The full npm/next output stays on the machine. Without a reason the master * shows only "update failed at build", and a machine failing for a cause of * its own (a Node upgrade leaving better-sqlite3 on the old ABI, a file * Defender holds, a full disk) can't be told from a broken push. The line * rides on .last-check, which every app build and the master already carry. */ function failureReason(text) { // eslint-disable-next-line no-control-regex const lines = text.replace(/\x1b\[[0-9;]*m/g, '').split(/\r?\n/).map((l) => l.trim()).filter(Boolean); // npm's own epilogue says only that the script failed, and where its log // is; and this script's own log lines (they go to the console too) aren't // the step's output. const boilerplate = /^npm (ERR!|error)( (code|path|command|errno|syscall|signal|workspace|location|A complete log|Lifecycle)\b|\s*$)|^> |^\d{4}-\d\d-\d\dT\S+ {2}/; const first = (re) => lines.find((l) => re.test(l) && !boilerplate.test(l)); const line = first(/NODE_MODULE_VERSION|ERR_DLOPEN_FAILED|\bE(PERM|BUSY|ACCES|NOSPC)\b|heap out of memory|Type error:/) ?? first(/\b[Ee]rror\b/) ?? lines.filter((l) => !boilerplate.test(l)).pop(); return line ? line.replace(/\s+/g, ' ').slice(0, 100) : null; } /** 15 min after the first failure, an hour after repeated ones: long enough * that a deterministic failure (a broken push) doesn't rebuild every minute * on a trading PC, short enough that a transient one (network, EPERM) heals * on its own. * * A first attempt that never reported (still 'started': its worker was * killed, or the machine went down under it) is retried at once instead. That * is a reboot or a person far more often than the update itself, and until * the retry the only complete build may be the one set aside in .next-prev, * where neither a restart nor a boot can find it: after a reboot mid-build * the instances would sit on a partial .next for the whole backoff. A second * death is a pattern, and waits like any other failure. */ const retryAt = (f) => Number(f.at) + (f.count > 1 ? REPEAT_RETRY_MS : f.step === 'started' ? 0 : FIRST_RETRY_MS); // ── what is deployed ──────────────────────────────────────────────────────── /** * When HEAD last moved (epoch ms), or null when there is no way to tell. * The reflog's own timestamp, not the commit date: what matters is when this * checkout changed, not when the commit was written. * * The selector is picked out of the default output (" HEAD@{}: ...") * rather than asked for with --format=%gd: on Windows this goes through cmd, * and a % on a cmd command line is one typo away from variable expansion. A * git too old for --date=unix prints HEAD@{0}, which would read as "HEAD never * moved" and adopt anything, hence the sanity bound. */ function headMovedAt() { try { const m = /HEAD@\{(\d+)\}/.exec(git('reflog', '-1', '--date=unix')); if (m && Number(m[1]) > 1e9) return Number(m[1]) * 1000; } catch { /* no reflog: fall through */ } try { const head = readFileSync(path.join(ROOT, '.git', 'HEAD'), 'utf8').trim(); const ref = /^ref: (.+)$/.exec(head)?.[1]; return statSync(path.join(ROOT, '.git', ref ?? 'HEAD')).mtimeMs; } catch { return null; } } /** PM2's process list, or null when PM2 can't be asked. */ function pm2List() { const r = run('pm2', ['jlist'], { capture: true, timeout: 30_000 }); if (r.status !== 0) return null; // jlist prints the process list as one JSON line, possibly after "[PM2] ..." chatter. const line = (r.stdout ?? '').split('\n').find((l) => /^\s*\[\s*[{\]]/.test(l)); try { const procs = JSON.parse(line); return Array.isArray(procs) ? procs : null; } catch { return null; } } /** * Names of this checkout's PM2 instances that started before `ms`, i.e. still * running code from before the build on disk. Empty when PM2 can't be asked: * this only sharpens a heuristic and must never block an update. */ function instancesStartedBefore(ms) { const procs = pm2List(); if (!procs) return []; const names = new Set(INSTANCES.map((i) => i.name)); return procs .filter((p) => names.has(p?.name) && p.pm2_env?.status === 'online' && Number(p.pm2_env.pm_uptime) < ms) .map((p) => p.name); } /** * The commit .next was built from: { commit } or { commit: null, why }. * * scripts/.built-commit is written only after `npm run build` succeeds. When it * is absent (every machine, the first time this version runs) the build on * disk is judged instead. Adopting it as HEAD avoids a pointless rebuild and * restart on every healthy machine. Anything doubtful counts as unknown, and * unknown means rebuild. That is what rescues machines the old updater * stranded: their HEAD already equals origin, but their build is stale, * partial or missing. */ function builtCommit(head, { adopt }) { try { const recorded = readFileSync(BUILT_FILE, 'utf8').trim(); if (/^[0-9a-f]{40}([0-9a-f]{24})?$/.test(recorded)) return { commit: recorded }; } catch { /* absent: first run of this version on this machine */ } if (!buildComplete(NEXT_DIR)) return { commit: null, why: 'no complete build on disk' }; const builtAt = statSync(BUILD_ID_FILE).mtimeMs; const movedAt = headMovedAt(); if (movedAt === null) return { commit: null, why: 'cannot tell when HEAD last moved' }; if (builtAt < movedAt) return { commit: null, why: 'the build on disk is older than HEAD' }; // Built after HEAD moved, but a run killed between the build and the // restarts leaves the old processes serving the new .next. const behind = instancesStartedBefore(builtAt); if (behind.length) return { commit: null, why: `${behind.join(', ')} started before the build on disk` }; if (adopt) { writeFileSync(BUILT_FILE, head + '\n'); log(`adopted the build on disk as ${short(head)} (first run with scripts/.built-commit)`); } return { commit: head }; } function isAncestor(a, b) { const r = run('git', ['merge-base', '--is-ancestor', a, b], { capture: true }); if (r.status === 0) return true; if (r.status === 1) return false; throw new Error(`git merge-base failed: ${(r.stderr ?? '').trim()}`); } /** Files changed from→to, or null when `from` is unknown (treat everything as changed). */ function changedFiles(from, to) { if (!from) return null; if (from === to) return []; try { return git('diff', '--name-only', from, to).split('\n').filter(Boolean); } catch { return null; // e.g. the recorded commit is gone } } /** Fetch and work out what, if anything, needs doing. Shared by CHECK and APPLY. */ function inspect({ adopt }) { fetchOrigin(); const local = git('rev-parse', 'HEAD'); const remote = git('rev-parse', `origin/${BRANCH}`); // A checkout AHEAD of origin (a commit made on this machine) has nothing to // pull. `pull --ff-only` would "succeed" as a no-op and the old updater then // rebuilt and restarted on every tick. const needsPull = local !== remote && !isAncestor(remote, local); const target = needsPull ? remote : local; const built = builtCommit(local, { adopt }); // Complete, not merely present: a build of the SAME commit killed after // BUILD_ID was written (a by-hand `npm run build`, or a worker redoing a // missing build) would otherwise pass for up to date indefinitely. const buildMissing = !buildComplete(NEXT_DIR); const needsBuild = built.commit !== target || buildMissing; const failed = readFailure(); const pending = failed?.target === target ? failed : null; const needsRestart = !needsBuild && pending?.restart === true; return { local, remote, target, needsPull, built, buildMissing, needsBuild, needsRestart, pending }; } function describe(plan) { if (plan.needsPull) return `${short(plan.local)} -> ${short(plan.remote)}`; if (plan.needsBuild) { const why = plan.buildMissing ? 'no complete build on disk' : plan.built.why ?? `last built ${short(plan.built.commit)}`; return `rebuild ${short(plan.target)} (${why})`; } return `restart onto ${short(plan.target)} (the last restart did not complete)`; } /** Which of the optional steps this plan needs. */ function workFor(plan) { const changed = changedFiles(plan.built.commit, plan.target); const touched = (f) => changed === null || changed.includes(f); return { changed, npmInstall: plan.needsBuild && touched('package-lock.json'), pythonDeps: plan.needsBuild && touched('clicker/requirements.txt'), restartClicker: plan.needsBuild ? changed === null || changed.some((f) => f.startsWith('clicker/')) : plan.needsRestart && plan.pending?.clicker === true, }; } // ── stuck states ──────────────────────────────────────────────────────────── // Two states that fail every update until a person gets onto the machine. /** What a step prints when it may not touch a file. */ const PERMISSION_DENIED = /permission denied|insufficient permission|\bEACCES\b|\bEPERM\b/i; /** * A file in the checkout this user doesn't own, as the fix to run, or null. * * One `sudo git pull` or `sudo npm run build` leaves files this user cannot * replace, and from then on every update fails: the build can't clear .next, * and a fetch can't write into .git (Ziad's Pi, 2026-10-02: ~1,800 of them). * The failure itself only says EACCES somewhere in npm's output, so name the * file and the one command that fixes it — the master's updater chip shows * this line. Asked only once a step has failed on a permission: a stray * root-owned file that blocks nothing must not stop an update. POSIX only. */ function ownershipProblem() { if (typeof process.getuid !== 'function') return null; const r = run('find', [ROOT, '!', '-user', String(process.getuid()), '-print', '-quit'], { capture: true, timeout: 20_000 }); const hit = (r.stdout ?? '').split('\n')[0].trim(); if (!hit) return null; let owner = 'another user'; try { const uid = lstatSync(hit).uid; owner = uid === 0 ? 'root' : `uid ${uid}`; } catch { /* gone since: the generic name will do */ } return `${path.relative(ROOT, hit) || '.'} is owned by ${owner}: run sudo chown -R ${os.userInfo().username} ${ROOT}`; } /** inspect(), with a fetch refused by a .git this user can't write named as such. */ function inspectNamingOwnership(opts) { try { return inspect(opts); } catch (err) { const why = PERMISSION_DENIED.test(err.message) ? ownershipProblem() : null; throw why ? new Error(why) : err; } } /** * npm install rewrites package-lock.json on some machines (another npm * version, platform-specific optional dependencies), and that edit then * refuses every fast-forward that touches the file. It is generated, so when * it is the only tracked edit, put it back. Any other edit is a person's and * stays, showing as "edited here" on the master. */ function restoreRewrittenLockfile() { let changed; try { changed = git('diff', '--name-only', 'HEAD').split('\n').filter(Boolean); } catch { return; } if (changed.length !== 1 || changed[0] !== 'package-lock.json') return; git('checkout', 'HEAD', '--', 'package-lock.json'); log('package-lock.json had been rewritten here (npm) — put it back before the pull'); } // ── build ─────────────────────────────────────────────────────────────────── /** * npm run build, with the previous build set aside so a failure can be undone. * * next build empties .next as its first step, under the live server, whatever * we do. Renaming it to .next-prev first costs nothing extra, and it means a * FAILED build can be rolled back instead of leaving the box on a half-written * .next that a restart or a reboot would then crash-loop on. When the rename * is refused (Windows, something holding a handle), build in place without a * safety net rather than not update at all. */ function build() { // A worker that died between setting the old build aside and putting it // back leaves the only complete build in .next-prev. Restore it first, or // the cleanup below would delete it. If it can't be put back, it stays // where it is (clearing it would leave no complete build anywhere), this // build runs in place, and a failure tries the restore once more. let prevIsOnlyBuild = false; if (buildComplete(PREV_DIR) && !buildComplete(NEXT_DIR)) { try { removeDir(NEXT_DIR); renameWithRetries(PREV_DIR, NEXT_DIR); log('put back the previous build left in .next-prev by an interrupted update'); } catch (err) { prevIsOnlyBuild = true; log(`could not put back .next-prev (${err.code ?? err.message}) — leaving it there and building in place`); } } let backedUp = false; const hadBuild = existsSync(NEXT_DIR); if (hadBuild && !prevIsOnlyBuild) { try { removeDir(PREV_DIR); renameWithRetries(NEXT_DIR, PREV_DIR); backedUp = true; } catch (err) { log(`could not set the previous build aside (${err.code ?? err.message}) — building in place, no rollback this time`); } } // next build keeps .next/cache (the incremental type-check state among it) // from one build to the next. Carry it over so setting .next aside doesn't // make every build a cold one. Best effort: without it a build is only // slower. Not carried back on failure, so a retry starts from a clean cache. if (backedUp) { try { mkdirSync(NEXT_DIR); renameSync(path.join(PREV_DIR, 'cache'), path.join(NEXT_DIR, 'cache')); } catch { /* a cold build */ } } log('building'); if (run('npm', ['run', 'build']).status === 0) return true; if (backedUp || prevIsOnlyBuild) { try { removeDir(NEXT_DIR); renameWithRetries(PREV_DIR, NEXT_DIR); log('build failed — restored the previous build'); } catch (err) { log(`build failed, and restoring the previous build failed too (${err.code ?? err.message}) — it is still in .next-prev`); } } else if (hadBuild) { log('build failed — it ran in place, so there is no previous build to restore'); } else { log('build failed — there was no previous build to restore'); } return false; } function removePrevBuild() { if (!existsSync(PREV_DIR)) return; try { removeDir(PREV_DIR); } catch (err) { log(`could not remove .next-prev (${err.code ?? err.message}) — the next build clears it`); } } // ── heal ──────────────────────────────────────────────────────────────────── // The checker runs only while PM2 fires it, and PM2 loses it easily: a // `pm2 stop`, a `pm2 delete all`, a reboot after a list change nobody saved. /** How long ago the last check ran, in ms; null = never on this checkout. */ function lastCheckAge() { try { const at = Date.parse(readFileSync(LAST_CHECK_FILE, 'utf8').split(/\s/, 1)[0]); return Date.now() - (Number.isFinite(at) ? at : statSync(LAST_CHECK_FILE).mtimeMs); } catch { return null; } } const fmtAge = (ms) => (ms >= 90 * 60 * 1000 ? `${Math.round(ms / 3_600_000)} h` : `${Math.round(ms / 60_000)} min`); /** * The fields PM2 adds to an app's environment. * * PM2 hands every app the scalar fields of its own record as environment * (name, pm_id, autorestart, …), and `pm2 start` captures the caller's * environment for the new app, then copies it over the new app's record. A * checker registered by a heal, which the dashboard app starts, would take * that app's name and id, and its autorestart would override * --no-autorestart. These are the fields PM2 put on real records, plus the * ones only some records carry. PORT and AUTOFIRMER_DB are the instance's * own, and the checker serves every instance. */ const INHERITED_FIELDS = new Set([ 'NODE_APP_INSTANCE', 'automation', 'autorestart', 'autostart', 'created_at', 'cron_restart', 'exec_interpreter', 'exec_mode', 'exit_code', 'filter_env', 'instance_var', 'instances', 'kill_retry_time', 'km_link', 'merge_logs', 'name', 'namespace', 'node_version', 'pmx', 'prev_restart_delay', 'restart_time', 'status', 'stop_exit_codes', 'treekill', 'unique_id', 'unstable_restarts', 'username', 'version', 'versioning', 'vizion', 'vizion_running', 'watch', 'windowsHide', 'PORT', 'AUTOFIRMER_DB', ]); /** Drop what PM2 (and the instance) put in this process's environment, so * nothing started from here inherits it. */ function scrubInheritedEnv() { // PM2 also files the app's own config under its name. if (process.env.name) delete process.env[process.env.name]; for (const key of Object.keys(process.env)) { if (INHERITED_FIELDS.has(key) || key.startsWith('pm_') || key.startsWith('axm_')) delete process.env[key]; } } /** * Get PM2 firing this script again once its checks have stopped. Returns * false when this checkout is opted out of updates altogether. * * What the fleet shows (2026-10, six Windows machines): the cron dies inside * a live PM2 daemon, nobody involved. Every fire runs restartProcessId, * whose first step drops the cron; only a restart that gets as far as * launching the app registers it again. croner sometimes fires twice in * one second, the second restart has to stop a check that is still * launching, and a refused or timed-out stop leaves the checker `stopped` * or `errored` with no cron at all. * * So a checker PM2 still lists is restarted, which registers its cron again * with the schedule and environment it already had, and nothing is deleted: * `pm2 delete` kills the pid an `errored` entry kept, and Windows hands pids * out again quickly, so that could be any program by now. If that pid is * alive, PM2 is left alone and this run checks instead. Only a checker PM2 * has lost entirely is registered again, the way install-autostart.* does. */ function heal() { if (existsSync(OPT_OUT_FILE)) { console.log('scripts/.no-auto-update is here — this checkout is not auto-updated'); return false; } // Before anything is started from here, the worker below included: the // app that asked for this heal is a PM2 app, and may be instance 2. scrubInheritedEnv(); const age = lastCheckAge(); if (age !== null && age <= STALE_CHECK_MS) return true; // checking on schedule const why = age === null ? 'the update checker had never run here' : `the update checker had stopped (last check ${fmtAge(age)} ago)`; const pm2 = (...args) => run('pm2', args.map(shellArg), { capture: true, timeout: PM2_TIMEOUT_MS }); const lastLine = (r) => (((r.stderr || r.stdout) ?? '').trim().split('\n').pop() || `exit ${r.status}`); const listed = pm2List()?.find((p) => p?.name === CHECKER); const pid = Number(listed?.pid); if (listed && pid > 0 && pidAlive(pid)) { log(`${why}, but PM2 still has it on pid ${pid}, which is alive — left PM2 alone (a restart would kill that pid); checking from here`); return true; } let how; if (listed && pm2('restart', CHECKER).status === 0) { how = `restarted it in PM2 (${listed.pm2_env?.status ?? 'unknown'}), which registers its cron again`; } else { const cron = listed?.pm2_env?.cron_restart || DEFAULT_CRON; if (listed) pm2('delete', CHECKER); // its restart failed, and its pid is dead const r = pm2('start', SELF, '--name', CHECKER, '--no-autorestart', '--cron-restart', cron); if (r.status !== 0) { log(`${why} — ERROR: could not register it with PM2: ${lastLine(r)}`); return true; // the check below can still update this machine } how = `registered it with PM2 again (${cron})`; } const saved = pm2('save').status === 0; log(`${why} — ${how}${saved ? '' : ', but pm2 save failed: a reboot would lose it'}`); return true; } // ── check ─────────────────────────────────────────────────────────────────── /** * Start `update-check.mjs --apply` detached, logging to update-worker.log. * The log is truncated per run: it holds the full npm/next output of the * latest attempt, the first place to look when one fails. */ function startWorker() { let out = 'ignore'; try { out = openSync(WORKER_LOG_FILE, 'w'); } catch { /* run it without a log rather than not at all */ } try { const child = spawn(process.execPath, [SELF, '--apply'], { cwd: ROOT, detached: true, windowsHide: true, stdio: ['ignore', out, out], }); child.on('error', () => { /* surfaces below as the missing pid */ }); child.unref(); if (!child.pid) throw new Error('could not start the update worker'); return child.pid; } finally { if (typeof out === 'number') closeSync(out); } } function dryRunReport(plan) { const work = workFor(plan); log(`update available: ${describe(plan)}`); if (work.changed) log(`${work.changed.length} file(s) changed since the last build`); log('dry run — stopping before any change'); const restarts = plan.needsBuild || plan.needsRestart; log(`would run: ${[ plan.needsPull && 'git pull --ff-only', work.npmInstall && 'npm install', work.pythonDeps && 'pip install', plan.needsBuild && 'npm run build', ...(restarts ? INSTANCES.map((i) => `pm2 restart ${i.name}`) : []), work.restartClicker && 'pm2 restart clicker', ].filter(Boolean).join(', ')}`); if (plan.pending) { log(`last attempt at ${short(plan.target)}: ${plan.pending.step} (attempt ${plan.pending.count}); next automatic try after ${new Date(retryAt(plan.pending)).toISOString()}`); } } function check() { const lock = readLock(); if (lock && !lock.stale) { const who = !lock.pid ? 'an older updater' : lock.orphaned ? `the build left by dead worker pid ${lock.pid}` : `worker pid ${lock.pid}`; console.log(`an update is already running (${who}) — exiting`); if (!DRY_RUN) recordCheck(`update in progress (${who}, ${Math.round(lock.age / 60000)} min)`); return 0; } let outcome = null; try { const plan = inspectNamingOwnership({ adopt: !DRY_RUN }); if (!plan.needsPull && !plan.needsBuild && !plan.needsRestart) { outcome = `up to date at ${short(plan.target)}`; // silent in the log by design if (DRY_RUN) console.log(outcome); return 0; } if (DRY_RUN) { dryRunReport(plan); return 0; } // Silent in update.log too: the failure itself was logged once, and a // line a minute until the retry would bury it. --now (the master's // "Update now") is a person asking for the retry. if (plan.pending && !NOW && Date.now() < retryAt(plan.pending)) { outcome = `waiting to retry ${short(plan.target)}: ${plan.pending.step} (attempt ${plan.pending.count}), next try after ${new Date(retryAt(plan.pending)).toISOString()}` + (plan.pending.detail ? ` — ${plan.pending.detail}` : ''); return 0; } const pid = startWorker(); log(`update available: ${describe(plan)} — started worker pid ${pid}`); outcome = `worker pid ${pid} started: ${describe(plan)}`; return 0; } catch (err) { log(`ERROR: ${err.message}`); outcome = `ERROR: ${err.message}`; return 1; } finally { if (!DRY_RUN && outcome) recordCheck(outcome); } } // ── apply ─────────────────────────────────────────────────────────────────── /** * A kill skips `finally`, so the lock is released here too. The next check * then retries after the usual backoff instead of waiting out a stale lock. * Node runs this on the event loop, so it never lands in the middle of a * spawnSync. An interrupted build still comes back through build()'s restore. */ function interrupted(signal) { log(`${signal} — stopping; lock released${readFailure() ? ', the next check retries after the backoff' : ''}`); releaseLock(); recordCheck(`interrupted by ${signal}`); process.exit(1); } async function apply() { let locked; try { locked = acquireLock(); } catch (err) { log(`ERROR: could not take the update lock: ${err.message}`); return 1; } if (!locked) { console.log('another update is already running — exiting'); return 0; } for (const signal of ['SIGINT', 'SIGTERM', 'SIGHUP']) process.on(signal, () => interrupted(signal)); let outcome = 'interrupted'; let attempt = null; const abort = (step, why, detail = null) => { log(`ABORTED: ${why}${detail ? ` (${detail})` : ''}`); outcome = detail ? `${why}: ${detail}` : why; if (attempt) writeFailure({ ...attempt, step, at: Date.now(), ...(detail ? { detail } : {}) }); return 1; }; try { const plan = inspectNamingOwnership({ adopt: true }); if (!plan.needsPull && !plan.needsBuild && !plan.needsRestart) { outcome = `up to date at ${short(plan.target)}`; clearFailure(); return 0; } const { target } = plan; const work = workFor(plan); const because = (what) => (work.changed ? `${what} changed` : 'last build unknown'); attempt = { target, step: 'started', at: Date.now(), count: (plan.pending?.count ?? 0) + 1, restart: plan.needsRestart, clicker: plan.needsRestart && plan.pending?.clicker === true, }; writeFailure(attempt); log(`applying ${describe(plan)} (worker pid ${process.pid}, attempt ${attempt.count})`); if (work.changed) log(`${work.changed.length} file(s) changed since the last build`); else if (plan.needsBuild) log('last build unknown — treating every file as changed'); // The exact commit the check saw, not whatever `git pull` would fetch now. if (plan.needsPull) { restoreRewrittenLockfile(); const from = workerLogSize(); if (run('git', ['merge', '--ff-only', target]).status !== 0) { return abort('pull', `git pull --ff-only to ${short(target)} failed — see scripts/update-worker.log`, explainFailure(from)); } } if (plan.needsBuild) { if (work.npmInstall) { log(`${because('package-lock.json')} — npm install`); const from = workerLogSize(); if (run('npm', ['install']).status !== 0) return abort('npm install', 'npm install failed — nothing was built or restarted', explainFailure(from)); } if (work.pythonDeps) { log(`${because('clicker/requirements.txt')} — refreshing python deps`); ensurePythonDeps({ log: (m) => log(m) }); // non-fatal by design } // Build BEFORE restarting. A failed build leaves the running process // untouched, which is the whole point of doing it in this order. const from = workerLogSize(); if (!build()) return abort('build', 'build failed — not deployed, nothing was restarted', explainFailure(from)); // .built-commit says what .next holds, so it changes the moment .next // does. From here a failed restart must not rebuild, only restart, // which the `restart` flag on the attempt tells the next check. writeFileSync(BUILT_FILE, target + '\n'); attempt = { ...attempt, step: 'restart', at: Date.now(), restart: true, clicker: work.restartClicker }; writeFailure(attempt); } if (plan.needsBuild || plan.needsRestart) { for (const instance of INSTANCES) { log(`restarting ${instance.name}`); if (run('pm2', ['restart', instance.name], { timeout: PM2_TIMEOUT_MS }).status !== 0) return abort('restart', `pm2 restart ${instance.name} failed`); } if (work.restartClicker) { log(`${because('clicker')} — restarting it too`); run('pm2', ['restart', 'clicker'], { timeout: PM2_TIMEOUT_MS }); } } removePrevBuild(); clearFailure(); await warmUp(); log(`updated to ${short(target)}`); outcome = `updated to ${short(target)}`; return 0; } catch (err) { log(`ERROR: ${err.message}`); outcome = `ERROR: ${err.message}`; if (attempt) writeFailure({ ...attempt, step: 'error', at: Date.now() }); return 1; } finally { releaseLock(); recordCheck(outcome); } } async function warmUp() { for (const instance of INSTANCES) { const base = `http://127.0.0.1:${instance.port}`; const deadline = Date.now() + 120_000; while (Date.now() < deadline) { try { if ((await fetch(`${base}/api/auto-trade`, { signal: AbortSignal.timeout(5_000) })).ok) break; } catch { /* still restarting */ } await new Promise((r) => setTimeout(r, 2_000)); } try { await fetch(`${base}/api/state`, { signal: AbortSignal.timeout(60_000) }); const status = await (await fetch(`${base}/api/auto-trade`)).json(); log(status.running ? `${instance.name}: scheduler resumed — ${status.action} ${status.symbol}` : `${instance.name}: scheduler is stopped`); // The clicker/extension runner only ever serves instance 1. if (instance.port === 3000) { const runner = await (await fetch(`${base}/api/autobuyer/runner`)).json(); log(`runner ${runner.online ? 'online' : 'OFFLINE'}${runner.stale ? ' (version stale — reload the extension)' : ''}`); } } catch (err) { log(`${instance.name}: post-restart check failed: ${err.message}`); } await logHealth(instance, base); } } /** * One line from /api/health, when the build has it. Informational only: right * after a restart the clients may not have synced yet, so this never fails an * update. Parsed loosely, because the endpoint may be newer than this script. */ async function logHealth(instance, base) { let res; try { res = await fetch(`${base}/api/health`, { signal: AbortSignal.timeout(10_000) }); } catch { return; } if (res.status === 404) return; // a build from before the endpoint let body = null; try { body = await res.json(); } catch { /* not JSON: the status alone */ } const parts = [`HTTP ${res.status}`]; if (typeof body?.ok === 'boolean') parts.push(body.ok ? 'ok' : 'NOT ok'); // A tradable firm with no client (no credentials yet) is listed with // hasClient: false and is not something that failed to sync. const listed = Array.isArray(body?.clients) ? body.clients : Array.isArray(body) ? body : null; const clients = listed?.filter((c) => c?.hasClient !== false); if (clients) { const now = Date.now(); const ready = clients.filter((c) => { if (typeof c?.ready === 'boolean') return c.ready; const h = c?.health ?? c; return h?.syncComplete === true && h.lastSyncAt > 0 && now - h.lastSyncAt < 5 * 60_000; }).length; parts.push(`${ready}/${clients.length} client(s) synced`); } log(`${instance.name}: health ${parts.join(', ')}`); } async function main() { mkdirSync(path.join(ROOT, 'scripts'), { recursive: true }); trimLog(); if (HEAL && !DRY_RUN && !heal()) return 0; return APPLY && !DRY_RUN ? apply() : check(); } process.exit(await main());