Brandon LiandClaude Opus 5.5
13e03c3d0f
PM2's cron for autofirmer-update dies inside a live daemon (six Windows machines in October), and nothing on the machine noticed. The app now watches scripts/.last-check and, past 10 minutes, runs update-check.mjs --heal: a checker PM2 still lists is restarted (which re-registers its cron), one it lost is registered again, and one whose errored entry still holds a live pid is left alone, since PM2 would kill that pid. The heal scrubs PM2's per-app fields from its environment first. The master's new 'update-now' command runs --heal --now, skipping a failed update's retry wait. A failed pull, npm install or build records one line of reason (or the chown to run when a file isn't ours), which rides on .last-check to the master. A package-lock.json rewritten by npm is put back before the pull. The fetch timeout drops under the cron's minute. scripts/.no-auto-update opts a checkout out. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
1159 lines
52 KiB
JavaScript
1159 lines
52 KiB
JavaScript
/**
|
|
* Keep this checkout on master: fetch, and when something needs doing, pull,
|
|
* rebuild and restart, without ever leaving a deploy half-applied.
|
|
*
|
|
* PM2 fires this every minute (--no-autorestart --cron-restart, registered by
|
|
* install-autostart.*), and that schedule dictates the shape of the script.
|
|
* PM2's cron handler *restarts* the process, and a restart first stops a
|
|
* process that is still running: it is tree-killed at the next minute boundary.
|
|
* So anything that outlasts ~55 s (npm install, next build) used to die
|
|
* half-way through, and a kill skips `finally`. The lock stayed behind, HEAD
|
|
* had already moved at the pull (so the next run saw nothing to pull and never
|
|
* retried), and next build had already wiped .next under the live server.
|
|
*
|
|
* Hence two halves:
|
|
*
|
|
* CHECK (no arguments; what the cron runs) takes seconds. It fetches, decides
|
|
* whether anything needs doing, hands the work to a detached worker and
|
|
* exits. PM2 sees the process stop on its own, so the next tick has nothing
|
|
* to kill, and the worker, orphaned once this process exits, is outside the
|
|
* process tree PM2 kills.
|
|
*
|
|
* APPLY (--apply) is the worker, and is also safe to run by hand in the
|
|
* foreground. It takes the lock, re-checks under it, then pulls, installs,
|
|
* builds, restarts and warms up.
|
|
*
|
|
* "Needs doing" is judged against scripts/.built-commit, the commit .next was
|
|
* last built from, not against HEAD. HEAD moves at the pull, before the work
|
|
* that makes it real, so HEAD === origin says nothing about whether the build
|
|
* finished.
|
|
*
|
|
* A broken push still cannot take a trading PC down. The build runs before
|
|
* anything is restarted, the previous build is set aside first and put back if
|
|
* the build fails, and a failure is retried on a backoff (scripts/.update-failed)
|
|
* rather than every minute.
|
|
*
|
|
* A machine whose checker stopped firing can't notice by itself, so the app
|
|
* asks for a heal (lib/updater-watchdog.ts) once the last check goes stale, and
|
|
* the master's "Update now" asks for a heal plus an immediate check.
|
|
*
|
|
* node scripts/update-check.mjs # check; start a worker if needed
|
|
* node scripts/update-check.mjs --apply # update now, here, in the foreground
|
|
* node scripts/update-check.mjs --dry-run # report only, change nothing
|
|
* node scripts/update-check.mjs --heal # re-register a checker that stopped, then check
|
|
* node scripts/update-check.mjs --now # check without waiting out a failure's backoff
|
|
*/
|
|
import { spawn, spawnSync } from 'node:child_process';
|
|
import {
|
|
appendFileSync, closeSync, existsSync, lstatSync, mkdirSync, openSync, readdirSync, readFileSync,
|
|
renameSync, rmdirSync, rmSync, statSync, unlinkSync, writeFileSync, writeSync,
|
|
} from 'node:fs';
|
|
import os from 'node:os';
|
|
import path from 'node:path';
|
|
import { fileURLToPath } from 'node:url';
|
|
import { ensurePythonDeps } from './ensure-python-deps.mjs';
|
|
import { loadInstances } from './instances.mjs';
|
|
|
|
const SELF = fileURLToPath(import.meta.url);
|
|
const ROOT = path.resolve(path.dirname(SELF), '..');
|
|
const LOG_FILE = path.join(ROOT, 'scripts', 'update.log');
|
|
const WORKER_LOG_FILE = path.join(ROOT, 'scripts', 'update-worker.log');
|
|
const LOCK_FILE = path.join(ROOT, 'scripts', '.update.lock');
|
|
const LAST_CHECK_FILE = path.join(ROOT, 'scripts', '.last-check');
|
|
const BUILT_FILE = path.join(ROOT, 'scripts', '.built-commit');
|
|
const FAILED_FILE = path.join(ROOT, 'scripts', '.update-failed');
|
|
const NEXT_DIR = path.join(ROOT, '.next');
|
|
const PREV_DIR = path.join(ROOT, '.next-prev');
|
|
const BUILD_ID_FILE = path.join(NEXT_DIR, 'BUILD_ID');
|
|
const BRANCH = process.env.AUTOFIRMER_BRANCH ?? 'master';
|
|
// One checkout, one build — but possibly several PM2 processes to restart and
|
|
// warm afterwards (see scripts/instances.mjs).
|
|
const INSTANCES = loadInstances();
|
|
const APPLY = process.argv.includes('--apply');
|
|
const DRY_RUN = process.argv.includes('--dry-run');
|
|
const HEAL = process.argv.includes('--heal');
|
|
const NOW = process.argv.includes('--now');
|
|
const OPT_OUT_FILE = path.join(ROOT, 'scripts', '.no-auto-update');
|
|
// The PM2 app install-autostart.* registers, and its schedule.
|
|
const CHECKER = 'autofirmer-update';
|
|
const DEFAULT_CRON = '*/1 * * * *';
|
|
// The master's UPDATER_STALE_MS: past this its dashboard says "updater not running".
|
|
const STALE_CHECK_MS = 10 * 60 * 1000;
|
|
const STALE_LOCK_MS = 2 * 60 * 60 * 1000;
|
|
const LEGACY_STALE_LOCK_MS = 10 * 60 * 1000;
|
|
const LOCK_SETTLE_MS = 1000;
|
|
const FIRST_RETRY_MS = 15 * 60 * 1000;
|
|
const REPEAT_RETRY_MS = 60 * 60 * 1000;
|
|
// Bounds on the steps that wait on something else (the network, the PM2
|
|
// daemon). Without them a stalled one holds the worker, and the lock, until the
|
|
// lock goes stale by age, and the worker that takes over then runs alongside
|
|
// it. npm and next build are left unbounded: on Windows a timeout would kill
|
|
// only the cmd.exe in front of them, and an orphaned build still writing into
|
|
// .next is worse than a slow one.
|
|
// The fetch also stays under the cron's minute: a check still running at the next
|
|
// fire has to be killed by PM2, and that kill is where PM2 loses the cron
|
|
// (see heal).
|
|
const FETCH_TIMEOUT_MS = 45 * 1000;
|
|
const PM2_TIMEOUT_MS = 3 * 60 * 1000;
|
|
const LOG_MAX_BYTES = 256 * 1024;
|
|
const LOG_KEEP_LINES = 500;
|
|
|
|
const short = (commit) => (commit ? commit.slice(0, 8) : 'unknown');
|
|
|
|
/**
|
|
* Keep update.log from growing without bound.
|
|
*
|
|
* Trimmed once per run, before anything is written, rather than on every line:
|
|
* the no-change path writes nothing at all, so this is the only moment the file
|
|
* can have grown since last time. Housekeeping must never break an update, so
|
|
* any failure here is swallowed.
|
|
*/
|
|
function trimLog() {
|
|
try {
|
|
if (!existsSync(LOG_FILE) || statSync(LOG_FILE).size <= LOG_MAX_BYTES) return;
|
|
const kept = readFileSync(LOG_FILE, 'utf8').split('\n').filter(Boolean).slice(-LOG_KEEP_LINES);
|
|
writeFileSync(LOG_FILE, kept.join('\n') + '\n');
|
|
} catch { /* ignore */ }
|
|
}
|
|
|
|
function log(msg) {
|
|
const line = `${new Date().toISOString()} ${msg}`;
|
|
console.log(line);
|
|
try { appendFileSync(LOG_FILE, line + '\n'); } catch { /* logging must never throw */ }
|
|
}
|
|
|
|
/**
|
|
* Record that a check happened, whether or not it found anything.
|
|
*
|
|
* Overwritten rather than appended, so it stays one line and needs no trimming.
|
|
* Without it there is no way to tell "running, nothing to pull" from "not
|
|
* running at all": the no-change path logs nothing by design, and PM2 reports a
|
|
* cron-restart process as `stopped` with a restart count of 0 even while it is
|
|
* firing on schedule.
|
|
*/
|
|
function recordCheck(status) {
|
|
try {
|
|
writeFileSync(LAST_CHECK_FILE, `${new Date().toISOString()} ${status}\n`);
|
|
} catch { /* never let bookkeeping break an update */ }
|
|
}
|
|
|
|
/**
|
|
* An argument for run() as cmd.exe will see it. On Windows run() goes through
|
|
* the shell, which joins the arguments with plain spaces, so a cron string or
|
|
* a path with a space in it would arrive as several.
|
|
*/
|
|
const shellArg = (arg) => (process.platform === 'win32' && /[\s"]/.test(arg) ? `"${arg.replace(/"/g, '""')}"` : arg);
|
|
|
|
function run(cmd, args, { capture = false, timeout } = {}) {
|
|
return spawnSync(cmd, args, {
|
|
cwd: ROOT,
|
|
encoding: 'utf8',
|
|
stdio: capture ? 'pipe' : 'inherit',
|
|
shell: process.platform === 'win32',
|
|
// The worker is detached, so on Windows it has no console of its own
|
|
// and every npm/git/pm2 child would otherwise open a visible console
|
|
// window on the trading desktop, stealing focus from the clicker.
|
|
windowsHide: true,
|
|
maxBuffer: 16 * 1024 * 1024,
|
|
timeout,
|
|
});
|
|
}
|
|
|
|
function git(...args) {
|
|
const r = run('git', args, { capture: true });
|
|
if (r.status !== 0) throw new Error(`git ${args.join(' ')} failed: ${(r.stderr ?? '').trim()}`);
|
|
return (r.stdout ?? '').trim();
|
|
}
|
|
|
|
/** git fetch, bounded: git sets no network timeout of its own, so a stalled
|
|
* connection would otherwise wait forever (see FETCH_TIMEOUT_MS). */
|
|
function fetchOrigin() {
|
|
const r = run('git', ['fetch', 'origin', BRANCH], { capture: true, timeout: FETCH_TIMEOUT_MS });
|
|
if (r.status === 0) return;
|
|
const why = r.error?.code === 'ETIMEDOUT' ? `timed out after ${FETCH_TIMEOUT_MS / 1000} s` : (r.stderr ?? '').trim();
|
|
throw new Error(`git fetch origin ${BRANCH} failed: ${why}`);
|
|
}
|
|
|
|
/** A complete build: BUILD_ID alone is not proof. next build writes it before
|
|
* static generation, so a build killed late has one; export-marker.json is
|
|
* among the last things it writes. */
|
|
const buildComplete = (dir) => existsSync(path.join(dir, 'BUILD_ID')) && existsSync(path.join(dir, 'export-marker.json'));
|
|
|
|
/** Block for a moment without an event loop turn (retries on Windows only). */
|
|
function sleepSync(ms) {
|
|
Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
|
|
}
|
|
|
|
/**
|
|
* renameSync with the retries Windows needs. Defender or the search indexer
|
|
* briefly holding a handle inside the directory turns a rename into
|
|
* EPERM/EBUSY/EACCES, and unlike rmSync there is no maxRetries option.
|
|
*/
|
|
function renameWithRetries(from, to) {
|
|
for (let attempt = 1; ; attempt++) {
|
|
try {
|
|
renameSync(from, to);
|
|
return;
|
|
} catch (err) {
|
|
const transient = ['EPERM', 'EBUSY', 'EACCES'].includes(err.code);
|
|
if (process.platform !== 'win32' || !transient || attempt >= 10) throw err;
|
|
sleepSync(attempt * 200);
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Delete a build directory without ever following a link out of it.
|
|
*
|
|
* .next/node_modules holds links into the real node_modules for the external
|
|
* packages (better-sqlite3, ws, undici, signalr) — symlinks on macOS/Linux,
|
|
* possibly junctions on Windows. A recursive delete that followed one would
|
|
* empty ROOT/node_modules/better-sqlite3, and the next restart would crash-loop
|
|
* until someone ran npm install. So walk the tree with lstat and unlink every
|
|
* link as a link (lstat reports Windows junctions as symbolic links too); only
|
|
* real files and directories are removed by content.
|
|
*/
|
|
function removeDir(dir) {
|
|
let st;
|
|
try { st = lstatSync(dir); } catch (err) { if (err.code === 'ENOENT') return; throw err; }
|
|
if (st.isSymbolicLink()) { removeLink(dir); return; }
|
|
if (!st.isDirectory()) { rmSync(dir, { force: true, maxRetries: 10, retryDelay: 200 }); return; }
|
|
for (const name of readdirSync(dir)) removeDir(path.join(dir, name));
|
|
// Now empty. Windows can briefly refuse (indexer/Defender handles).
|
|
for (let attempt = 1; ; attempt++) {
|
|
try {
|
|
rmdirSync(dir);
|
|
return;
|
|
} catch (err) {
|
|
if (err.code === 'ENOENT') return;
|
|
const transient = ['EPERM', 'EBUSY', 'EACCES', 'ENOTEMPTY'].includes(err.code);
|
|
if (process.platform !== 'win32' || !transient || attempt >= 10) throw err;
|
|
sleepSync(attempt * 200);
|
|
}
|
|
}
|
|
}
|
|
|
|
/** A symlink unlinks; a Windows junction (a directory link) needs rmdir, which
|
|
* removes the link itself and never touches its target. */
|
|
function removeLink(link) {
|
|
try {
|
|
unlinkSync(link);
|
|
} catch (err) {
|
|
if (process.platform === 'win32' && ['EPERM', 'EISDIR', 'EACCES'].includes(err.code)) rmdirSync(link);
|
|
else if (err.code !== 'ENOENT') throw err;
|
|
}
|
|
}
|
|
|
|
// ── lock ────────────────────────────────────────────────────────────────────
|
|
// A build outlasts the schedule interval, so overlapping runs are otherwise a
|
|
// certainty rather than a risk. Only APPLY takes the lock; CHECK only looks.
|
|
|
|
function pidAlive(pid) {
|
|
try {
|
|
process.kill(pid, 0);
|
|
return true;
|
|
} catch (err) {
|
|
return err.code === 'EPERM'; // exists, just not ours to signal
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Whether anything is left in the process group `pid` led. A worker is a group
|
|
* leader (startWorker spawns it detached, and a shell gives a foreground run a
|
|
* group of its own), and its npm and next build children are in that group, so
|
|
* they outlive a worker that is killed on its own. POSIX only: Windows has no
|
|
* process groups to ask, and says no.
|
|
*/
|
|
function groupAlive(pid) {
|
|
if (process.platform === 'win32' || pid <= 1) return false;
|
|
try {
|
|
process.kill(-pid, 0);
|
|
return true;
|
|
} catch (err) {
|
|
return err.code === 'EPERM';
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Who holds the lock, and whether they still can. null = no lock.
|
|
*
|
|
* The pid check is what matters: a worker killed outright (or a machine
|
|
* rebooted mid-update) never reaches its `finally`, and waiting out an age
|
|
* limit would hold every later update back for it. The age limit is the
|
|
* backstop for a worker that is alive but wedged, and for a pid reused by an
|
|
* unrelated process.
|
|
*
|
|
* A dead pid is not the whole story, though. A worker killed on its own leaves
|
|
* its build running, still writing into .next, and a worker that took over
|
|
* would pull .next out from under it and start a second build into the same
|
|
* directory. So the lock stays held until the dead worker's process group is
|
|
* empty too (orphaned: true).
|
|
*
|
|
* Age and "taken before the last boot" are measured on the uptime clock the
|
|
* lock records, not on the wall clock. The wall clock can step by hours under
|
|
* a live worker: a Pi has no RTC and jumps forward at its first NTP sync after
|
|
* boot, and a VM resumed from a pause does the same. A jump must never make a
|
|
* running worker's lock look abandoned, because the worker that took over
|
|
* would put .next-prev back under the first one's build, and both would build
|
|
* and restart. Uptime only goes backwards across a reboot.
|
|
*/
|
|
function readLock() {
|
|
let raw;
|
|
let mtime;
|
|
try {
|
|
raw = readFileSync(LOCK_FILE, 'utf8').trim();
|
|
mtime = statSync(LOCK_FILE).mtimeMs;
|
|
} catch {
|
|
return null;
|
|
}
|
|
let info = null;
|
|
try { info = JSON.parse(raw); } catch { /* legacy, or caught between create and write */ }
|
|
|
|
if (info && typeof info === 'object' && Number.isInteger(info.pid)) {
|
|
const upNow = os.uptime();
|
|
const upThen = Number(info.uptime);
|
|
const hasUptime = Number.isFinite(upThen) && upThen > 0;
|
|
const rebooted = hasUptime && upNow < upThen;
|
|
const age = hasUptime && !rebooted
|
|
? (upNow - upThen) * 1000
|
|
: Date.now() - (Number(info.startedAt) || mtime);
|
|
const mins = Math.round(age / 60000);
|
|
if (!pidAlive(info.pid)) {
|
|
// Nothing outlives a reboot, and the age limit caps a wedged
|
|
// orphaned build just as it caps a wedged worker.
|
|
if (!rebooted && age <= STALE_LOCK_MS && groupAlive(info.pid)) return { stale: false, pid: info.pid, age, orphaned: true };
|
|
return { stale: true, pid: info.pid, age, why: `pid ${info.pid} is gone, ${mins} min old` };
|
|
}
|
|
if (rebooted) return { stale: true, pid: info.pid, age, why: 'taken before the last boot' };
|
|
if (age > STALE_LOCK_MS) return { stale: true, pid: info.pid, age, why: `pid ${info.pid} has held it for ${mins} min` };
|
|
return { stale: false, pid: info.pid, age };
|
|
}
|
|
|
|
// The pre-worker version of this script wrote a bare Date.now(). It ran
|
|
// under the cron, which kills it at the next minute boundary, so one of
|
|
// those locks older than a few minutes was left by a killed run, and the
|
|
// box behind it may be sitting on a wiped .next. Waiting the old hour out
|
|
// before repairing it would be the wrong trade. An empty file is a new lock
|
|
// caught between create and write; its mtime is as good as a timestamp.
|
|
const since = /^\d+$/.test(raw) ? Number(raw) : mtime;
|
|
const age = Date.now() - since;
|
|
if (age < LEGACY_STALE_LOCK_MS) return { stale: false, pid: null, age };
|
|
return { stale: true, pid: null, age, why: `left by an older updater, ${Math.round(age / 60000)} min old` };
|
|
}
|
|
|
|
/** Whether the lock on disk names this process. */
|
|
function ownsLock() {
|
|
try {
|
|
return JSON.parse(readFileSync(LOCK_FILE, 'utf8')).pid === process.pid;
|
|
} catch {
|
|
return false; // absent or unreadable: not ours
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Atomic create ('wx'), so two workers can never both believe they won.
|
|
*
|
|
* Breaking a stale lock is not atomic, though: it is a read, then a delete.
|
|
* Workers racing over the same stale lock (two checks at once, or a check and
|
|
* a by-hand --apply) each judge it stale, and a later one's delete then removes
|
|
* the lock an earlier one has just created, so several carry on at once. The
|
|
* delete follows its read at once (nothing, not even the log line, sits
|
|
* between them), so after a pause the file names the one worker that really
|
|
* holds the lock, and the rest back off.
|
|
*/
|
|
function acquireLock() {
|
|
for (let attempt = 0; attempt < 2; attempt++) {
|
|
try {
|
|
const fd = openSync(LOCK_FILE, 'wx');
|
|
try {
|
|
writeSync(fd, JSON.stringify({ pid: process.pid, startedAt: Date.now(), uptime: os.uptime() }));
|
|
} finally {
|
|
closeSync(fd);
|
|
}
|
|
sleepSync(LOCK_SETTLE_MS);
|
|
return ownsLock();
|
|
} catch (err) {
|
|
if (err.code !== 'EEXIST') throw err;
|
|
}
|
|
const lock = readLock();
|
|
if (lock && !lock.stale) return false;
|
|
// No lock by now means its holder just released it: nothing to delete,
|
|
// and a delete here could only hit a lock someone made since.
|
|
if (lock) {
|
|
rmSync(LOCK_FILE, { force: true });
|
|
log(`clearing a stale lock (${lock.why})`);
|
|
}
|
|
}
|
|
return false;
|
|
}
|
|
|
|
/** Remove the lock only if it is still ours. One taken over as stale belongs to its new holder. */
|
|
function releaseLock() {
|
|
if (!ownsLock()) return;
|
|
try { rmSync(LOCK_FILE, { force: true }); } catch { /* already gone */ }
|
|
}
|
|
|
|
// ── failure record ──────────────────────────────────────────────────────────
|
|
// scripts/.update-failed = { target, step, at, count, restart, clicker, detail? }.
|
|
//
|
|
// Written when an attempt STARTS and deleted when it succeeds, so a worker
|
|
// that dies without reporting (killed, rebooted) still counts as a failed
|
|
// attempt. Without that a crash would be retried every minute (the first one
|
|
// is retried at once, see retryAt, but only the first). `restart`
|
|
// marks a target whose build finished but whose restarts did not; the check
|
|
// has no other way to see that, because .built-commit already matches.
|
|
|
|
function readFailure() {
|
|
try {
|
|
const f = JSON.parse(readFileSync(FAILED_FILE, 'utf8'));
|
|
return f && typeof f === 'object' && typeof f.target === 'string' ? f : null;
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
function writeFailure(f) {
|
|
try { writeFileSync(FAILED_FILE, JSON.stringify(f) + '\n'); } catch { /* bookkeeping */ }
|
|
}
|
|
function clearFailure() {
|
|
try { rmSync(FAILED_FILE, { force: true }); } catch { /* bookkeeping */ }
|
|
}
|
|
/** Size of update-worker.log right now, so a step's own output can be told apart. */
|
|
function workerLogSize() {
|
|
try { return statSync(WORKER_LOG_FILE).size; } catch { return 0; }
|
|
}
|
|
|
|
/** What update-worker.log gained since `from` ('' when a by-hand --apply
|
|
* wrote to the terminal instead). */
|
|
function workerLogSince(from) {
|
|
try {
|
|
const buf = readFileSync(WORKER_LOG_FILE);
|
|
return buf.length > from ? buf.subarray(from).toString('utf8') : '';
|
|
} catch {
|
|
return '';
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Why a step failed, from its output since `from`: the file this user can't
|
|
* replace when a permission is what stopped it, else one line of the output.
|
|
*/
|
|
function explainFailure(from) {
|
|
const text = workerLogSince(from);
|
|
return (PERMISSION_DENIED.test(text) && ownershipProblem()) || failureReason(text);
|
|
}
|
|
|
|
/**
|
|
* One line on why a step failed, from its output, or null.
|
|
*
|
|
* The full npm/next output stays on the machine. Without a reason the master
|
|
* shows only "update failed at build", and a machine failing for a cause of
|
|
* its own (a Node upgrade leaving better-sqlite3 on the old ABI, a file
|
|
* Defender holds, a full disk) can't be told from a broken push. The line
|
|
* rides on .last-check, which every app build and the master already carry.
|
|
*/
|
|
function failureReason(text) {
|
|
// eslint-disable-next-line no-control-regex
|
|
const lines = text.replace(/\x1b\[[0-9;]*m/g, '').split(/\r?\n/).map((l) => l.trim()).filter(Boolean);
|
|
// npm's own epilogue says only that the script failed, and where its log
|
|
// is; and this script's own log lines (they go to the console too) aren't
|
|
// the step's output.
|
|
const boilerplate = /^npm (ERR!|error)( (code|path|command|errno|syscall|signal|workspace|location|A complete log|Lifecycle)\b|\s*$)|^> |^\d{4}-\d\d-\d\dT\S+ {2}/;
|
|
const first = (re) => lines.find((l) => re.test(l) && !boilerplate.test(l));
|
|
const line = first(/NODE_MODULE_VERSION|ERR_DLOPEN_FAILED|\bE(PERM|BUSY|ACCES|NOSPC)\b|heap out of memory|Type error:/)
|
|
?? first(/\b[Ee]rror\b/)
|
|
?? lines.filter((l) => !boilerplate.test(l)).pop();
|
|
return line ? line.replace(/\s+/g, ' ').slice(0, 100) : null;
|
|
}
|
|
|
|
/** 15 min after the first failure, an hour after repeated ones: long enough
|
|
* that a deterministic failure (a broken push) doesn't rebuild every minute
|
|
* on a trading PC, short enough that a transient one (network, EPERM) heals
|
|
* on its own.
|
|
*
|
|
* A first attempt that never reported (still 'started': its worker was
|
|
* killed, or the machine went down under it) is retried at once instead. That
|
|
* is a reboot or a person far more often than the update itself, and until
|
|
* the retry the only complete build may be the one set aside in .next-prev,
|
|
* where neither a restart nor a boot can find it: after a reboot mid-build
|
|
* the instances would sit on a partial .next for the whole backoff. A second
|
|
* death is a pattern, and waits like any other failure. */
|
|
const retryAt = (f) => Number(f.at) + (f.count > 1 ? REPEAT_RETRY_MS : f.step === 'started' ? 0 : FIRST_RETRY_MS);
|
|
|
|
// ── what is deployed ────────────────────────────────────────────────────────
|
|
|
|
/**
|
|
* When HEAD last moved (epoch ms), or null when there is no way to tell.
|
|
* The reflog's own timestamp, not the commit date: what matters is when this
|
|
* checkout changed, not when the commit was written.
|
|
*
|
|
* The selector is picked out of the default output ("<hash> HEAD@{<unix>}: ...")
|
|
* rather than asked for with --format=%gd: on Windows this goes through cmd,
|
|
* and a % on a cmd command line is one typo away from variable expansion. A
|
|
* git too old for --date=unix prints HEAD@{0}, which would read as "HEAD never
|
|
* moved" and adopt anything, hence the sanity bound.
|
|
*/
|
|
function headMovedAt() {
|
|
try {
|
|
const m = /HEAD@\{(\d+)\}/.exec(git('reflog', '-1', '--date=unix'));
|
|
if (m && Number(m[1]) > 1e9) return Number(m[1]) * 1000;
|
|
} catch { /* no reflog: fall through */ }
|
|
try {
|
|
const head = readFileSync(path.join(ROOT, '.git', 'HEAD'), 'utf8').trim();
|
|
const ref = /^ref: (.+)$/.exec(head)?.[1];
|
|
return statSync(path.join(ROOT, '.git', ref ?? 'HEAD')).mtimeMs;
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
/** PM2's process list, or null when PM2 can't be asked. */
|
|
function pm2List() {
|
|
const r = run('pm2', ['jlist'], { capture: true, timeout: 30_000 });
|
|
if (r.status !== 0) return null;
|
|
// jlist prints the process list as one JSON line, possibly after "[PM2] ..." chatter.
|
|
const line = (r.stdout ?? '').split('\n').find((l) => /^\s*\[\s*[{\]]/.test(l));
|
|
try {
|
|
const procs = JSON.parse(line);
|
|
return Array.isArray(procs) ? procs : null;
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Names of this checkout's PM2 instances that started before `ms`, i.e. still
|
|
* running code from before the build on disk. Empty when PM2 can't be asked:
|
|
* this only sharpens a heuristic and must never block an update.
|
|
*/
|
|
function instancesStartedBefore(ms) {
|
|
const procs = pm2List();
|
|
if (!procs) return [];
|
|
const names = new Set(INSTANCES.map((i) => i.name));
|
|
return procs
|
|
.filter((p) => names.has(p?.name) && p.pm2_env?.status === 'online' && Number(p.pm2_env.pm_uptime) < ms)
|
|
.map((p) => p.name);
|
|
}
|
|
|
|
/**
|
|
* The commit .next was built from: { commit } or { commit: null, why }.
|
|
*
|
|
* scripts/.built-commit is written only after `npm run build` succeeds. When it
|
|
* is absent (every machine, the first time this version runs) the build on
|
|
* disk is judged instead. Adopting it as HEAD avoids a pointless rebuild and
|
|
* restart on every healthy machine. Anything doubtful counts as unknown, and
|
|
* unknown means rebuild. That is what rescues machines the old updater
|
|
* stranded: their HEAD already equals origin, but their build is stale,
|
|
* partial or missing.
|
|
*/
|
|
function builtCommit(head, { adopt }) {
|
|
try {
|
|
const recorded = readFileSync(BUILT_FILE, 'utf8').trim();
|
|
if (/^[0-9a-f]{40}([0-9a-f]{24})?$/.test(recorded)) return { commit: recorded };
|
|
} catch { /* absent: first run of this version on this machine */ }
|
|
|
|
if (!buildComplete(NEXT_DIR)) return { commit: null, why: 'no complete build on disk' };
|
|
const builtAt = statSync(BUILD_ID_FILE).mtimeMs;
|
|
const movedAt = headMovedAt();
|
|
if (movedAt === null) return { commit: null, why: 'cannot tell when HEAD last moved' };
|
|
if (builtAt < movedAt) return { commit: null, why: 'the build on disk is older than HEAD' };
|
|
// Built after HEAD moved, but a run killed between the build and the
|
|
// restarts leaves the old processes serving the new .next.
|
|
const behind = instancesStartedBefore(builtAt);
|
|
if (behind.length) return { commit: null, why: `${behind.join(', ')} started before the build on disk` };
|
|
|
|
if (adopt) {
|
|
writeFileSync(BUILT_FILE, head + '\n');
|
|
log(`adopted the build on disk as ${short(head)} (first run with scripts/.built-commit)`);
|
|
}
|
|
return { commit: head };
|
|
}
|
|
|
|
function isAncestor(a, b) {
|
|
const r = run('git', ['merge-base', '--is-ancestor', a, b], { capture: true });
|
|
if (r.status === 0) return true;
|
|
if (r.status === 1) return false;
|
|
throw new Error(`git merge-base failed: ${(r.stderr ?? '').trim()}`);
|
|
}
|
|
|
|
/** Files changed from→to, or null when `from` is unknown (treat everything as changed). */
|
|
function changedFiles(from, to) {
|
|
if (!from) return null;
|
|
if (from === to) return [];
|
|
try {
|
|
return git('diff', '--name-only', from, to).split('\n').filter(Boolean);
|
|
} catch {
|
|
return null; // e.g. the recorded commit is gone
|
|
}
|
|
}
|
|
|
|
/** Fetch and work out what, if anything, needs doing. Shared by CHECK and APPLY. */
|
|
function inspect({ adopt }) {
|
|
fetchOrigin();
|
|
const local = git('rev-parse', 'HEAD');
|
|
const remote = git('rev-parse', `origin/${BRANCH}`);
|
|
// A checkout AHEAD of origin (a commit made on this machine) has nothing to
|
|
// pull. `pull --ff-only` would "succeed" as a no-op and the old updater then
|
|
// rebuilt and restarted on every tick.
|
|
const needsPull = local !== remote && !isAncestor(remote, local);
|
|
const target = needsPull ? remote : local;
|
|
const built = builtCommit(local, { adopt });
|
|
// Complete, not merely present: a build of the SAME commit killed after
|
|
// BUILD_ID was written (a by-hand `npm run build`, or a worker redoing a
|
|
// missing build) would otherwise pass for up to date indefinitely.
|
|
const buildMissing = !buildComplete(NEXT_DIR);
|
|
const needsBuild = built.commit !== target || buildMissing;
|
|
const failed = readFailure();
|
|
const pending = failed?.target === target ? failed : null;
|
|
const needsRestart = !needsBuild && pending?.restart === true;
|
|
return { local, remote, target, needsPull, built, buildMissing, needsBuild, needsRestart, pending };
|
|
}
|
|
|
|
function describe(plan) {
|
|
if (plan.needsPull) return `${short(plan.local)} -> ${short(plan.remote)}`;
|
|
if (plan.needsBuild) {
|
|
const why = plan.buildMissing ? 'no complete build on disk' : plan.built.why ?? `last built ${short(plan.built.commit)}`;
|
|
return `rebuild ${short(plan.target)} (${why})`;
|
|
}
|
|
return `restart onto ${short(plan.target)} (the last restart did not complete)`;
|
|
}
|
|
|
|
/** Which of the optional steps this plan needs. */
|
|
function workFor(plan) {
|
|
const changed = changedFiles(plan.built.commit, plan.target);
|
|
const touched = (f) => changed === null || changed.includes(f);
|
|
return {
|
|
changed,
|
|
npmInstall: plan.needsBuild && touched('package-lock.json'),
|
|
pythonDeps: plan.needsBuild && touched('clicker/requirements.txt'),
|
|
restartClicker: plan.needsBuild
|
|
? changed === null || changed.some((f) => f.startsWith('clicker/'))
|
|
: plan.needsRestart && plan.pending?.clicker === true,
|
|
};
|
|
}
|
|
|
|
// ── stuck states ────────────────────────────────────────────────────────────
|
|
// Two states that fail every update until a person gets onto the machine.
|
|
|
|
/** What a step prints when it may not touch a file. */
|
|
const PERMISSION_DENIED = /permission denied|insufficient permission|\bEACCES\b|\bEPERM\b/i;
|
|
|
|
/**
|
|
* A file in the checkout this user doesn't own, as the fix to run, or null.
|
|
*
|
|
* One `sudo git pull` or `sudo npm run build` leaves files this user cannot
|
|
* replace, and from then on every update fails: the build can't clear .next,
|
|
* and a fetch can't write into .git (Ziad's Pi, 2026-10-02: ~1,800 of them).
|
|
* The failure itself only says EACCES somewhere in npm's output, so name the
|
|
* file and the one command that fixes it — the master's updater chip shows
|
|
* this line. Asked only once a step has failed on a permission: a stray
|
|
* root-owned file that blocks nothing must not stop an update. POSIX only.
|
|
*/
|
|
function ownershipProblem() {
|
|
if (typeof process.getuid !== 'function') return null;
|
|
const r = run('find', [ROOT, '!', '-user', String(process.getuid()), '-print', '-quit'], { capture: true, timeout: 20_000 });
|
|
const hit = (r.stdout ?? '').split('\n')[0].trim();
|
|
if (!hit) return null;
|
|
let owner = 'another user';
|
|
try {
|
|
const uid = lstatSync(hit).uid;
|
|
owner = uid === 0 ? 'root' : `uid ${uid}`;
|
|
} catch { /* gone since: the generic name will do */ }
|
|
return `${path.relative(ROOT, hit) || '.'} is owned by ${owner}: run sudo chown -R ${os.userInfo().username} ${ROOT}`;
|
|
}
|
|
|
|
/** inspect(), with a fetch refused by a .git this user can't write named as such. */
|
|
function inspectNamingOwnership(opts) {
|
|
try {
|
|
return inspect(opts);
|
|
} catch (err) {
|
|
const why = PERMISSION_DENIED.test(err.message) ? ownershipProblem() : null;
|
|
throw why ? new Error(why) : err;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* npm install rewrites package-lock.json on some machines (another npm
|
|
* version, platform-specific optional dependencies), and that edit then
|
|
* refuses every fast-forward that touches the file. It is generated, so when
|
|
* it is the only tracked edit, put it back. Any other edit is a person's and
|
|
* stays, showing as "edited here" on the master.
|
|
*/
|
|
function restoreRewrittenLockfile() {
|
|
let changed;
|
|
try {
|
|
changed = git('diff', '--name-only', 'HEAD').split('\n').filter(Boolean);
|
|
} catch {
|
|
return;
|
|
}
|
|
if (changed.length !== 1 || changed[0] !== 'package-lock.json') return;
|
|
git('checkout', 'HEAD', '--', 'package-lock.json');
|
|
log('package-lock.json had been rewritten here (npm) — put it back before the pull');
|
|
}
|
|
|
|
// ── build ───────────────────────────────────────────────────────────────────
|
|
|
|
/**
|
|
* npm run build, with the previous build set aside so a failure can be undone.
|
|
*
|
|
* next build empties .next as its first step, under the live server, whatever
|
|
* we do. Renaming it to .next-prev first costs nothing extra, and it means a
|
|
* FAILED build can be rolled back instead of leaving the box on a half-written
|
|
* .next that a restart or a reboot would then crash-loop on. When the rename
|
|
* is refused (Windows, something holding a handle), build in place without a
|
|
* safety net rather than not update at all.
|
|
*/
|
|
function build() {
|
|
// A worker that died between setting the old build aside and putting it
|
|
// back leaves the only complete build in .next-prev. Restore it first, or
|
|
// the cleanup below would delete it. If it can't be put back, it stays
|
|
// where it is (clearing it would leave no complete build anywhere), this
|
|
// build runs in place, and a failure tries the restore once more.
|
|
let prevIsOnlyBuild = false;
|
|
if (buildComplete(PREV_DIR) && !buildComplete(NEXT_DIR)) {
|
|
try {
|
|
removeDir(NEXT_DIR);
|
|
renameWithRetries(PREV_DIR, NEXT_DIR);
|
|
log('put back the previous build left in .next-prev by an interrupted update');
|
|
} catch (err) {
|
|
prevIsOnlyBuild = true;
|
|
log(`could not put back .next-prev (${err.code ?? err.message}) — leaving it there and building in place`);
|
|
}
|
|
}
|
|
|
|
let backedUp = false;
|
|
const hadBuild = existsSync(NEXT_DIR);
|
|
if (hadBuild && !prevIsOnlyBuild) {
|
|
try {
|
|
removeDir(PREV_DIR);
|
|
renameWithRetries(NEXT_DIR, PREV_DIR);
|
|
backedUp = true;
|
|
} catch (err) {
|
|
log(`could not set the previous build aside (${err.code ?? err.message}) — building in place, no rollback this time`);
|
|
}
|
|
}
|
|
// next build keeps .next/cache (the incremental type-check state among it)
|
|
// from one build to the next. Carry it over so setting .next aside doesn't
|
|
// make every build a cold one. Best effort: without it a build is only
|
|
// slower. Not carried back on failure, so a retry starts from a clean cache.
|
|
if (backedUp) {
|
|
try {
|
|
mkdirSync(NEXT_DIR);
|
|
renameSync(path.join(PREV_DIR, 'cache'), path.join(NEXT_DIR, 'cache'));
|
|
} catch { /* a cold build */ }
|
|
}
|
|
|
|
log('building');
|
|
if (run('npm', ['run', 'build']).status === 0) return true;
|
|
|
|
if (backedUp || prevIsOnlyBuild) {
|
|
try {
|
|
removeDir(NEXT_DIR);
|
|
renameWithRetries(PREV_DIR, NEXT_DIR);
|
|
log('build failed — restored the previous build');
|
|
} catch (err) {
|
|
log(`build failed, and restoring the previous build failed too (${err.code ?? err.message}) — it is still in .next-prev`);
|
|
}
|
|
} else if (hadBuild) {
|
|
log('build failed — it ran in place, so there is no previous build to restore');
|
|
} else {
|
|
log('build failed — there was no previous build to restore');
|
|
}
|
|
return false;
|
|
}
|
|
|
|
function removePrevBuild() {
|
|
if (!existsSync(PREV_DIR)) return;
|
|
try {
|
|
removeDir(PREV_DIR);
|
|
} catch (err) {
|
|
log(`could not remove .next-prev (${err.code ?? err.message}) — the next build clears it`);
|
|
}
|
|
}
|
|
|
|
// ── heal ────────────────────────────────────────────────────────────────────
|
|
// The checker runs only while PM2 fires it, and PM2 loses it easily: a
|
|
// `pm2 stop`, a `pm2 delete all`, a reboot after a list change nobody saved.
|
|
|
|
/** How long ago the last check ran, in ms; null = never on this checkout. */
|
|
function lastCheckAge() {
|
|
try {
|
|
const at = Date.parse(readFileSync(LAST_CHECK_FILE, 'utf8').split(/\s/, 1)[0]);
|
|
return Date.now() - (Number.isFinite(at) ? at : statSync(LAST_CHECK_FILE).mtimeMs);
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
const fmtAge = (ms) => (ms >= 90 * 60 * 1000 ? `${Math.round(ms / 3_600_000)} h` : `${Math.round(ms / 60_000)} min`);
|
|
|
|
/**
|
|
* The fields PM2 adds to an app's environment.
|
|
*
|
|
* PM2 hands every app the scalar fields of its own record as environment
|
|
* (name, pm_id, autorestart, …), and `pm2 start` captures the caller's
|
|
* environment for the new app, then copies it over the new app's record. A
|
|
* checker registered by a heal, which the dashboard app starts, would take
|
|
* that app's name and id, and its autorestart would override
|
|
* --no-autorestart. These are the fields PM2 put on real records, plus the
|
|
* ones only some records carry. PORT and AUTOFIRMER_DB are the instance's
|
|
* own, and the checker serves every instance.
|
|
*/
|
|
const INHERITED_FIELDS = new Set([
|
|
'NODE_APP_INSTANCE', 'automation', 'autorestart', 'autostart', 'created_at', 'cron_restart',
|
|
'exec_interpreter', 'exec_mode', 'exit_code', 'filter_env', 'instance_var', 'instances',
|
|
'kill_retry_time', 'km_link', 'merge_logs', 'name', 'namespace', 'node_version', 'pmx',
|
|
'prev_restart_delay', 'restart_time', 'status', 'stop_exit_codes', 'treekill', 'unique_id',
|
|
'unstable_restarts', 'username', 'version', 'versioning', 'vizion', 'vizion_running', 'watch',
|
|
'windowsHide', 'PORT', 'AUTOFIRMER_DB',
|
|
]);
|
|
|
|
/** Drop what PM2 (and the instance) put in this process's environment, so
|
|
* nothing started from here inherits it. */
|
|
function scrubInheritedEnv() {
|
|
// PM2 also files the app's own config under its name.
|
|
if (process.env.name) delete process.env[process.env.name];
|
|
for (const key of Object.keys(process.env)) {
|
|
if (INHERITED_FIELDS.has(key) || key.startsWith('pm_') || key.startsWith('axm_')) delete process.env[key];
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Get PM2 firing this script again once its checks have stopped. Returns
|
|
* false when this checkout is opted out of updates altogether.
|
|
*
|
|
* What the fleet shows (2026-10, six Windows machines): the cron dies inside
|
|
* a live PM2 daemon, nobody involved. Every fire runs restartProcessId,
|
|
* whose first step drops the cron; only a restart that gets as far as
|
|
* launching the app registers it again. croner sometimes fires twice in
|
|
* one second, the second restart has to stop a check that is still
|
|
* launching, and a refused or timed-out stop leaves the checker `stopped`
|
|
* or `errored` with no cron at all.
|
|
*
|
|
* So a checker PM2 still lists is restarted, which registers its cron again
|
|
* with the schedule and environment it already had, and nothing is deleted:
|
|
* `pm2 delete` kills the pid an `errored` entry kept, and Windows hands pids
|
|
* out again quickly, so that could be any program by now. If that pid is
|
|
* alive, PM2 is left alone and this run checks instead. Only a checker PM2
|
|
* has lost entirely is registered again, the way install-autostart.* does.
|
|
*/
|
|
function heal() {
|
|
if (existsSync(OPT_OUT_FILE)) {
|
|
console.log('scripts/.no-auto-update is here — this checkout is not auto-updated');
|
|
return false;
|
|
}
|
|
// Before anything is started from here, the worker below included: the
|
|
// app that asked for this heal is a PM2 app, and may be instance 2.
|
|
scrubInheritedEnv();
|
|
const age = lastCheckAge();
|
|
if (age !== null && age <= STALE_CHECK_MS) return true; // checking on schedule
|
|
const why = age === null ? 'the update checker had never run here' : `the update checker had stopped (last check ${fmtAge(age)} ago)`;
|
|
|
|
const pm2 = (...args) => run('pm2', args.map(shellArg), { capture: true, timeout: PM2_TIMEOUT_MS });
|
|
const lastLine = (r) => (((r.stderr || r.stdout) ?? '').trim().split('\n').pop() || `exit ${r.status}`);
|
|
const listed = pm2List()?.find((p) => p?.name === CHECKER);
|
|
const pid = Number(listed?.pid);
|
|
if (listed && pid > 0 && pidAlive(pid)) {
|
|
log(`${why}, but PM2 still has it on pid ${pid}, which is alive — left PM2 alone (a restart would kill that pid); checking from here`);
|
|
return true;
|
|
}
|
|
|
|
let how;
|
|
if (listed && pm2('restart', CHECKER).status === 0) {
|
|
how = `restarted it in PM2 (${listed.pm2_env?.status ?? 'unknown'}), which registers its cron again`;
|
|
} else {
|
|
const cron = listed?.pm2_env?.cron_restart || DEFAULT_CRON;
|
|
if (listed) pm2('delete', CHECKER); // its restart failed, and its pid is dead
|
|
const r = pm2('start', SELF, '--name', CHECKER, '--no-autorestart', '--cron-restart', cron);
|
|
if (r.status !== 0) {
|
|
log(`${why} — ERROR: could not register it with PM2: ${lastLine(r)}`);
|
|
return true; // the check below can still update this machine
|
|
}
|
|
how = `registered it with PM2 again (${cron})`;
|
|
}
|
|
const saved = pm2('save').status === 0;
|
|
log(`${why} — ${how}${saved ? '' : ', but pm2 save failed: a reboot would lose it'}`);
|
|
return true;
|
|
}
|
|
|
|
// ── check ───────────────────────────────────────────────────────────────────
|
|
|
|
/**
|
|
* Start `update-check.mjs --apply` detached, logging to update-worker.log.
|
|
* The log is truncated per run: it holds the full npm/next output of the
|
|
* latest attempt, the first place to look when one fails.
|
|
*/
|
|
function startWorker() {
|
|
let out = 'ignore';
|
|
try { out = openSync(WORKER_LOG_FILE, 'w'); } catch { /* run it without a log rather than not at all */ }
|
|
try {
|
|
const child = spawn(process.execPath, [SELF, '--apply'], {
|
|
cwd: ROOT,
|
|
detached: true,
|
|
windowsHide: true,
|
|
stdio: ['ignore', out, out],
|
|
});
|
|
child.on('error', () => { /* surfaces below as the missing pid */ });
|
|
child.unref();
|
|
if (!child.pid) throw new Error('could not start the update worker');
|
|
return child.pid;
|
|
} finally {
|
|
if (typeof out === 'number') closeSync(out);
|
|
}
|
|
}
|
|
|
|
function dryRunReport(plan) {
|
|
const work = workFor(plan);
|
|
log(`update available: ${describe(plan)}`);
|
|
if (work.changed) log(`${work.changed.length} file(s) changed since the last build`);
|
|
log('dry run — stopping before any change');
|
|
const restarts = plan.needsBuild || plan.needsRestart;
|
|
log(`would run: ${[
|
|
plan.needsPull && 'git pull --ff-only',
|
|
work.npmInstall && 'npm install',
|
|
work.pythonDeps && 'pip install',
|
|
plan.needsBuild && 'npm run build',
|
|
...(restarts ? INSTANCES.map((i) => `pm2 restart ${i.name}`) : []),
|
|
work.restartClicker && 'pm2 restart clicker',
|
|
].filter(Boolean).join(', ')}`);
|
|
if (plan.pending) {
|
|
log(`last attempt at ${short(plan.target)}: ${plan.pending.step} (attempt ${plan.pending.count}); next automatic try after ${new Date(retryAt(plan.pending)).toISOString()}`);
|
|
}
|
|
}
|
|
|
|
function check() {
|
|
const lock = readLock();
|
|
if (lock && !lock.stale) {
|
|
const who = !lock.pid ? 'an older updater'
|
|
: lock.orphaned ? `the build left by dead worker pid ${lock.pid}`
|
|
: `worker pid ${lock.pid}`;
|
|
console.log(`an update is already running (${who}) — exiting`);
|
|
if (!DRY_RUN) recordCheck(`update in progress (${who}, ${Math.round(lock.age / 60000)} min)`);
|
|
return 0;
|
|
}
|
|
|
|
let outcome = null;
|
|
try {
|
|
const plan = inspectNamingOwnership({ adopt: !DRY_RUN });
|
|
if (!plan.needsPull && !plan.needsBuild && !plan.needsRestart) {
|
|
outcome = `up to date at ${short(plan.target)}`; // silent in the log by design
|
|
if (DRY_RUN) console.log(outcome);
|
|
return 0;
|
|
}
|
|
if (DRY_RUN) {
|
|
dryRunReport(plan);
|
|
return 0;
|
|
}
|
|
// Silent in update.log too: the failure itself was logged once, and a
|
|
// line a minute until the retry would bury it. --now (the master's
|
|
// "Update now") is a person asking for the retry.
|
|
if (plan.pending && !NOW && Date.now() < retryAt(plan.pending)) {
|
|
outcome = `waiting to retry ${short(plan.target)}: ${plan.pending.step} (attempt ${plan.pending.count}), next try after ${new Date(retryAt(plan.pending)).toISOString()}`
|
|
+ (plan.pending.detail ? ` — ${plan.pending.detail}` : '');
|
|
return 0;
|
|
}
|
|
const pid = startWorker();
|
|
log(`update available: ${describe(plan)} — started worker pid ${pid}`);
|
|
outcome = `worker pid ${pid} started: ${describe(plan)}`;
|
|
return 0;
|
|
} catch (err) {
|
|
log(`ERROR: ${err.message}`);
|
|
outcome = `ERROR: ${err.message}`;
|
|
return 1;
|
|
} finally {
|
|
if (!DRY_RUN && outcome) recordCheck(outcome);
|
|
}
|
|
}
|
|
|
|
// ── apply ───────────────────────────────────────────────────────────────────
|
|
|
|
/**
|
|
* A kill skips `finally`, so the lock is released here too. The next check
|
|
* then retries after the usual backoff instead of waiting out a stale lock.
|
|
* Node runs this on the event loop, so it never lands in the middle of a
|
|
* spawnSync. An interrupted build still comes back through build()'s restore.
|
|
*/
|
|
function interrupted(signal) {
|
|
log(`${signal} — stopping; lock released${readFailure() ? ', the next check retries after the backoff' : ''}`);
|
|
releaseLock();
|
|
recordCheck(`interrupted by ${signal}`);
|
|
process.exit(1);
|
|
}
|
|
|
|
async function apply() {
|
|
let locked;
|
|
try {
|
|
locked = acquireLock();
|
|
} catch (err) {
|
|
log(`ERROR: could not take the update lock: ${err.message}`);
|
|
return 1;
|
|
}
|
|
if (!locked) {
|
|
console.log('another update is already running — exiting');
|
|
return 0;
|
|
}
|
|
for (const signal of ['SIGINT', 'SIGTERM', 'SIGHUP']) process.on(signal, () => interrupted(signal));
|
|
|
|
let outcome = 'interrupted';
|
|
let attempt = null;
|
|
const abort = (step, why, detail = null) => {
|
|
log(`ABORTED: ${why}${detail ? ` (${detail})` : ''}`);
|
|
outcome = detail ? `${why}: ${detail}` : why;
|
|
if (attempt) writeFailure({ ...attempt, step, at: Date.now(), ...(detail ? { detail } : {}) });
|
|
return 1;
|
|
};
|
|
|
|
try {
|
|
const plan = inspectNamingOwnership({ adopt: true });
|
|
if (!plan.needsPull && !plan.needsBuild && !plan.needsRestart) {
|
|
outcome = `up to date at ${short(plan.target)}`;
|
|
clearFailure();
|
|
return 0;
|
|
}
|
|
const { target } = plan;
|
|
const work = workFor(plan);
|
|
const because = (what) => (work.changed ? `${what} changed` : 'last build unknown');
|
|
attempt = {
|
|
target,
|
|
step: 'started',
|
|
at: Date.now(),
|
|
count: (plan.pending?.count ?? 0) + 1,
|
|
restart: plan.needsRestart,
|
|
clicker: plan.needsRestart && plan.pending?.clicker === true,
|
|
};
|
|
writeFailure(attempt);
|
|
|
|
log(`applying ${describe(plan)} (worker pid ${process.pid}, attempt ${attempt.count})`);
|
|
if (work.changed) log(`${work.changed.length} file(s) changed since the last build`);
|
|
else if (plan.needsBuild) log('last build unknown — treating every file as changed');
|
|
|
|
// The exact commit the check saw, not whatever `git pull` would fetch now.
|
|
if (plan.needsPull) {
|
|
restoreRewrittenLockfile();
|
|
const from = workerLogSize();
|
|
if (run('git', ['merge', '--ff-only', target]).status !== 0) {
|
|
return abort('pull', `git pull --ff-only to ${short(target)} failed — see scripts/update-worker.log`, explainFailure(from));
|
|
}
|
|
}
|
|
|
|
if (plan.needsBuild) {
|
|
if (work.npmInstall) {
|
|
log(`${because('package-lock.json')} — npm install`);
|
|
const from = workerLogSize();
|
|
if (run('npm', ['install']).status !== 0) return abort('npm install', 'npm install failed — nothing was built or restarted', explainFailure(from));
|
|
}
|
|
if (work.pythonDeps) {
|
|
log(`${because('clicker/requirements.txt')} — refreshing python deps`);
|
|
ensurePythonDeps({ log: (m) => log(m) }); // non-fatal by design
|
|
}
|
|
|
|
// Build BEFORE restarting. A failed build leaves the running process
|
|
// untouched, which is the whole point of doing it in this order.
|
|
const from = workerLogSize();
|
|
if (!build()) return abort('build', 'build failed — not deployed, nothing was restarted', explainFailure(from));
|
|
|
|
// .built-commit says what .next holds, so it changes the moment .next
|
|
// does. From here a failed restart must not rebuild, only restart,
|
|
// which the `restart` flag on the attempt tells the next check.
|
|
writeFileSync(BUILT_FILE, target + '\n');
|
|
attempt = { ...attempt, step: 'restart', at: Date.now(), restart: true, clicker: work.restartClicker };
|
|
writeFailure(attempt);
|
|
}
|
|
|
|
if (plan.needsBuild || plan.needsRestart) {
|
|
for (const instance of INSTANCES) {
|
|
log(`restarting ${instance.name}`);
|
|
if (run('pm2', ['restart', instance.name], { timeout: PM2_TIMEOUT_MS }).status !== 0) return abort('restart', `pm2 restart ${instance.name} failed`);
|
|
}
|
|
if (work.restartClicker) {
|
|
log(`${because('clicker')} — restarting it too`);
|
|
run('pm2', ['restart', 'clicker'], { timeout: PM2_TIMEOUT_MS });
|
|
}
|
|
}
|
|
|
|
removePrevBuild();
|
|
clearFailure();
|
|
await warmUp();
|
|
log(`updated to ${short(target)}`);
|
|
outcome = `updated to ${short(target)}`;
|
|
return 0;
|
|
} catch (err) {
|
|
log(`ERROR: ${err.message}`);
|
|
outcome = `ERROR: ${err.message}`;
|
|
if (attempt) writeFailure({ ...attempt, step: 'error', at: Date.now() });
|
|
return 1;
|
|
} finally {
|
|
releaseLock();
|
|
recordCheck(outcome);
|
|
}
|
|
}
|
|
|
|
async function warmUp() {
|
|
for (const instance of INSTANCES) {
|
|
const base = `http://127.0.0.1:${instance.port}`;
|
|
const deadline = Date.now() + 120_000;
|
|
while (Date.now() < deadline) {
|
|
try {
|
|
if ((await fetch(`${base}/api/auto-trade`, { signal: AbortSignal.timeout(5_000) })).ok) break;
|
|
} catch { /* still restarting */ }
|
|
await new Promise((r) => setTimeout(r, 2_000));
|
|
}
|
|
try {
|
|
await fetch(`${base}/api/state`, { signal: AbortSignal.timeout(60_000) });
|
|
const status = await (await fetch(`${base}/api/auto-trade`)).json();
|
|
log(status.running
|
|
? `${instance.name}: scheduler resumed — ${status.action} ${status.symbol}`
|
|
: `${instance.name}: scheduler is stopped`);
|
|
// The clicker/extension runner only ever serves instance 1.
|
|
if (instance.port === 3000) {
|
|
const runner = await (await fetch(`${base}/api/autobuyer/runner`)).json();
|
|
log(`runner ${runner.online ? 'online' : 'OFFLINE'}${runner.stale ? ' (version stale — reload the extension)' : ''}`);
|
|
}
|
|
} catch (err) {
|
|
log(`${instance.name}: post-restart check failed: ${err.message}`);
|
|
}
|
|
await logHealth(instance, base);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* One line from /api/health, when the build has it. Informational only: right
|
|
* after a restart the clients may not have synced yet, so this never fails an
|
|
* update. Parsed loosely, because the endpoint may be newer than this script.
|
|
*/
|
|
async function logHealth(instance, base) {
|
|
let res;
|
|
try {
|
|
res = await fetch(`${base}/api/health`, { signal: AbortSignal.timeout(10_000) });
|
|
} catch {
|
|
return;
|
|
}
|
|
if (res.status === 404) return; // a build from before the endpoint
|
|
let body = null;
|
|
try { body = await res.json(); } catch { /* not JSON: the status alone */ }
|
|
const parts = [`HTTP ${res.status}`];
|
|
if (typeof body?.ok === 'boolean') parts.push(body.ok ? 'ok' : 'NOT ok');
|
|
// A tradable firm with no client (no credentials yet) is listed with
|
|
// hasClient: false and is not something that failed to sync.
|
|
const listed = Array.isArray(body?.clients) ? body.clients : Array.isArray(body) ? body : null;
|
|
const clients = listed?.filter((c) => c?.hasClient !== false);
|
|
if (clients) {
|
|
const now = Date.now();
|
|
const ready = clients.filter((c) => {
|
|
if (typeof c?.ready === 'boolean') return c.ready;
|
|
const h = c?.health ?? c;
|
|
return h?.syncComplete === true && h.lastSyncAt > 0 && now - h.lastSyncAt < 5 * 60_000;
|
|
}).length;
|
|
parts.push(`${ready}/${clients.length} client(s) synced`);
|
|
}
|
|
log(`${instance.name}: health ${parts.join(', ')}`);
|
|
}
|
|
|
|
async function main() {
|
|
mkdirSync(path.join(ROOT, 'scripts'), { recursive: true });
|
|
trimLog();
|
|
if (HEAL && !DRY_RUN && !heal()) return 0;
|
|
return APPLY && !DRY_RUN ? apply() : check();
|
|
}
|
|
|
|
process.exit(await main());
|