Updater heals itself: PM2 cron revived, Update now, failure reasons

PM2's cron for autofirmer-update dies inside a live daemon (six Windows machines in October), and nothing on the machine noticed. The app now watches scripts/.last-check and, past 10 minutes, runs update-check.mjs --heal: a checker PM2 still lists is restarted (which re-registers its cron), one it lost is registered again, and one whose errored entry still holds a live pid is left alone, since PM2 would kill that pid. The heal scrubs PM2's per-app fields from its environment first. The master's new 'update-now' command runs --heal --now, skipping a failed update's retry wait. A failed pull, npm install or build records one line of reason (or the chown to run when a file isn't ours), which rides on .last-check to the master. A package-lock.json rewritten by npm is put back before the pull. The fetch timeout drops under the cron's minute. scripts/.no-auto-update opts a checkout out. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-19 00:41:04 -05:00
parent c0899620d4
commit 13e03c3d0f
6 changed files with 396 additions and 23 deletions
@@ -63,6 +63,8 @@ scripts/update.log
 scripts/.built-commit
 scripts/.update-failed
 scripts/update-worker.log
+# present = this checkout is never auto-updated or healed (a dev checkout)
+scripts/.no-auto-update
 # the previous build, set aside while the updater builds (Tailwind's source
 # scan also honours this, so it never reads the old bundles)
 /.next-prev/
@@ -12,6 +12,7 @@ import { startReporter, sendReport } from './reporter';
 import { startBackupSync } from './backup-sync';
 import { applyBrokerHttpDefaults, applyBrokerProxy } from './outbound-proxy';
 import { startMasterLink } from './master-link';
+import { startUpdaterWatchdog } from './updater-watchdog';
 import { resumeSchedulerIfPersisted } from './auto-trade';
 import { superviseQuotes, SUPERVISE_INTERVAL_MS, type QuoteClient } from './live-quotes';

@@ -227,6 +228,9 @@ export function getClients(): Map<string, PlatformClient> {
            // Start master dashboard reporter
            startReporter();

+            // Bring this machine's auto-updater back if its PM2 cron stops firing.
+            startUpdaterWatchdog();
+
            // Push settings+credentials backups to the master, so a dead VPS
            // can be replaced and this identity adopted.
            startBackupSync();
@@ -24,6 +24,7 @@ import { queueRun } from './queue-run';
 import { pushBackup } from './backup-sync';
 import { getBuildVersion } from './version';
 import { updaterStatus } from './updater-status';
+import { runUpdateNow } from './updater-watchdog';
 import type {
    Command,
    InstanceCapabilities,
@@ -87,6 +88,7 @@ export function collectCapabilities(): InstanceCapabilities {
                }))
        ),
        runnerOnline,
+        updateNow: true,
    };
 }

@@ -131,6 +133,9 @@ async function execute(command: Command): Promise<unknown> {
        case 'backup-now':
            return pushBackup(true);

+        case 'update-now':
+            return runUpdateNow();
+
        default: {
            // A newer master may know commands this build does not. Say so
            // rather than failing silently.
@@ -318,6 +318,8 @@ export interface InstanceCapabilities {
    symbols: string[];
    automations: AutomationSummary[];
    runnerOnline: boolean;
+    /** Answers the 'update-now' command; the master offers it only then. */
+    updateNow: boolean;
 }

 // ── Firm catalog ─────────────────────────────────────────────────────────────
@@ -508,7 +510,10 @@ export type Command =
    | { kind: 'autobuyer-run'; automationId: string; inputs?: Record<string, number> }
    | { kind: 'report-now' }
    /** Push a fresh settings/credentials backup right now. */
-    | { kind: 'backup-now' };
+    | { kind: 'backup-now' }
+    /** Heal this machine's auto-updater if it stopped, then check for an
+     *  update at once, skipping a failed update's retry wait. */
+    | { kind: 'update-now' };

 export type MasterMessage =
    /** personId is the person's stable uuid; the instance stores it and
@@ -0,0 +1,103 @@
+/**
+ * Keeps this machine's auto-updater (scripts/update-check.mjs) alive.
+ *
+ * PM2 fires the update checker every minute, and nothing on the machine
+ * notices when it stops. On the fleet it stops on its own: PM2's cron dies
+ * inside a live daemon (six Windows machines in October 2026; the mechanism
+ * is in update-check.mjs heal), and a `pm2 stop` or a reboot after an unsaved
+ * process-list change does the same. The master's dashboard notices ("updater
+ * not running"), but then someone has to get onto the machine. The app is a
+ * long-running PM2 process, which PM2 keeps alive reliably, so it checks: once
+ * the last check is older than the master's 10 minutes, it asks the script to
+ * heal, i.e. get PM2 firing it again, and run a check.
+ *
+ * It only revives an updater that has run here before (scripts/.last-check
+ * exists). A checkout that never had one is someone's choice, and
+ * scripts/.no-auto-update switches a checkout off outright (a dev checkout).
+ * Only the checkout's first instance watches — PM2 name `autofirmer`, which
+ * scripts/instances.mjs keeps for instance 1 — since the others share its
+ * updater.
+ *
+ * The master's "Update now" command comes through here as well.
+ */
+import { spawn } from 'node:child_process';
+import { existsSync, readFileSync, statSync } from 'node:fs';
+import path from 'node:path';
+
+const EVERY_MS = 5 * 60_000;
+/** The master's UPDATER_STALE_MS: past this its dashboard says "not running". */
+const STALE_MS = 10 * 60_000;
+/** A heal that didn't take (PM2 missing, say) is retried no sooner than this. */
+const RETRY_MS = 30 * 60_000;
+
+const scriptsDir = () => path.join(process.cwd(), 'scripts');
+const optedOut = () => existsSync(path.join(scriptsDir(), '.no-auto-update'));
+
+// Per process: instrumentation.ts and the route handlers each load this module.
+const g = globalThis as typeof globalThis & { __updaterWatchdog?: { lastHealAt: number } };
+
+/** When the updater last checked (epoch ms), or null if it never has here. */
+function lastCheckAt(): number | null {
+    const file = path.join(scriptsDir(), '.last-check');
+    try {
+        // "<ISO time>  <status>", rewritten on every check.
+        const at = Date.parse(readFileSync(file, 'utf8').split(/\s/, 1)[0]);
+        return Number.isFinite(at) ? at : statSync(file).mtimeMs;
+    } catch {
+        return null;
+    }
+}
+
+/**
+ * Start update-check.mjs on its own. Detached, because what it starts may
+ * restart this very process; the script logs to scripts/update.log itself.
+ */
+function spawnUpdateCheck(args: string[]): number {
+    const child = spawn(process.execPath, [path.join(scriptsDir(), 'update-check.mjs'), ...args], {
+        cwd: process.cwd(),
+        detached: true,
+        // No console window popping up on a Windows trading desktop.
+        windowsHide: true,
+        stdio: 'ignore',
+    });
+    child.on('error', (err) => console.error(`[updater-watchdog] update-check.mjs failed to start: ${err.message}`));
+    child.unref();
+    if (!child.pid) throw new Error('could not start scripts/update-check.mjs');
+    return child.pid;
+}
+
+function tick(): void {
+    const state = g.__updaterWatchdog;
+    if (!state || optedOut()) return;
+    const last = lastCheckAt();
+    if (last === null) return;                       // never ran here: not ours to start
+    const now = Date.now();
+    if (now - last <= STALE_MS || now - state.lastHealAt < RETRY_MS) return;
+    state.lastHealAt = now;
+    try {
+        const pid = spawnUpdateCheck(['--heal']);
+        console.log(`[updater-watchdog] last update check ${Math.round((now - last) / 60_000)} min ago — asked update-check.mjs to heal (pid ${pid})`);
+    } catch (err) {
+        console.error(`[updater-watchdog] ${err instanceof Error ? err.message : String(err)}`);
+    }
+}
+
+export function startUpdaterWatchdog(): void {
+    if (g.__updaterWatchdog) return;
+    // Under PM2 only (pm_id is PM2's): `next dev`, a sandbox copy or a
+    // by-hand `next start` has no updater to keep alive.
+    if (process.env.pm_id === undefined || process.env.name !== 'autofirmer') return;
+    g.__updaterWatchdog = { lastHealAt: 0 };
+    setInterval(tick, EVERY_MS).unref();
+}
+
+/**
+ * The master's "Update now": heal the updater if it stopped, then check at
+ * once, skipping a failed update's retry wait. Answers before the update
+ * runs — it may restart this process — so its outcome reaches the master
+ * with the next report's updater status.
+ */
+export function runUpdateNow(): { started: true; pid: number } {
+    if (optedOut()) throw new Error('updates are switched off on this machine (scripts/.no-auto-update)');
+    return { started: true, pid: spawnUpdateCheck(['--heal', '--now']) };
+}
@@ -33,9 +33,15 @@
 * the build fails, and a failure is retried on a backoff (scripts/.update-failed)
 * rather than every minute.
 *
+ * A machine whose checker stopped firing can't notice by itself, so the app
+ * asks for a heal (lib/updater-watchdog.ts) once the last check goes stale, and
+ * the master's "Update now" asks for a heal plus an immediate check.
+ *
 *   node scripts/update-check.mjs            # check; start a worker if needed
 *   node scripts/update-check.mjs --apply    # update now, here, in the foreground
 *   node scripts/update-check.mjs --dry-run  # report only, change nothing
+ *   node scripts/update-check.mjs --heal     # re-register a checker that stopped, then check
+ *   node scripts/update-check.mjs --now      # check without waiting out a failure's backoff
 */
 import { spawn, spawnSync } from 'node:child_process';
 import {
@@ -65,6 +71,14 @@ const BRANCH = process.env.AUTOFIRMER_BRANCH ?? 'master';
 const INSTANCES = loadInstances();
 const APPLY = process.argv.includes('--apply');
 const DRY_RUN = process.argv.includes('--dry-run');
+const HEAL = process.argv.includes('--heal');
+const NOW = process.argv.includes('--now');
+const OPT_OUT_FILE = path.join(ROOT, 'scripts', '.no-auto-update');
+// The PM2 app install-autostart.* registers, and its schedule.
+const CHECKER = 'autofirmer-update';
+const DEFAULT_CRON = '*/1 * * * *';
+// The master's UPDATER_STALE_MS: past this its dashboard says "updater not running".
+const STALE_CHECK_MS = 10 * 60 * 1000;
 const STALE_LOCK_MS = 2 * 60 * 60 * 1000;
 const LEGACY_STALE_LOCK_MS = 10 * 60 * 1000;
 const LOCK_SETTLE_MS = 1000;
@@ -76,7 +90,10 @@ const REPEAT_RETRY_MS = 60 * 60 * 1000;
 // it. npm and next build are left unbounded: on Windows a timeout would kill
 // only the cmd.exe in front of them, and an orphaned build still writing into
 // .next is worse than a slow one.
-const FETCH_TIMEOUT_MS = 2 * 60 * 1000;
+// The fetch also stays under the cron's minute: a check still running at the next
+// fire has to be killed by PM2, and that kill is where PM2 loses the cron
+// (see heal).
+const FETCH_TIMEOUT_MS = 45 * 1000;
 const PM2_TIMEOUT_MS = 3 * 60 * 1000;
 const LOG_MAX_BYTES = 256 * 1024;
 const LOG_KEEP_LINES = 500;
@@ -120,6 +137,13 @@ function recordCheck(status) {
    } catch { /* never let bookkeeping break an update */ }
 }

+/**
+ * An argument for run() as cmd.exe will see it. On Windows run() goes through
+ * the shell, which joins the arguments with plain spaces, so a cron string or
+ * a path with a space in it would arrive as several.
+ */
+const shellArg = (arg) => (process.platform === 'win32' && /[\s"]/.test(arg) ? `"${arg.replace(/"/g, '""')}"` : arg);
+
 function run(cmd, args, { capture = false, timeout } = {}) {
    return spawnSync(cmd, args, {
        cwd: ROOT,
@@ -370,7 +394,7 @@ function releaseLock() {
 }

 // ── failure record ──────────────────────────────────────────────────────────
-// scripts/.update-failed = { target, step, at, count, restart, clicker }.
+// scripts/.update-failed = { target, step, at, count, restart, clicker, detail? }.
 //
 // Written when an attempt STARTS and deleted when it succeeds, so a worker
 // that dies without reporting (killed, rebooted) still counts as a failed
@@ -393,6 +417,54 @@ function writeFailure(f) {
 function clearFailure() {
    try { rmSync(FAILED_FILE, { force: true }); } catch { /* bookkeeping */ }
 }
+/** Size of update-worker.log right now, so a step's own output can be told apart. */
+function workerLogSize() {
+    try { return statSync(WORKER_LOG_FILE).size; } catch { return 0; }
+}
+
+/** What update-worker.log gained since `from` ('' when a by-hand --apply
+ *  wrote to the terminal instead). */
+function workerLogSince(from) {
+    try {
+        const buf = readFileSync(WORKER_LOG_FILE);
+        return buf.length > from ? buf.subarray(from).toString('utf8') : '';
+    } catch {
+        return '';
+    }
+}
+
+/**
+ * Why a step failed, from its output since `from`: the file this user can't
+ * replace when a permission is what stopped it, else one line of the output.
+ */
+function explainFailure(from) {
+    const text = workerLogSince(from);
+    return (PERMISSION_DENIED.test(text) && ownershipProblem()) || failureReason(text);
+}
+
+/**
+ * One line on why a step failed, from its output, or null.
+ *
+ * The full npm/next output stays on the machine. Without a reason the master
+ * shows only "update failed at build", and a machine failing for a cause of
+ * its own (a Node upgrade leaving better-sqlite3 on the old ABI, a file
+ * Defender holds, a full disk) can't be told from a broken push. The line
+ * rides on .last-check, which every app build and the master already carry.
+ */
+function failureReason(text) {
+    // eslint-disable-next-line no-control-regex
+    const lines = text.replace(/\x1b\[[0-9;]*m/g, '').split(/\r?\n/).map((l) => l.trim()).filter(Boolean);
+    // npm's own epilogue says only that the script failed, and where its log
+    // is; and this script's own log lines (they go to the console too) aren't
+    // the step's output.
+    const boilerplate = /^npm (ERR!|error)( (code|path|command|errno|syscall|signal|workspace|location|A complete log|Lifecycle)\b|\s*$)|^> |^\d{4}-\d\d-\d\dT\S+ {2}/;
+    const first = (re) => lines.find((l) => re.test(l) && !boilerplate.test(l));
+    const line = first(/NODE_MODULE_VERSION|ERR_DLOPEN_FAILED|\bE(PERM|BUSY|ACCES|NOSPC)\b|heap out of memory|Type error:/)
+        ?? first(/\b[Ee]rror\b/)
+        ?? lines.filter((l) => !boilerplate.test(l)).pop();
+    return line ? line.replace(/\s+/g, ' ').slice(0, 100) : null;
+}
+
 /** 15 min after the first failure, an hour after repeated ones: long enough
 *  that a deterministic failure (a broken push) doesn't rebuild every minute
 *  on a trading PC, short enough that a transient one (network, EPERM) heals
@@ -434,19 +506,28 @@ function headMovedAt() {
    }
 }

+/** PM2's process list, or null when PM2 can't be asked. */
+function pm2List() {
+    const r = run('pm2', ['jlist'], { capture: true, timeout: 30_000 });
+    if (r.status !== 0) return null;
+    // jlist prints the process list as one JSON line, possibly after "[PM2] ..." chatter.
+    const line = (r.stdout ?? '').split('\n').find((l) => /^\s*\[\s*[{\]]/.test(l));
+    try {
+        const procs = JSON.parse(line);
+        return Array.isArray(procs) ? procs : null;
+    } catch {
+        return null;
+    }
+}
+
 /**
 * Names of this checkout's PM2 instances that started before `ms`, i.e. still
 * running code from before the build on disk. Empty when PM2 can't be asked:
 * this only sharpens a heuristic and must never block an update.
 */
 function instancesStartedBefore(ms) {
-    const r = run('pm2', ['jlist'], { capture: true, timeout: 30_000 });
-    if (r.status !== 0) return [];
-    // jlist prints the process list as one JSON line, possibly after "[PM2] ..." chatter.
-    const line = (r.stdout ?? '').split('\n').find((l) => /^\s*\[\s*[{\]]/.test(l));
-    let procs;
-    try { procs = JSON.parse(line); } catch { return []; }
-    if (!Array.isArray(procs)) return [];
+    const procs = pm2List();
+    if (!procs) return [];
    const names = new Set(INSTANCES.map((i) => i.name));
    return procs
        .filter((p) => names.has(p?.name) && p.pm2_env?.status === 'online' && Number(p.pm2_env.pm_uptime) < ms)
@@ -550,6 +631,65 @@ function workFor(plan) {
    };
 }

+// ── stuck states ────────────────────────────────────────────────────────────
+// Two states that fail every update until a person gets onto the machine.
+
+/** What a step prints when it may not touch a file. */
+const PERMISSION_DENIED = /permission denied|insufficient permission|\bEACCES\b|\bEPERM\b/i;
+
+/**
+ * A file in the checkout this user doesn't own, as the fix to run, or null.
+ *
+ * One `sudo git pull` or `sudo npm run build` leaves files this user cannot
+ * replace, and from then on every update fails: the build can't clear .next,
+ * and a fetch can't write into .git (Ziad's Pi, 2026-10-02: ~1,800 of them).
+ * The failure itself only says EACCES somewhere in npm's output, so name the
+ * file and the one command that fixes it — the master's updater chip shows
+ * this line. Asked only once a step has failed on a permission: a stray
+ * root-owned file that blocks nothing must not stop an update. POSIX only.
+ */
+function ownershipProblem() {
+    if (typeof process.getuid !== 'function') return null;
+    const r = run('find', [ROOT, '!', '-user', String(process.getuid()), '-print', '-quit'], { capture: true, timeout: 20_000 });
+    const hit = (r.stdout ?? '').split('\n')[0].trim();
+    if (!hit) return null;
+    let owner = 'another user';
+    try {
+        const uid = lstatSync(hit).uid;
+        owner = uid === 0 ? 'root' : `uid ${uid}`;
+    } catch { /* gone since: the generic name will do */ }
+    return `${path.relative(ROOT, hit) || '.'} is owned by ${owner}: run sudo chown -R ${os.userInfo().username} ${ROOT}`;
+}
+
+/** inspect(), with a fetch refused by a .git this user can't write named as such. */
+function inspectNamingOwnership(opts) {
+    try {
+        return inspect(opts);
+    } catch (err) {
+        const why = PERMISSION_DENIED.test(err.message) ? ownershipProblem() : null;
+        throw why ? new Error(why) : err;
+    }
+}
+
+/**
+ * npm install rewrites package-lock.json on some machines (another npm
+ * version, platform-specific optional dependencies), and that edit then
+ * refuses every fast-forward that touches the file. It is generated, so when
+ * it is the only tracked edit, put it back. Any other edit is a person's and
+ * stays, showing as "edited here" on the master.
+ */
+function restoreRewrittenLockfile() {
+    let changed;
+    try {
+        changed = git('diff', '--name-only', 'HEAD').split('\n').filter(Boolean);
+    } catch {
+        return;
+    }
+    if (changed.length !== 1 || changed[0] !== 'package-lock.json') return;
+    git('checkout', 'HEAD', '--', 'package-lock.json');
+    log('package-lock.json had been rewritten here (npm) — put it back before the pull');
+}
+
 // ── build ───────────────────────────────────────────────────────────────────

 /**
@@ -630,6 +770,111 @@ function removePrevBuild() {
    }
 }

+// ── heal ────────────────────────────────────────────────────────────────────
+// The checker runs only while PM2 fires it, and PM2 loses it easily: a
+// `pm2 stop`, a `pm2 delete all`, a reboot after a list change nobody saved.
+
+/** How long ago the last check ran, in ms; null = never on this checkout. */
+function lastCheckAge() {
+    try {
+        const at = Date.parse(readFileSync(LAST_CHECK_FILE, 'utf8').split(/\s/, 1)[0]);
+        return Date.now() - (Number.isFinite(at) ? at : statSync(LAST_CHECK_FILE).mtimeMs);
+    } catch {
+        return null;
+    }
+}
+
+const fmtAge = (ms) => (ms >= 90 * 60 * 1000 ? `${Math.round(ms / 3_600_000)} h` : `${Math.round(ms / 60_000)} min`);
+
+/**
+ * The fields PM2 adds to an app's environment.
+ *
+ * PM2 hands every app the scalar fields of its own record as environment
+ * (name, pm_id, autorestart, …), and `pm2 start` captures the caller's
+ * environment for the new app, then copies it over the new app's record. A
+ * checker registered by a heal, which the dashboard app starts, would take
+ * that app's name and id, and its autorestart would override
+ * --no-autorestart. These are the fields PM2 put on real records, plus the
+ * ones only some records carry. PORT and AUTOFIRMER_DB are the instance's
+ * own, and the checker serves every instance.
+ */
+const INHERITED_FIELDS = new Set([
+    'NODE_APP_INSTANCE', 'automation', 'autorestart', 'autostart', 'created_at', 'cron_restart',
+    'exec_interpreter', 'exec_mode', 'exit_code', 'filter_env', 'instance_var', 'instances',
+    'kill_retry_time', 'km_link', 'merge_logs', 'name', 'namespace', 'node_version', 'pmx',
+    'prev_restart_delay', 'restart_time', 'status', 'stop_exit_codes', 'treekill', 'unique_id',
+    'unstable_restarts', 'username', 'version', 'versioning', 'vizion', 'vizion_running', 'watch',
+    'windowsHide', 'PORT', 'AUTOFIRMER_DB',
+]);
+
+/** Drop what PM2 (and the instance) put in this process's environment, so
+ *  nothing started from here inherits it. */
+function scrubInheritedEnv() {
+    // PM2 also files the app's own config under its name.
+    if (process.env.name) delete process.env[process.env.name];
+    for (const key of Object.keys(process.env)) {
+        if (INHERITED_FIELDS.has(key) || key.startsWith('pm_') || key.startsWith('axm_')) delete process.env[key];
+    }
+}
+
+/**
+ * Get PM2 firing this script again once its checks have stopped. Returns
+ * false when this checkout is opted out of updates altogether.
+ *
+ * What the fleet shows (2026-10, six Windows machines): the cron dies inside
+ * a live PM2 daemon, nobody involved. Every fire runs restartProcessId,
+ * whose first step drops the cron; only a restart that gets as far as
+ * launching the app registers it again. croner sometimes fires twice in
+ * one second, the second restart has to stop a check that is still
+ * launching, and a refused or timed-out stop leaves the checker `stopped`
+ * or `errored` with no cron at all.
+ *
+ * So a checker PM2 still lists is restarted, which registers its cron again
+ * with the schedule and environment it already had, and nothing is deleted:
+ * `pm2 delete` kills the pid an `errored` entry kept, and Windows hands pids
+ * out again quickly, so that could be any program by now. If that pid is
+ * alive, PM2 is left alone and this run checks instead. Only a checker PM2
+ * has lost entirely is registered again, the way install-autostart.* does.
+ */
+function heal() {
+    if (existsSync(OPT_OUT_FILE)) {
+        console.log('scripts/.no-auto-update is here — this checkout is not auto-updated');
+        return false;
+    }
+    // Before anything is started from here, the worker below included: the
+    // app that asked for this heal is a PM2 app, and may be instance 2.
+    scrubInheritedEnv();
+    const age = lastCheckAge();
+    if (age !== null && age <= STALE_CHECK_MS) return true;     // checking on schedule
+    const why = age === null ? 'the update checker had never run here' : `the update checker had stopped (last check ${fmtAge(age)} ago)`;
+
+    const pm2 = (...args) => run('pm2', args.map(shellArg), { capture: true, timeout: PM2_TIMEOUT_MS });
+    const lastLine = (r) => (((r.stderr || r.stdout) ?? '').trim().split('\n').pop() || `exit ${r.status}`);
+    const listed = pm2List()?.find((p) => p?.name === CHECKER);
+    const pid = Number(listed?.pid);
+    if (listed && pid > 0 && pidAlive(pid)) {
+        log(`${why}, but PM2 still has it on pid ${pid}, which is alive — left PM2 alone (a restart would kill that pid); checking from here`);
+        return true;
+    }
+
+    let how;
+    if (listed && pm2('restart', CHECKER).status === 0) {
+        how = `restarted it in PM2 (${listed.pm2_env?.status ?? 'unknown'}), which registers its cron again`;
+    } else {
+        const cron = listed?.pm2_env?.cron_restart || DEFAULT_CRON;
+        if (listed) pm2('delete', CHECKER);                      // its restart failed, and its pid is dead
+        const r = pm2('start', SELF, '--name', CHECKER, '--no-autorestart', '--cron-restart', cron);
+        if (r.status !== 0) {
+            log(`${why} — ERROR: could not register it with PM2: ${lastLine(r)}`);
+            return true;                                         // the check below can still update this machine
+        }
+        how = `registered it with PM2 again (${cron})`;
+    }
+    const saved = pm2('save').status === 0;
+    log(`${why} — ${how}${saved ? '' : ', but pm2 save failed: a reboot would lose it'}`);
+    return true;
+}
+
 // ── check ───────────────────────────────────────────────────────────────────

 /**
@@ -688,7 +933,7 @@ function check() {

    let outcome = null;
    try {
-        const plan = inspect({ adopt: !DRY_RUN });
+        const plan = inspectNamingOwnership({ adopt: !DRY_RUN });
        if (!plan.needsPull && !plan.needsBuild && !plan.needsRestart) {
            outcome = `up to date at ${short(plan.target)}`;    // silent in the log by design
            if (DRY_RUN) console.log(outcome);
@@ -699,9 +944,11 @@ function check() {
            return 0;
        }
        // Silent in update.log too: the failure itself was logged once, and a
-        // line a minute until the retry would bury it.
-        if (plan.pending && Date.now() < retryAt(plan.pending)) {
-            outcome = `waiting to retry ${short(plan.target)}: ${plan.pending.step} (attempt ${plan.pending.count}), next try after ${new Date(retryAt(plan.pending)).toISOString()}`;
+        // line a minute until the retry would bury it. --now (the master's
+        // "Update now") is a person asking for the retry.
+        if (plan.pending && !NOW && Date.now() < retryAt(plan.pending)) {
+            outcome = `waiting to retry ${short(plan.target)}: ${plan.pending.step} (attempt ${plan.pending.count}), next try after ${new Date(retryAt(plan.pending)).toISOString()}`
+                + (plan.pending.detail ? ` — ${plan.pending.detail}` : '');
            return 0;
        }
        const pid = startWorker();
@@ -748,15 +995,15 @@ async function apply() {

    let outcome = 'interrupted';
    let attempt = null;
-    const abort = (step, why) => {
-        log(`ABORTED: ${why}`);
-        outcome = why;
-        if (attempt) writeFailure({ ...attempt, step, at: Date.now() });
+    const abort = (step, why, detail = null) => {
+        log(`ABORTED: ${why}${detail ? ` (${detail})` : ''}`);
+        outcome = detail ? `${why}: ${detail}` : why;
+        if (attempt) writeFailure({ ...attempt, step, at: Date.now(), ...(detail ? { detail } : {}) });
        return 1;
    };

    try {
-        const plan = inspect({ adopt: true });
+        const plan = inspectNamingOwnership({ adopt: true });
        if (!plan.needsPull && !plan.needsBuild && !plan.needsRestart) {
            outcome = `up to date at ${short(plan.target)}`;
            clearFailure();
@@ -780,14 +1027,19 @@ async function apply() {
        else if (plan.needsBuild) log('last build unknown — treating every file as changed');

        // The exact commit the check saw, not whatever `git pull` would fetch now.
-        if (plan.needsPull && run('git', ['merge', '--ff-only', target]).status !== 0) {
-            return abort('pull', `git pull --ff-only to ${short(target)} failed — see scripts/update-worker.log`);
+        if (plan.needsPull) {
+            restoreRewrittenLockfile();
+            const from = workerLogSize();
+            if (run('git', ['merge', '--ff-only', target]).status !== 0) {
+                return abort('pull', `git pull --ff-only to ${short(target)} failed — see scripts/update-worker.log`, explainFailure(from));
+            }
        }

        if (plan.needsBuild) {
            if (work.npmInstall) {
                log(`${because('package-lock.json')} — npm install`);
-                if (run('npm', ['install']).status !== 0) return abort('npm install', 'npm install failed — nothing was built or restarted');
+                const from = workerLogSize();
+                if (run('npm', ['install']).status !== 0) return abort('npm install', 'npm install failed — nothing was built or restarted', explainFailure(from));
            }
            if (work.pythonDeps) {
                log(`${because('clicker/requirements.txt')} — refreshing python deps`);
@@ -796,7 +1048,8 @@ async function apply() {

            // Build BEFORE restarting. A failed build leaves the running process
            // untouched, which is the whole point of doing it in this order.
-            if (!build()) return abort('build', 'build failed — not deployed, nothing was restarted');
+            const from = workerLogSize();
+            if (!build()) return abort('build', 'build failed — not deployed, nothing was restarted', explainFailure(from));

            // .built-commit says what .next holds, so it changes the moment .next
            // does. From here a failed restart must not rebuild, only restart,
@@ -898,6 +1151,7 @@ async function logHealth(instance, base) {
 async function main() {
    mkdirSync(path.join(ROOT, 'scripts'), { recursive: true });
    trimLog();
+    if (HEAL && !DRY_RUN && !heal()) return 0;
    return APPLY && !DRY_RUN ? apply() : check();
 }