Add scrollToLoad and an absent variant of waitFor

scrollToLoad walks a progressively-loading list to the bottom before the steps
that act on its items run. Stopping is two-part: no new matches appeared AND the
container was already pinned to the bottom — counting alone stops early on a slow
fetch. Hitting the scroll cap is reported rather than passed off as done, so a
later step never works quietly on a partial list.

The scrolling element is usually not the window. Lists like this live in a div
with its own overflow, and scrolling the document does nothing at all, so the
step walks up from a matched item to the ancestor that actually scrolls —
overflow allows it and there is more content than fits — with containerSelector
to name one outright when the guess is wrong. Verified against a page whose
document also scrolls, which is the case that tells the two apart: it found the
inner div and pulled 12 items up to 60 in 7 scrolls.

waitFor gains `absent`, for waiting on something to go rather than arrive — a
modal closing after a reset. It only accepts a genuine "selector matched
nothing"; an unreachable extension looks the same from a distance and would
otherwise satisfy the gate for the wrong reason, sending the next iteration into
a page that still has the modal open.

The locate queue carries a free-form options blob now, so a new kind of request
stops meaning a new column each time.

Also fixes a missing comma in the reset flow that broke the build.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

CLAUDE.md

@@ -1,6 +1,6 @@
 # Claude Instructions
 
 ## Working Directory
-Always work directly on `main`. Do **not** create worktrees or feature branches unless explicitly asked.
+Always work directly on `master` — this repo has no `main`. Do **not** create worktrees or feature branches unless explicitly asked.
 
 The project root is `D:\Development\market-dev\autotrader-firms\autotrader`.

app/api/autobuyer/locate/claim/route.ts

@@ -14,6 +14,7 @@ export async function POST() {
             urlPattern: row.url_pattern,
             openUrl: row.open_url,
             navigateUrl: row.navigate_url,
+            options: (() => { try { return JSON.parse(row.options); } catch { return {}; } })(),
         },
     });
 }

app/api/autobuyer/locate/route.ts

@@ -5,7 +5,7 @@ import { corsJson, corsPreflight } from '../cors';
 /** Python enqueues "find this selector and tell me where it is on screen". */
 export async function POST(req: NextRequest) {
     try {
-        const body = await req.json() as { selector?: unknown; index?: unknown; urlPattern?: unknown; openUrl?: unknown; navigateUrl?: unknown };
+        const body = await req.json() as { selector?: unknown; index?: unknown; urlPattern?: unknown; openUrl?: unknown; navigateUrl?: unknown; options?: unknown };
         if (typeof body.selector !== 'string' || !body.selector.trim()) {
             return corsJson({ error: '`selector` is required' }, { status: 400 });
         }
@@ -13,8 +13,9 @@ export async function POST(req: NextRequest) {
         const urlPattern = typeof body.urlPattern === 'string' ? body.urlPattern : '';
         const openUrl = typeof body.openUrl === 'string' ? body.openUrl : '';
         const navigateUrl = typeof body.navigateUrl === 'string' ? body.navigateUrl : '';
+        const options = body.options && typeof body.options === 'object' ? JSON.stringify(body.options) : '{}';
 
-        const row = createLocateRequest(body.selector.trim(), index, urlPattern, openUrl, navigateUrl);
+        const row = createLocateRequest(body.selector.trim(), index, urlPattern, openUrl, navigateUrl, options);
         return corsJson({ id: row.id, status: row.status });
     } catch (err: any) {
         return corsJson({ error: err?.message ?? 'Failed to queue request' }, { status: 500 });

clicker/actions.py

@@ -162,7 +162,9 @@ def perform(
     # A click on a background window is consumed activating it and never reaches
     # the page, so raise the browser immediately before pressing.
     if activate:
-        focused = focus.ensure_frontmost()
+        # Raise the browser the extension actually measured from, not whichever
+        # one happens to be first in the list.
+        focused = focus.ensure_frontmost(browser=found.get("browser"))
         say(focused.detail)
         if not focused.ok:
             raise StepError(

clicker/clicker.py

@@ -21,6 +21,7 @@ Requires the AutoBuyer page switch to be ON — that's the master arming switch.
 
 import argparse
 import json
+import re
 import random
 import sys
 import time
@@ -38,6 +39,16 @@ class DashboardError(RuntimeError):
     pass
 
 
+class TransientError(DashboardError):
+    """A failure that is worth retrying: a 5xx, or the server briefly unreachable.
+
+    Next's dev server intermittently serves a 500 while it recompiles a route —
+    it reads a build manifest mid-write and fails to parse it. That is a blip in
+    the pipeline, not a verdict about the page, and it should not kill a run that
+    is halfway through spending money.
+    """
+
+
 class NotFoundError(DashboardError):
     """The extension reached the page and the element simply isn't there.
 
@@ -53,6 +64,26 @@ class NotFoundError(DashboardError):
 _ABSENT_MARKERS = ("No element matches", "no index")
 
 
+def _summarise_error_body(body: str, limit: int = 200) -> str:
+    """Keep an error readable.
+
+    A dev-server 500 answers with a full HTML page — several kilobytes of script
+    tags and a stack trace — and printing it raw buries the one line that says
+    what went wrong.
+    """
+    try:
+        return str(json.loads(body).get("error", body))[:limit]
+    except json.JSONDecodeError:
+        pass
+
+    if "<!DOCTYPE" in body or "<html" in body:
+        match = re.search(r'"message":"(.*?)"', body)
+        detail = match.group(1) if match else "no detail in the page"
+        return f"server returned an HTML error page ({detail[:limit]})"
+
+    return body[:limit]
+
+
 def _is_absent(message: str) -> bool:
     return any(marker in message for marker in _ABSENT_MARKERS)
 
@@ -72,20 +103,18 @@ class Dashboard:
             with urllib.request.urlopen(req, timeout=self.timeout) as resp:
                 return json.loads(resp.read().decode())
         except urllib.error.HTTPError as exc:
-            body = exc.read().decode(errors="replace")
+            message = _summarise_error_body(exc.read().decode(errors="replace"))
-            try:
+            cls = TransientError if exc.code >= 500 else DashboardError
-                message = json.loads(body).get("error", body)
+            raise cls(f"{method} {path} -> HTTP {exc.code}: {message}") from None
-            except json.JSONDecodeError:
-                message = body
-            raise DashboardError(f"{method} {path} -> HTTP {exc.code}: {message}") from None
         except urllib.error.URLError as exc:
-            raise DashboardError(f"Cannot reach {self.base} — {exc.reason}") from None
+            raise TransientError(f"Cannot reach {self.base} — {exc.reason}") from None
 
     def status(self) -> dict:
         return self._request("/api/autobuyer/status")
 
     def locate(self, selector: str, index: int, url_pattern: str, timeout: float,
-               open_url: str = "", navigate_url: str = "") -> dict:
+               open_url: str = "", navigate_url: str = "",
+               options: dict | None = None) -> dict:
         """Queue a lookup and block until the extension answers it.
 
         `open_url` is the page the extension should open if no tab matches
@@ -95,13 +124,23 @@ class Dashboard:
             "/api/autobuyer/locate",
             "POST",
             {"selector": selector, "index": index, "urlPattern": url_pattern,
-             "openUrl": open_url, "navigateUrl": navigate_url},
+             "openUrl": open_url, "navigateUrl": navigate_url,
+             "options": options or {}},
         )
         request_id = queued["id"]
 
         deadline = time.monotonic() + timeout
+        last_transient = None
         while time.monotonic() < deadline:
+            try:
                 row = self._request(f"/api/autobuyer/locate?id={request_id}")
+            except TransientError as exc:
+                # The extension may well answer while the server is having a
+                # moment; keep polling rather than failing the run over a blip.
+                last_transient = exc
+                time.sleep(POLL_INTERVAL)
+                continue
+
             if row["status"] == "done":
                 return row["result"]
             if row["status"] == "error":
@@ -110,6 +149,11 @@ class Dashboard:
                 raise cls(f"Extension could not locate it: {detail}")
             time.sleep(POLL_INTERVAL)
 
+        if last_transient is not None:
+            raise DashboardError(
+                f"No answer within {timeout:g}s, and the dashboard kept erroring "
+                f"({last_transient})"
+            )
         raise DashboardError(
             f"No answer within {timeout:g}s. Is the extension installed, is Chrome "
             f"running, and is the AutoBuyer switch ON?"

clicker/diagnose.py

@@ -0,0 +1,138 @@
+#!/usr/bin/env python3
+"""Check whether this machine can actually be driven.
+
+Run it on the box that will do the clicking, before trusting a run:
+
+    python diagnose.py
+
+It answers the question a remote desktop makes hard — is the mouse really moving,
+or is the viewer just not drawing it? The cursor position is read back from the
+OS after each move, so the answer doesn't depend on anything being rendered.
+
+Nothing is clicked. The cursor is moved and put back where it started.
+"""
+
+import sys
+import time
+
+
+def line(label: str, value: str) -> None:
+    print(f"  {label:<22} {value}")
+
+
+def check_platform() -> None:
+    print("\nPlatform")
+    line("os", sys.platform)
+    line("python", sys.version.split()[0])
+
+    if sys.platform != "win32":
+        return
+
+    import ctypes
+
+    # SM_REMOTESESSION: non-zero when this process is running inside an RDP
+    # session rather than at the physical console.
+    remote = ctypes.windll.user32.GetSystemMetrics(0x1000)
+    line("remote session", "yes — RDP/terminal services" if remote else "no — physical console")
+    if remote:
+        print("     Input injection still works over RDP, but the session's desktop")
+        print("     is locked when you disconnect, and clicks go nowhere until you")
+        print("     reconnect. Keep the window open for the duration of a run.")
+
+
+def check_dpi() -> None:
+    print("\nDisplay")
+    try:
+        import focus
+    except ImportError:
+        line("dpi awareness", "focus.py not importable — run this from clicker/")
+        return
+
+    result = focus.enable_dpi_awareness()
+    line("dpi awareness", result or "n/a (not Windows)")
+
+    try:
+        import pyautogui
+    except ImportError:
+        line("screen size", "pyautogui not installed")
+        return
+    size = pyautogui.size()
+    line("screen size", f"{size.width}x{size.height} (as the OS reports it)")
+    print("     Compare against what `clicker.py locate` reports for screenSize.")
+    print("     A mismatch that isn't a clean scaling factor means clicks land off.")
+
+
+def check_mouse() -> bool:
+    """Move the cursor and read it back. Returns True if the OS agreed."""
+    print("\nMouse")
+    try:
+        import pyautogui
+    except ImportError:
+        line("result", "pyautogui not installed — run: pip install -r requirements.txt")
+        return False
+
+    pyautogui.FAILSAFE = False        # a deliberate corner move would abort us
+    start = pyautogui.position()
+    line("start position", f"{start[0]},{start[1]}")
+
+    width, height = pyautogui.size()
+    targets = [(width // 4, height // 4), (width // 2, height // 2)]
+
+    agreed = True
+    for x, y in targets:
+        pyautogui.moveTo(x, y, duration=0.3)
+        time.sleep(0.1)
+        got = pyautogui.position()
+        ok = abs(got[0] - x) <= 2 and abs(got[1] - y) <= 2
+        agreed &= ok
+        line("moved to", f"{x},{y}  ->  OS reports {got[0]},{got[1]}  {'OK' if ok else 'MISMATCH'}")
+
+    pyautogui.moveTo(start[0], start[1], duration=0.2)
+
+    print()
+    if agreed:
+        print("  The OS moved the cursor to every requested point.")
+        print("  If you saw nothing move, that is your viewer not drawing it —")
+        print("  the clicks are landing where they should.")
+    else:
+        print("  The cursor did NOT land where it was asked to.")
+        print("  On Windows this is usually display scaling: the process is being")
+        print("  fed virtualised coordinates. Check the dpi awareness line above,")
+        print("  and pass --scale to clicker.py to compensate.")
+    return agreed
+
+
+def check_foreground() -> None:
+    print("\nForeground window")
+    try:
+        import focus
+    except ImportError:
+        line("frontmost", "focus.py not importable")
+        return
+
+    front = focus.frontmost()
+    line("frontmost", front or "could not determine on this platform")
+    line("browsers known", ", ".join(focus.browser_ids()) or "none for this platform")
+
+    result = focus.ensure_frontmost()
+    line("raise browser", f"{'OK' if result.ok else 'FAILED'} — {result.detail}")
+    print("     With no browser named, any known one counts — that is this check")
+    print("     only. A real run raises the browser the extension reported, so if")
+    print("     this raised one you do not automate, that is not a fault.")
+    if not result.ok:
+        print("     Every click is refused while this fails: a click on an")
+        print("     unfocused window is consumed activating it.")
+
+
+def main() -> int:
+    print("AutoFirmer clicker diagnostics")
+    check_platform()
+    check_dpi()
+    ok = check_mouse()
+    check_foreground()
+    print()
+    return 0 if ok else 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())

clicker/focus.py

@@ -16,20 +16,59 @@ import subprocess
 import sys
 import time
 
-# Chrome ships under several bundle ids; accept whichever is installed.
+# Per browser, how to recognise it on each platform. The extension reports which
-MAC_BUNDLES = (
+# one is hosting it, because picking by list order raises the wrong browser as
-    "com.google.Chrome",
+# soon as two are installed — and then clicks land in a window the coordinates
-    "com.google.Chrome.beta",
+# were never measured from.
-    "com.google.Chrome.dev",
+BROWSERS = {
-    "com.google.Chrome.canary",
+    "chrome": {
-    "com.brave.Browser",
+        "darwin": ("com.google.Chrome", "com.google.Chrome.beta",
-    "com.microsoft.edgemac",
+                   "com.google.Chrome.dev", "com.google.Chrome.canary",
-)
+                   "org.chromium.Chromium"),
+        "win32": ("chrome.exe",),
+        "titles": ("Chrome", "Chromium"),
+    },
+    "edge": {
+        "darwin": ("com.microsoft.edgemac",),
+        "win32": ("msedge.exe",),
+        "titles": ("Edge",),
+    },
+    "brave": {
+        "darwin": ("com.brave.Browser",),
+        "win32": ("brave.exe",),
+        "titles": ("Brave",),
+    },
+    "opera": {
+        "darwin": ("com.operasoftware.Opera",),
+        "win32": ("opera.exe", "launcher.exe"),
+        "titles": ("Opera",),
+    },
+    "vivaldi": {
+        "darwin": ("com.vivaldi.Vivaldi",),
+        "win32": ("vivaldi.exe",),
+        "titles": ("Vivaldi",),
+    },
+}
 
-# Windows browsers, matched on the owning process. A title match would also hit
+
-# an editor with chrome.js open or a folder named Chrome; a process name cannot
+def _ids_for(browser: str | None) -> tuple[str, ...]:
-# collide that way.
+    """Identifiers to accept on this platform. Without a named browser, every
-WINDOWS_PROCESSES = ("chrome.exe", "msedge.exe", "brave.exe")
+    known one — the old behaviour, and still right on a single-browser box."""
+    key = "darwin" if sys.platform == "darwin" else "win32"
+    if browser and browser in BROWSERS:
+        return BROWSERS[browser][key]
+    return tuple(i for b in BROWSERS.values() for i in b[key])
+
+
+def _titles_for(browser: str | None) -> tuple[str, ...]:
+    if browser and browser in BROWSERS:
+        return BROWSERS[browser]["titles"]
+    return tuple(t for b in BROWSERS.values() for t in b["titles"])
+
+
+# Kept for callers that just want "any known browser".
+MAC_BUNDLES = _ids_for(None) if sys.platform == "darwin" else BROWSERS["chrome"]["darwin"]
+WINDOWS_PROCESSES = BROWSERS["chrome"]["win32"] + BROWSERS["edge"]["win32"] + BROWSERS["brave"]["win32"]
 
 SETTLE = 0.20   # let the window manager finish raising before measuring or clicking
 
@@ -103,12 +142,10 @@ def _mac_workspace():
     return NSWorkspace.sharedWorkspace()
 
 
-def browser_ids() -> tuple[str, ...]:
+def browser_ids(browser: str | None = None) -> tuple[str, ...]:
-    """What counts as "the browser" on this platform."""
+    """What counts as "the browser" on this platform, optionally narrowed to one."""
-    if sys.platform == "darwin":
+    if sys.platform in ("darwin", "win32"):
-        return MAC_BUNDLES
+        return _ids_for(browser)
-    if sys.platform == "win32":
-        return WINDOWS_PROCESSES
     return ()
 
 
@@ -138,7 +175,7 @@ def frontmost() -> str | None:
     return None
 
 
-def _mac_activate() -> FocusResult:
+def _mac_activate(browser: str | None = None) -> FocusResult:
     ws = _mac_workspace()
     if ws is None:
         # pyobjc's AppKit isn't present. osascript works but may prompt for
@@ -153,7 +190,7 @@ def _mac_activate() -> FocusResult:
             return FocusResult(False, f"could not activate Chrome ({exc})")
 
     running = {a.bundleIdentifier(): a for a in ws.runningApplications()}
-    for bundle in MAC_BUNDLES:
+    for bundle in _ids_for(browser):
         app = running.get(bundle)
         if app is None:
             continue
@@ -162,10 +199,11 @@ def _mac_activate() -> FocusResult:
         app.activateWithOptions_(1 << 1)
         return FocusResult(True, f"activated {bundle}")
 
-    return FocusResult(False, "no Chrome-family browser is running")
+    wanted = browser or "any known browser"
+    return FocusResult(False, f"{wanted} is not running")
 
 
-def _is_browser_window(win) -> bool:
+def _is_browser_window(win, browser: str | None = None) -> bool:
     """Match on the owning process where we can, title only as a fallback.
 
     A title match alone catches an editor with chrome.js open, or a folder window
@@ -175,13 +213,13 @@ def _is_browser_window(win) -> bool:
         try:
             name = _win_process_name(win._hWnd)
             if name:
-                return name in WINDOWS_PROCESSES
+                return name in _ids_for(browser)
         except Exception:
             pass          # fall through to the title check
-    return bool(win.title) and "Chrome" in win.title
+    return bool(win.title) and any(t in win.title for t in _titles_for(browser))
 
 
-def _other_activate() -> FocusResult:
+def _other_activate(browser: str | None = None) -> FocusResult:
     """Windows (and any platform pygetwindow supports)."""
     try:
         import pygetwindow
@@ -189,14 +227,15 @@ def _other_activate() -> FocusResult:
         return FocusResult(False, "pygetwindow unavailable — cannot raise the browser")
 
     try:
-        wins = [w for w in pygetwindow.getAllWindows() if w.visible and _is_browser_window(w)]
+        wins = [w for w in pygetwindow.getAllWindows()
+                if w.visible and _is_browser_window(w, browser)]
     except NotImplementedError:
         # pygetwindow has no X11 backend; say so rather than looking like no
         # browser is open.
         return FocusResult(False, f"window management is unsupported on {sys.platform}")
 
     if not wins:
-        return FocusResult(False, "no browser window found")
+        return FocusResult(False, f"no {browser or 'browser'} window found")
 
     try:
         win = wins[0]
@@ -211,25 +250,30 @@ def _other_activate() -> FocusResult:
         return FocusResult(False, f"could not activate window ({exc})")
 
 
-def activate_browser() -> FocusResult:
+def activate_browser(browser: str | None = None) -> FocusResult:
-    """Raise the browser application above everything else."""
+    """Raise the browser application above everything else.
-    result = _mac_activate() if sys.platform == "darwin" else _other_activate()
+
+    `browser` is the id the extension reported ("chrome", "edge", ...). Without
+    it, any known browser will do — fine on a machine with one installed, wrong
+    on a machine with two.
+    """
+    result = _mac_activate(browser) if sys.platform == "darwin" else _other_activate(browser)
     if result.ok:
         time.sleep(SETTLE)
     return result
 
 
-def ensure_frontmost(timeout: float = 1.5) -> FocusResult:
+def ensure_frontmost(timeout: float = 1.5, browser: str | None = None) -> FocusResult:
     """Raise the browser and, where we can check, confirm it actually came forward.
 
     Returning ok=False does not mean the click will fail — only that we could not
     verify. The caller decides whether to proceed.
     """
-    result = activate_browser()
+    result = activate_browser(browser)
     if not result.ok:
         return result
 
-    ids = browser_ids()
+    ids = browser_ids(browser)
     if not ids:
         return FocusResult(True, result.detail + " (unverified)")
 

clicker/runner.py

@@ -24,6 +24,7 @@ import threading
 import time
 
 import actions
+import focus
 from clicker import Dashboard, DashboardError, NotFoundError
 
 POLL_SECONDS = 1.0
@@ -32,7 +33,7 @@ HEARTBEAT_SECONDS = 2.0
 # Bumped whenever the step vocabulary or the locate protocol changes. Reported in
 # the heartbeat so the dashboard can say "restart your runner" instead of letting
 # a stale process fail on a step type it has never heard of.
-VERSION = "0.12.0"
+VERSION = "0.16.0"
 
 # Shared with the heartbeat thread: whether a run is currently executing.
 _busy = threading.Event()
@@ -74,10 +75,13 @@ def run_one_step(dash: Dashboard, step: dict, opts, rng, report, run_id: int | N
         report(f"tab is on {landed.get('url', target)}")
 
     elif step["action"] == "waitFor":
-        # A gate, not an action: block until the element exists. Nothing is
+        # A gate, not an action: block until the element exists — or, with
-        # clicked or typed. Whatever has to make it appear — a person solving a
+        # `absent`, until it is gone. Nothing is clicked or typed. Whatever has to
-        # challenge, a slow server, a background job — happens outside this run.
+        # change the page — a person solving a challenge, a modal closing itself,
+        # a slow server — happens outside this run.
         selector = step["selector"]
+        absent = bool(step.get("absent"))
+        goal = "disappear" if absent else "appear"
         timeout_s = float(step.get("timeoutSeconds", 120))
         deadline = time.time() + timeout_s
         announced = False
@@ -91,20 +95,54 @@ def run_one_step(dash: Dashboard, step: dict, opts, rng, report, run_id: int | N
                 dash.locate(selector, int(step.get("index", 0)),
                             step.get("urlPattern", "") or opts.url,
                             opts.timeout, step.get("openUrl", "") or "")
+                if not absent:
                     report(f"{selector} appeared")
                     break
+                # Still there. Keep waiting for it to go.
+            except NotFoundError:
+                if absent:
+                    report(f"{selector} is gone")
+                    break
+                # Not there yet; keep waiting for it to arrive.
             except DashboardError:
-                pass          # not there yet, or the tab is mid-render
+                # A dead extension or an unreachable dashboard must not be read
+                # as "the element is gone" — that would satisfy an absent gate
+                # for entirely the wrong reason.
+                pass
 
             if time.time() >= deadline:
                 raise actions.StepError(
-                    f"{selector} did not appear within {timeout_s:.0f}s"
+                    f"{selector} did not {goal} within {timeout_s:.0f}s"
                 )
             if not announced:
-                report(f"waiting for {selector} (up to {timeout_s:.0f}s)")
+                report(f"waiting for {selector} to {goal} (up to {timeout_s:.0f}s)")
                 announced = True
             time.sleep(2.0)
 
+    elif step["action"] == "scrollToLoad":
+        # Lists that load progressively need walking to the bottom before the
+        # steps that act on their items can see everything.
+        result = dash.locate(
+            step["selector"], 0,
+            step.get("urlPattern", "") or opts.url,
+            max(opts.timeout, 120.0),        # scrolling a long list outlasts a normal step
+            step.get("openUrl", "") or "",
+            options={
+                "op": "scrollToLoad",
+                "containerSelector": step.get("containerSelector", ""),
+                "maxScrolls": int(step.get("maxScrolls", 25)),
+                "settleMs": int(step.get("settleMs", 800)),
+            },
+        )
+        found_n = result.get("after", 0)
+        report(f"{found_n} match(es) after {result.get('scrolls', 0)} scroll(s) "
+               f"of {result.get('container', '?')} (was {result.get('before', 0)})")
+        if not result.get("exhausted"):
+            # Stopping on the scroll cap is not a failure, but it does mean the
+            # list may still have more below — worth saying so rather than
+            # letting a later step quietly work on a partial list.
+            report("hit the scroll limit — there may be more not loaded")
+
     elif step["action"] not in ("click", "type"):
         # Almost always a stale runner: the server defines the step vocabulary,
         # so a step type this process has never heard of means automations.ts has

extension/background.js

@@ -154,6 +154,24 @@ async function captureAndSend(cfg) {
     });
 }
 
+/** Which browser is hosting this extension.
+ *
+ *  The clicker has to raise *this* browser before clicking, and picking by list
+ *  order gets it wrong the moment two are installed — it would raise Edge while
+ *  the coordinates came from a tab in Chrome, landing every click in the wrong
+ *  window. So the answer travels with the measurement.
+ */
+function detectBrowser() {
+    const ua = navigator.userAgent || '';
+    if (/\bEdg\//.test(ua)) return 'edge';
+    if (/\bOPR\//.test(ua)) return 'opera';
+    if (/\bVivaldi\//.test(ua)) return 'vivaldi';
+    try {
+        if (navigator.brave) return 'brave';   // Brave otherwise reports as Chrome
+    } catch { /* not Brave */ }
+    return 'chrome';
+}
+
 // ── Locate: turn a CSS selector into desktop coordinates ────────────────────
 
 /** Runs in the page. Scrolls the element into view, then reports where it ended up. */
@@ -245,6 +263,71 @@ function waitForTabLoad(tabId, timeoutMs = 15000) {
     });
 }
 
+/** Runs in the page. Scrolls until the list stops growing, then reports what it
+ *  ended up with.
+ *
+ *  The scrolling element is often NOT the window — lists like this usually live
+ *  in a div with its own overflow, and scrolling the document does nothing at
+ *  all. So walk up from a matched item looking for the ancestor that actually
+ *  scrolls, and let the caller name one outright when the guess is wrong.
+ */
+async function pageScrollToLoad(selector, containerSelector, maxScrolls, settleMs, stableRounds) {
+    const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
+    const count = () => document.querySelectorAll(selector).length;
+
+    function scroller() {
+        if (containerSelector) {
+            const named = document.querySelector(containerSelector);
+            if (!named) return { error: `No element matches container ${containerSelector}` };
+            return { el: named };
+        }
+
+        // An ancestor that can actually scroll: overflow allows it, and there is
+        // more content than fits.
+        let node = document.querySelector(selector)?.parentElement ?? null;
+        while (node && node !== document.body) {
+            const overflow = getComputedStyle(node).overflowY;
+            if (/(auto|scroll)/.test(overflow) && node.scrollHeight > node.clientHeight + 4) {
+                return { el: node };
+            }
+            node = node.parentElement;
+        }
+        return { el: document.scrollingElement || document.documentElement, isDocument: true };
+    }
+
+    const found = scroller();
+    if (found.error) return { error: found.error };
+    const el = found.el;
+
+    const before = count();
+    let last = before;
+    let stable = 0;
+    let scrolls = 0;
+
+    while (scrolls < maxScrolls) {
+        const wasAtBottom = el.scrollTop + el.clientHeight >= el.scrollHeight - 4;
+        el.scrollTop = el.scrollHeight;
+        scrolls++;
+        await sleep(settleMs);
+
+        const now = count();
+        // Nothing new AND we were already pinned to the bottom: the list is done
+        // growing, not merely slow.
+        stable = (now > last) ? 0 : stable + (wasAtBottom ? 1 : 0);
+        last = now;
+        if (stable >= stableRounds) break;
+    }
+
+    return {
+        before,
+        after: last,
+        scrolls,
+        exhausted: stable >= stableRounds,
+        container: found.isDocument ? 'document' : (el.className || el.tagName || 'element').toString().slice(0, 60),
+        url: location.href,
+    };
+}
+
 async function resolveLocateTab(cfg, request) {
     // Every step carries its firm's pattern; the manifest hosts are the fallback
     // for a bare request (the CLI's locate without --url).
@@ -305,6 +388,33 @@ async function serveLocateRequest(cfg) {
         await chrome.tabs.update(tab.id, { active: true });
         await new Promise((r) => setTimeout(r, 250));   // let the OS finish raising it
 
+        // A scroll request is a different operation on the same channel: no
+        // element is measured, the page is just walked to the bottom.
+        if (request.options && request.options.op === 'scrollToLoad') {
+            const [scrolled] = await chrome.scripting.executeScript({
+                target: { tabId: tab.id },
+                func: pageScrollToLoad,
+                args: [
+                    request.selector,
+                    request.options.containerSelector || '',
+                    Number(request.options.maxScrolls) || 25,
+                    Number(request.options.settleMs) || 800,
+                    Number(request.options.stableRounds) || 2,
+                ],
+            });
+            const out = scrolled?.result;
+            if (!out) throw new Error('Scroll injection returned nothing');
+            if (out.error) throw new Error(out.error);
+            result = { ...out, tabId: tab.id, windowId: tab.windowId, browser: detectBrowser() };
+
+            await fetch(`${cfg.apiBase}/api/autobuyer/locate/result`, {
+                method: 'POST',
+                headers: { 'Content-Type': 'application/json' },
+                body: JSON.stringify({ id: request.id, result, error: null }),
+            });
+            return true;
+        }
+
         // `complete` only means the document loaded — a React app still has to
         // mount and paint. Retry briefly rather than declaring the element missing,
         // with a longer budget when we just opened the page from cold.
@@ -322,7 +432,13 @@ async function serveLocateRequest(cfg) {
             if (Date.now() >= deadline) throw new Error(out.error);
             await new Promise((r) => setTimeout(r, 350));
         }
-        result = { ...out, tabId: tab.id, windowId: tab.windowId, openedTab: opened };
+        result = {
+            ...out,
+            tabId: tab.id,
+            windowId: tab.windowId,
+            openedTab: opened,
+            browser: detectBrowser(),
+        };
     } catch (err) {
         error = err.message;
     }

extension/manifest.json

@@ -1,7 +1,7 @@
 {
     "manifest_version": 3,
     "name": "AutoFirmer Capture",
-    "version": "0.6.0",
+    "version": "0.8.0",
     "description": "Scrapes the HTML of the target tab and posts it to the AutoFirmer dashboard while the AutoBuyer is switched on.",
     "permissions": ["scripting", "tabs", "storage", "alarms"],
     "host_permissions": [

lib/automations.ts

@@ -21,9 +21,24 @@ export type AutomationStep =
     // Point the firm's tab at a page. Omit `url` to use the firm's own. Skipped
     // when the tab is already there, so it doesn't reload and lose page state.
     | { action: 'navigate'; url?: string; label?: string }
-    // Block until `selector` exists, then carry on. Nothing is clicked or typed —
+    // Block until `selector` exists — or, with `absent`, until it is gone from the
-    // this is a gate, for conditions something outside the run has to satisfy.
+    // DOM. Nothing is clicked or typed; this is a gate, for conditions something
-    | { action: 'waitFor'; selector: string; index?: number; timeoutSeconds?: number; label?: string }
+    // outside the run has to satisfy. Note `absent` means removed, not merely
+    // hidden: an element still in the DOM with display:none keeps matching, so
+    // for those use a selector that only matches while it is visible.
+    | { action: 'waitFor'; selector: string; index?: number; absent?: boolean; timeoutSeconds?: number; label?: string }
+    // Scroll until the page stops adding elements matching `selector`, for lists
+    // that load progressively. `containerSelector` names the scrolling element
+    // when the automatic guess is wrong — these lists usually scroll inside a div
+    // rather than the window, and scrolling the document does nothing.
+    | {
+        action: 'scrollToLoad';
+        selector: string;
+        containerSelector?: string;
+        maxScrolls?: number;
+        settleMs?: number;
+        label?: string;
+      }
     // Run `steps` several times over. `times` fixes the count here; `timesFrom`
     // takes it from an input the user fills in on the dashboard. The block is
     // unrolled before the runner ever sees it — see resolveSteps.
@@ -156,11 +171,13 @@ export const FIRMS: Firm[] = [
 
                     { action: 'repeat', timesFrom: 'count', steps: [
                         // one purchase — the steps you already have
-                        { action: 'click', selector: 'button.reset_btn', label: 'Reset Account' },
+                        { action: 'click', selector: '.status-failed button.reset_btn', label: 'Reset Account' },
                         { action: 'waitFor', selector: "div.captcha-solver[data-state='ready']", timeoutSeconds: 30, label: 'Wait for the solver' },
                         { action: 'click', selector: 'div.captcha-solver[data-state="ready"]'},
                         { action: 'waitFor', selector: "div.captcha-solver[data-state='solved']", timeoutSeconds: 120, label: 'Wait for the challenge' },
-                        { action: 'click', selector: 'button.cancelBtn + button.modalActionBtn' }
+                        { action: 'click', selector: 'button.cancelBtn + button.modalActionBtn' },
+
+                        { action: 'waitFor', selector: '.reset_acc_modal', absent: true, timeoutSeconds: 30, label: 'Wait for the modal to close' },
                     ]}, 
                 ]
             }
@@ -265,6 +282,10 @@ function resolveOne(firm: Firm, step: AutomationStep): ResolvedStep {
     if (step.action === 'waitFor') {
         return { ...step, urlPattern: firm.urlPattern, openUrl: firm.url };
     }
+    if (step.action === 'scrollToLoad') {
+        // Acts on nothing, so no signed-out guard — same treatment as waitFor.
+        return { ...step, urlPattern: firm.urlPattern, openUrl: firm.url };
+    }
     if (step.action === 'repeat') {
         // expand() peels these off first; reaching here means a caller bypassed it.
         throw new Error('repeat steps must be expanded, not resolved directly');
@@ -341,7 +362,11 @@ export function describeStep(step: AutomationStep): string {
         case 'type': return `type into ${step.selector}`;
         case 'wait': return `wait ${step.seconds}s`;
         case 'navigate': return `open ${step.url ?? 'the firm page'}`;
-        case 'waitFor': return `wait for ${step.selector}`;
+        case 'waitFor':
+            return step.absent
+                ? `wait for ${step.selector} to disappear`
+                : `wait for ${step.selector}`;
+        case 'scrollToLoad': return `scroll to load all ${step.selector}`;
         case 'repeat': {
             const inner = step.steps.length;
             const count = step.timesFrom ? `{${step.timesFrom}}` : `${step.times ?? 1}`;

lib/db.ts

@@ -394,6 +394,7 @@ export interface LocateRow {
     url_pattern: string;
     open_url: string;
     navigate_url: string;
+    options: string;      // JSON, per-request extras (scroll parameters, ...)
     status: 'pending' | 'claimed' | 'done' | 'error';
     result: string | null;
     error: string | null;
@@ -403,10 +404,10 @@ export interface LocateRow {
 
 const LOCATE_HISTORY = 20;
 
-export function createLocateRequest(selector: string, matchIndex: number, urlPattern: string, openUrl = '', navigateUrl = ''): LocateRow {
+export function createLocateRequest(selector: string, matchIndex: number, urlPattern: string, openUrl = '', navigateUrl = '', options = '{}'): LocateRow {
     const res = db.prepare(
-        'INSERT INTO autobuyer_locate (selector, match_index, url_pattern, open_url, navigate_url, created_at) VALUES (?, ?, ?, ?, ?, ?)'
+        'INSERT INTO autobuyer_locate (selector, match_index, url_pattern, open_url, navigate_url, options, created_at) VALUES (?, ?, ?, ?, ?, ?, ?)'
-    ).run(selector, matchIndex, urlPattern, openUrl, navigateUrl, Date.now());
+    ).run(selector, matchIndex, urlPattern, openUrl, navigateUrl, options, Date.now());
 
     db.prepare(`
         DELETE FROM autobuyer_locate
@@ -459,6 +460,14 @@ db.exec(`
     );
 `);
 
+// Migration: free-form per-request options, so a new kind of request doesn't
+// need a new column each time.
+try {
+    db.exec("ALTER TABLE autobuyer_locate ADD COLUMN options TEXT NOT NULL DEFAULT '{}'");
+} catch {
+    // Column already exists
+}
+
 // Migration: the page to send the tab to before locating.
 try {
     db.exec("ALTER TABLE autobuyer_locate ADD COLUMN navigate_url TEXT NOT NULL DEFAULT ''");
@@ -604,7 +613,7 @@ export const RUNNER_TIMEOUT_MS = 7000;
 /** The runner version this server's step vocabulary requires. A running process
  *  doesn't reload when the source changes, so an older one silently fails on
  *  steps it predates — the dashboard warns instead. */
-export const RUNNER_EXPECTED_VERSION = '0.12.0';
+export const RUNNER_EXPECTED_VERSION = '0.16.0';
 
 export interface RunnerHeartbeat {
     at: number;