Add scrollToLoad and an absent variant of waitFor

scrollToLoad walks a progressively-loading list to the bottom before the steps
that act on its items run. Stopping is two-part: no new matches appeared AND the
container was already pinned to the bottom — counting alone stops early on a slow
fetch. Hitting the scroll cap is reported rather than passed off as done, so a
later step never works quietly on a partial list.

The scrolling element is usually not the window. Lists like this live in a div
with its own overflow, and scrolling the document does nothing at all, so the
step walks up from a matched item to the ancestor that actually scrolls —
overflow allows it and there is more content than fits — with containerSelector
to name one outright when the guess is wrong. Verified against a page whose
document also scrolls, which is the case that tells the two apart: it found the
inner div and pulled 12 items up to 60 in 7 scrolls.

waitFor gains `absent`, for waiting on something to go rather than arrive — a
modal closing after a reset. It only accepts a genuine "selector matched
nothing"; an unreachable extension looks the same from a distance and would
otherwise satisfy the gate for the wrong reason, sending the next iteration into
a page that still has the modal open.

The locate queue carries a free-form options blob now, so a new kind of request
stops meaning a new column each time.

Also fixes a missing comma in the reset flow that broke the build.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

app/api/autobuyer/locate/claim/route.ts

@@ -14,6 +14,7 @@ export async function POST() {
             urlPattern: row.url_pattern,
             openUrl: row.open_url,
             navigateUrl: row.navigate_url,
+            options: (() => { try { return JSON.parse(row.options); } catch { return {}; } })(),
         },
     });
 }

app/api/autobuyer/locate/route.ts

@@ -5,7 +5,7 @@ import { corsJson, corsPreflight } from '../cors';
 /** Python enqueues "find this selector and tell me where it is on screen". */
 export async function POST(req: NextRequest) {
     try {
-        const body = await req.json() as { selector?: unknown; index?: unknown; urlPattern?: unknown; openUrl?: unknown; navigateUrl?: unknown };
+        const body = await req.json() as { selector?: unknown; index?: unknown; urlPattern?: unknown; openUrl?: unknown; navigateUrl?: unknown; options?: unknown };
         if (typeof body.selector !== 'string' || !body.selector.trim()) {
             return corsJson({ error: '`selector` is required' }, { status: 400 });
         }
@@ -13,8 +13,9 @@ export async function POST(req: NextRequest) {
         const urlPattern = typeof body.urlPattern === 'string' ? body.urlPattern : '';
         const openUrl = typeof body.openUrl === 'string' ? body.openUrl : '';
         const navigateUrl = typeof body.navigateUrl === 'string' ? body.navigateUrl : '';
+        const options = body.options && typeof body.options === 'object' ? JSON.stringify(body.options) : '{}';
 
-        const row = createLocateRequest(body.selector.trim(), index, urlPattern, openUrl, navigateUrl);
+        const row = createLocateRequest(body.selector.trim(), index, urlPattern, openUrl, navigateUrl, options);
         return corsJson({ id: row.id, status: row.status });
     } catch (err: any) {
         return corsJson({ error: err?.message ?? 'Failed to queue request' }, { status: 500 });

clicker/clicker.py

@@ -113,7 +113,8 @@ class Dashboard:
         return self._request("/api/autobuyer/status")
 
     def locate(self, selector: str, index: int, url_pattern: str, timeout: float,
-               open_url: str = "", navigate_url: str = "") -> dict:
+               open_url: str = "", navigate_url: str = "",
+               options: dict | None = None) -> dict:
         """Queue a lookup and block until the extension answers it.
 
         `open_url` is the page the extension should open if no tab matches
@@ -123,7 +124,8 @@ class Dashboard:
             "/api/autobuyer/locate",
             "POST",
             {"selector": selector, "index": index, "urlPattern": url_pattern,
-             "openUrl": open_url, "navigateUrl": navigate_url},
+             "openUrl": open_url, "navigateUrl": navigate_url,
+             "options": options or {}},
         )
         request_id = queued["id"]
 

clicker/runner.py

@@ -33,7 +33,7 @@ HEARTBEAT_SECONDS = 2.0
 # Bumped whenever the step vocabulary or the locate protocol changes. Reported in
 # the heartbeat so the dashboard can say "restart your runner" instead of letting
 # a stale process fail on a step type it has never heard of.
-VERSION = "0.14.0"
+VERSION = "0.16.0"
 
 # Shared with the heartbeat thread: whether a run is currently executing.
 _busy = threading.Event()
@@ -75,10 +75,13 @@ def run_one_step(dash: Dashboard, step: dict, opts, rng, report, run_id: int | N
         report(f"tab is on {landed.get('url', target)}")
 
     elif step["action"] == "waitFor":
-        # A gate, not an action: block until the element exists. Nothing is
+        # A gate, not an action: block until the element exists — or, with
-        # clicked or typed. Whatever has to make it appear — a person solving a
+        # `absent`, until it is gone. Nothing is clicked or typed. Whatever has to
-        # challenge, a slow server, a background job — happens outside this run.
+        # change the page — a person solving a challenge, a modal closing itself,
+        # a slow server — happens outside this run.
         selector = step["selector"]
+        absent = bool(step.get("absent"))
+        goal = "disappear" if absent else "appear"
         timeout_s = float(step.get("timeoutSeconds", 120))
         deadline = time.time() + timeout_s
         announced = False
@@ -92,20 +95,54 @@ def run_one_step(dash: Dashboard, step: dict, opts, rng, report, run_id: int | N
                 dash.locate(selector, int(step.get("index", 0)),
                             step.get("urlPattern", "") or opts.url,
                             opts.timeout, step.get("openUrl", "") or "")
-                report(f"{selector} appeared")
+                if not absent:
-                break
+                    report(f"{selector} appeared")
+                    break
+                # Still there. Keep waiting for it to go.
+            except NotFoundError:
+                if absent:
+                    report(f"{selector} is gone")
+                    break
+                # Not there yet; keep waiting for it to arrive.
             except DashboardError:
-                pass          # not there yet, or the tab is mid-render
+                # A dead extension or an unreachable dashboard must not be read
+                # as "the element is gone" — that would satisfy an absent gate
+                # for entirely the wrong reason.
+                pass
 
             if time.time() >= deadline:
                 raise actions.StepError(
-                    f"{selector} did not appear within {timeout_s:.0f}s"
+                    f"{selector} did not {goal} within {timeout_s:.0f}s"
                 )
             if not announced:
-                report(f"waiting for {selector} (up to {timeout_s:.0f}s)")
+                report(f"waiting for {selector} to {goal} (up to {timeout_s:.0f}s)")
                 announced = True
             time.sleep(2.0)
 
+    elif step["action"] == "scrollToLoad":
+        # Lists that load progressively need walking to the bottom before the
+        # steps that act on their items can see everything.
+        result = dash.locate(
+            step["selector"], 0,
+            step.get("urlPattern", "") or opts.url,
+            max(opts.timeout, 120.0),        # scrolling a long list outlasts a normal step
+            step.get("openUrl", "") or "",
+            options={
+                "op": "scrollToLoad",
+                "containerSelector": step.get("containerSelector", ""),
+                "maxScrolls": int(step.get("maxScrolls", 25)),
+                "settleMs": int(step.get("settleMs", 800)),
+            },
+        )
+        found_n = result.get("after", 0)
+        report(f"{found_n} match(es) after {result.get('scrolls', 0)} scroll(s) "
+               f"of {result.get('container', '?')} (was {result.get('before', 0)})")
+        if not result.get("exhausted"):
+            # Stopping on the scroll cap is not a failure, but it does mean the
+            # list may still have more below — worth saying so rather than
+            # letting a later step quietly work on a partial list.
+            report("hit the scroll limit — there may be more not loaded")
+
     elif step["action"] not in ("click", "type"):
         # Almost always a stale runner: the server defines the step vocabulary,
         # so a step type this process has never heard of means automations.ts has

extension/background.js

@@ -263,6 +263,71 @@ function waitForTabLoad(tabId, timeoutMs = 15000) {
     });
 }
 
+/** Runs in the page. Scrolls until the list stops growing, then reports what it
+ *  ended up with.
+ *
+ *  The scrolling element is often NOT the window — lists like this usually live
+ *  in a div with its own overflow, and scrolling the document does nothing at
+ *  all. So walk up from a matched item looking for the ancestor that actually
+ *  scrolls, and let the caller name one outright when the guess is wrong.
+ */
+async function pageScrollToLoad(selector, containerSelector, maxScrolls, settleMs, stableRounds) {
+    const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
+    const count = () => document.querySelectorAll(selector).length;
+
+    function scroller() {
+        if (containerSelector) {
+            const named = document.querySelector(containerSelector);
+            if (!named) return { error: `No element matches container ${containerSelector}` };
+            return { el: named };
+        }
+
+        // An ancestor that can actually scroll: overflow allows it, and there is
+        // more content than fits.
+        let node = document.querySelector(selector)?.parentElement ?? null;
+        while (node && node !== document.body) {
+            const overflow = getComputedStyle(node).overflowY;
+            if (/(auto|scroll)/.test(overflow) && node.scrollHeight > node.clientHeight + 4) {
+                return { el: node };
+            }
+            node = node.parentElement;
+        }
+        return { el: document.scrollingElement || document.documentElement, isDocument: true };
+    }
+
+    const found = scroller();
+    if (found.error) return { error: found.error };
+    const el = found.el;
+
+    const before = count();
+    let last = before;
+    let stable = 0;
+    let scrolls = 0;
+
+    while (scrolls < maxScrolls) {
+        const wasAtBottom = el.scrollTop + el.clientHeight >= el.scrollHeight - 4;
+        el.scrollTop = el.scrollHeight;
+        scrolls++;
+        await sleep(settleMs);
+
+        const now = count();
+        // Nothing new AND we were already pinned to the bottom: the list is done
+        // growing, not merely slow.
+        stable = (now > last) ? 0 : stable + (wasAtBottom ? 1 : 0);
+        last = now;
+        if (stable >= stableRounds) break;
+    }
+
+    return {
+        before,
+        after: last,
+        scrolls,
+        exhausted: stable >= stableRounds,
+        container: found.isDocument ? 'document' : (el.className || el.tagName || 'element').toString().slice(0, 60),
+        url: location.href,
+    };
+}
+
 async function resolveLocateTab(cfg, request) {
     // Every step carries its firm's pattern; the manifest hosts are the fallback
     // for a bare request (the CLI's locate without --url).
@@ -323,6 +388,33 @@ async function serveLocateRequest(cfg) {
         await chrome.tabs.update(tab.id, { active: true });
         await new Promise((r) => setTimeout(r, 250));   // let the OS finish raising it
 
+        // A scroll request is a different operation on the same channel: no
+        // element is measured, the page is just walked to the bottom.
+        if (request.options && request.options.op === 'scrollToLoad') {
+            const [scrolled] = await chrome.scripting.executeScript({
+                target: { tabId: tab.id },
+                func: pageScrollToLoad,
+                args: [
+                    request.selector,
+                    request.options.containerSelector || '',
+                    Number(request.options.maxScrolls) || 25,
+                    Number(request.options.settleMs) || 800,
+                    Number(request.options.stableRounds) || 2,
+                ],
+            });
+            const out = scrolled?.result;
+            if (!out) throw new Error('Scroll injection returned nothing');
+            if (out.error) throw new Error(out.error);
+            result = { ...out, tabId: tab.id, windowId: tab.windowId, browser: detectBrowser() };
+
+            await fetch(`${cfg.apiBase}/api/autobuyer/locate/result`, {
+                method: 'POST',
+                headers: { 'Content-Type': 'application/json' },
+                body: JSON.stringify({ id: request.id, result, error: null }),
+            });
+            return true;
+        }
+
         // `complete` only means the document loaded — a React app still has to
         // mount and paint. Retry briefly rather than declaring the element missing,
         // with a longer budget when we just opened the page from cold.

extension/manifest.json

@@ -1,7 +1,7 @@
 {
     "manifest_version": 3,
     "name": "AutoFirmer Capture",
-    "version": "0.7.0",
+    "version": "0.8.0",
     "description": "Scrapes the HTML of the target tab and posts it to the AutoFirmer dashboard while the AutoBuyer is switched on.",
     "permissions": ["scripting", "tabs", "storage", "alarms"],
     "host_permissions": [

lib/automations.ts

@@ -21,9 +21,24 @@ export type AutomationStep =
     // Point the firm's tab at a page. Omit `url` to use the firm's own. Skipped
     // when the tab is already there, so it doesn't reload and lose page state.
     | { action: 'navigate'; url?: string; label?: string }
-    // Block until `selector` exists, then carry on. Nothing is clicked or typed —
+    // Block until `selector` exists — or, with `absent`, until it is gone from the
-    // this is a gate, for conditions something outside the run has to satisfy.
+    // DOM. Nothing is clicked or typed; this is a gate, for conditions something
-    | { action: 'waitFor'; selector: string; index?: number; timeoutSeconds?: number; label?: string }
+    // outside the run has to satisfy. Note `absent` means removed, not merely
+    // hidden: an element still in the DOM with display:none keeps matching, so
+    // for those use a selector that only matches while it is visible.
+    | { action: 'waitFor'; selector: string; index?: number; absent?: boolean; timeoutSeconds?: number; label?: string }
+    // Scroll until the page stops adding elements matching `selector`, for lists
+    // that load progressively. `containerSelector` names the scrolling element
+    // when the automatic guess is wrong — these lists usually scroll inside a div
+    // rather than the window, and scrolling the document does nothing.
+    | {
+        action: 'scrollToLoad';
+        selector: string;
+        containerSelector?: string;
+        maxScrolls?: number;
+        settleMs?: number;
+        label?: string;
+      }
     // Run `steps` several times over. `times` fixes the count here; `timesFrom`
     // takes it from an input the user fills in on the dashboard. The block is
     // unrolled before the runner ever sees it — see resolveSteps.
@@ -160,7 +175,9 @@ export const FIRMS: Firm[] = [
                         { action: 'waitFor', selector: "div.captcha-solver[data-state='ready']", timeoutSeconds: 30, label: 'Wait for the solver' },
                         { action: 'click', selector: 'div.captcha-solver[data-state="ready"]'},
                         { action: 'waitFor', selector: "div.captcha-solver[data-state='solved']", timeoutSeconds: 120, label: 'Wait for the challenge' },
-                        { action: 'click', selector: 'button.cancelBtn + button.modalActionBtn' }
+                        { action: 'click', selector: 'button.cancelBtn + button.modalActionBtn' },
+
+                        { action: 'waitFor', selector: '.reset_acc_modal', absent: true, timeoutSeconds: 30, label: 'Wait for the modal to close' },
                     ]}, 
                 ]
             }
@@ -265,6 +282,10 @@ function resolveOne(firm: Firm, step: AutomationStep): ResolvedStep {
     if (step.action === 'waitFor') {
         return { ...step, urlPattern: firm.urlPattern, openUrl: firm.url };
     }
+    if (step.action === 'scrollToLoad') {
+        // Acts on nothing, so no signed-out guard — same treatment as waitFor.
+        return { ...step, urlPattern: firm.urlPattern, openUrl: firm.url };
+    }
     if (step.action === 'repeat') {
         // expand() peels these off first; reaching here means a caller bypassed it.
         throw new Error('repeat steps must be expanded, not resolved directly');
@@ -341,7 +362,11 @@ export function describeStep(step: AutomationStep): string {
         case 'type': return `type into ${step.selector}`;
         case 'wait': return `wait ${step.seconds}s`;
         case 'navigate': return `open ${step.url ?? 'the firm page'}`;
-        case 'waitFor': return `wait for ${step.selector}`;
+        case 'waitFor':
+            return step.absent
+                ? `wait for ${step.selector} to disappear`
+                : `wait for ${step.selector}`;
+        case 'scrollToLoad': return `scroll to load all ${step.selector}`;
         case 'repeat': {
             const inner = step.steps.length;
             const count = step.timesFrom ? `{${step.timesFrom}}` : `${step.times ?? 1}`;

lib/db.ts

@@ -394,6 +394,7 @@ export interface LocateRow {
     url_pattern: string;
     open_url: string;
     navigate_url: string;
+    options: string;      // JSON, per-request extras (scroll parameters, ...)
     status: 'pending' | 'claimed' | 'done' | 'error';
     result: string | null;
     error: string | null;
@@ -403,10 +404,10 @@ export interface LocateRow {
 
 const LOCATE_HISTORY = 20;
 
-export function createLocateRequest(selector: string, matchIndex: number, urlPattern: string, openUrl = '', navigateUrl = ''): LocateRow {
+export function createLocateRequest(selector: string, matchIndex: number, urlPattern: string, openUrl = '', navigateUrl = '', options = '{}'): LocateRow {
     const res = db.prepare(
-        'INSERT INTO autobuyer_locate (selector, match_index, url_pattern, open_url, navigate_url, created_at) VALUES (?, ?, ?, ?, ?, ?)'
+        'INSERT INTO autobuyer_locate (selector, match_index, url_pattern, open_url, navigate_url, options, created_at) VALUES (?, ?, ?, ?, ?, ?, ?)'
-    ).run(selector, matchIndex, urlPattern, openUrl, navigateUrl, Date.now());
+    ).run(selector, matchIndex, urlPattern, openUrl, navigateUrl, options, Date.now());
 
     db.prepare(`
         DELETE FROM autobuyer_locate
@@ -459,6 +460,14 @@ db.exec(`
     );
 `);
 
+// Migration: free-form per-request options, so a new kind of request doesn't
+// need a new column each time.
+try {
+    db.exec("ALTER TABLE autobuyer_locate ADD COLUMN options TEXT NOT NULL DEFAULT '{}'");
+} catch {
+    // Column already exists
+}
+
 // Migration: the page to send the tab to before locating.
 try {
     db.exec("ALTER TABLE autobuyer_locate ADD COLUMN navigate_url TEXT NOT NULL DEFAULT ''");
@@ -604,7 +613,7 @@ export const RUNNER_TIMEOUT_MS = 7000;
 /** The runner version this server's step vocabulary requires. A running process
  *  doesn't reload when the source changes, so an older one silently fails on
  *  steps it predates — the dashboard warns instead. */
-export const RUNNER_EXPECTED_VERSION = '0.14.0';
+export const RUNNER_EXPECTED_VERSION = '0.16.0';
 
 export interface RunnerHeartbeat {
     at: number;