Add autobuyer page capture, browser extension, and desktop clicker

Builds the pipeline the autobuyer needs: see the page, find an element,
click it.

extension/ — MV3 Chromium extension. Polls /api/autobuyer/status and,
while on, scrapes the target tab's HTML and posts it back. Also serves
locate requests: focuses the window, scrolls the element into view, and
reports its position. host_permissions is scoped to tradeify plus
localhost so it cannot read other sites — an empty target pattern would
otherwise capture whatever tab happened to be active, including banking
or mail.

app/api/autobuyer/ — status toggle, capture store, and the locate request
queue. CORS is open because the extension's origin changes every time an
unpacked extension is reloaded.

app/autobuyer/page.tsx — ON switch, source view (default) and a rendered
view. The render uses sandbox="allow-scripts" without allow-same-origin:
the page's own JS is needed because sites ship content at opacity:0 and
fade it in, but the frame must not reach the dashboard's same-origin API
routes, which serve firm credentials.

clicker/ — Python CLI. Asks the extension where a selector is, adds the
element rect to the window's screen position and the browser chrome
height to get desktop coordinates, then clicks with a human motion model
(curved path, eased velocity, occasional overshoot, dwell before press).
Raises the browser application first, since macOS consumes a click on an
unfocused window rather than delivering it.

Refuses to click when the element is covered by an overlay, when the
coordinates fall off-screen, or when the browser cannot be confirmed
frontmost.

Verified: API round-trips, capture pruning, locate claim-once semantics,
motion geometry and timing, and focus activation — the last two against
stubs, since pyautogui and pyobjc are not installed here. NOT verified
end to end: Chrome is still running a stale build of the extension, so a
locate request has never completed against a real page and no real click
has been sent.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Brandon Li
2026-08-27 16:15:18 -05:00
co-authored by Claude Opus 5
parent 24a54c6642
commit 54221bbc0c
19 changed files with 1752 additions and 2 deletions
+80
View File
@@ -0,0 +1,80 @@
import { NextRequest } from 'next/server';
import { saveCapture, getLatestCapture, clearCaptures } from '@/lib/db';
import { corsJson, corsPreflight } from '../cors';
/** Guard against a runaway page dumping tens of megabytes into SQLite every poll. */
const MAX_HTML_BYTES = 8 * 1024 * 1024;
interface CapturePayload {
url?: string;
title?: string;
html?: string;
viewport?: unknown;
elements?: unknown;
capturedAt?: number;
}
/** Written by the Chromium extension each time it scrapes the target tab. */
export async function POST(req: NextRequest) {
try {
const body = await req.json() as CapturePayload;
if (typeof body.html !== 'string' || typeof body.url !== 'string') {
return corsJson({ error: '`url` and `html` are required' }, { status: 400 });
}
if (body.html.length > MAX_HTML_BYTES) {
return corsJson({ error: `HTML exceeds ${MAX_HTML_BYTES} bytes` }, { status: 413 });
}
const capturedAt = typeof body.capturedAt === 'number' ? body.capturedAt : Date.now();
saveCapture({
url: body.url,
title: typeof body.title === 'string' ? body.title : '',
html: body.html,
viewport: JSON.stringify(body.viewport ?? {}),
elements: JSON.stringify(body.elements ?? []),
captured_at: capturedAt,
});
return corsJson({ ok: true, capturedAt, bytes: body.html.length });
} catch (err: any) {
return corsJson({ error: err?.message ?? 'Failed to save capture' }, { status: 500 });
}
}
/** Read by the AutoBuyer page. Pass `?since=<capturedAt>` to skip re-sending
* an unchanged capture — the HTML blob is megabytes and the page polls often. */
export async function GET(req: NextRequest) {
const row = getLatestCapture();
if (!row) return corsJson({ capture: null });
const since = Number(req.nextUrl.searchParams.get('since'));
if (Number.isFinite(since) && since > 0 && since >= row.captured_at) {
return corsJson({ unchanged: true });
}
return corsJson({
capture: {
url: row.url,
title: row.title,
html: row.html,
viewport: safeParse(row.viewport, {}),
elements: safeParse(row.elements, []),
capturedAt: row.captured_at,
},
});
}
export async function DELETE() {
clearCaptures();
return corsJson({ ok: true });
}
export async function OPTIONS() {
return corsPreflight();
}
function safeParse<T>(json: string, fallback: T): T {
try { return JSON.parse(json) as T; } catch { return fallback; }
}
+19
View File
@@ -0,0 +1,19 @@
import { NextResponse } from 'next/server';
/** The Chromium extension calls these routes from a background service worker,
* whose Origin is `chrome-extension://<id>`. The id changes every time the
* unpacked extension is reloaded, so we allow any origin — these routes are
* only ever reachable on the LAN behind the port-3000 firewall rule. */
export const CORS_HEADERS = {
'Access-Control-Allow-Origin': '*',
'Access-Control-Allow-Methods': 'GET, POST, PATCH, DELETE, OPTIONS',
'Access-Control-Allow-Headers': 'Content-Type',
} as const;
export function corsJson(body: unknown, init?: { status?: number }) {
return NextResponse.json(body, { status: init?.status ?? 200, headers: CORS_HEADERS });
}
export function corsPreflight() {
return new NextResponse(null, { status: 204, headers: CORS_HEADERS });
}
+21
View File
@@ -0,0 +1,21 @@
import { claimLocateRequest } from '@/lib/db';
import { corsJson, corsPreflight } from '../../cors';
/** The extension takes the oldest pending request. Claiming marks it in-flight so
* the next poll doesn't run the same click a second time. */
export async function POST() {
const row = claimLocateRequest();
if (!row) return corsJson({ request: null });
return corsJson({
request: {
id: row.id,
selector: row.selector,
index: row.match_index,
urlPattern: row.url_pattern,
},
});
}
export async function OPTIONS() {
return corsPreflight();
}
+27
View File
@@ -0,0 +1,27 @@
import { NextRequest } from 'next/server';
import { resolveLocateRequest, getLocateRequest } from '@/lib/db';
import { corsJson, corsPreflight } from '../../cors';
/** The extension reports back where the element landed, or why it couldn't. */
export async function POST(req: NextRequest) {
try {
const body = await req.json() as { id?: unknown; result?: unknown; error?: unknown };
const id = Number(body.id);
if (!Number.isInteger(id) || id <= 0) {
return corsJson({ error: '`id` is required' }, { status: 400 });
}
if (!getLocateRequest(id)) {
return corsJson({ error: 'No such request' }, { status: 404 });
}
const error = typeof body.error === 'string' && body.error ? body.error : null;
resolveLocateRequest(id, error ? null : (body.result ?? null), error);
return corsJson({ ok: true });
} catch (err: any) {
return corsJson({ error: err?.message ?? 'Failed to record result' }, { status: 500 });
}
}
export async function OPTIONS() {
return corsPreflight();
}
+42
View File
@@ -0,0 +1,42 @@
import { NextRequest } from 'next/server';
import { createLocateRequest, getLocateRequest } from '@/lib/db';
import { corsJson, corsPreflight } from '../cors';
/** Python enqueues "find this selector and tell me where it is on screen". */
export async function POST(req: NextRequest) {
try {
const body = await req.json() as { selector?: unknown; index?: unknown; urlPattern?: unknown };
if (typeof body.selector !== 'string' || !body.selector.trim()) {
return corsJson({ error: '`selector` is required' }, { status: 400 });
}
const index = Number.isInteger(body.index) ? Number(body.index) : 0;
const urlPattern = typeof body.urlPattern === 'string' ? body.urlPattern : '';
const row = createLocateRequest(body.selector.trim(), index, urlPattern);
return corsJson({ id: row.id, status: row.status });
} catch (err: any) {
return corsJson({ error: err?.message ?? 'Failed to queue request' }, { status: 500 });
}
}
/** Python polls `?id=` until the extension resolves it. */
export async function GET(req: NextRequest) {
const id = Number(req.nextUrl.searchParams.get('id'));
if (!Number.isInteger(id) || id <= 0) {
return corsJson({ error: '`id` is required' }, { status: 400 });
}
const row = getLocateRequest(id);
if (!row) return corsJson({ error: 'No such request' }, { status: 404 });
return corsJson({
id: row.id,
selector: row.selector,
status: row.status,
result: row.result ? JSON.parse(row.result) : null,
error: row.error,
});
}
export async function OPTIONS() {
return corsPreflight();
}
+31
View File
@@ -0,0 +1,31 @@
import { NextRequest } from 'next/server';
import { getSetting, setSetting, countPendingLocate } from '@/lib/db';
import { corsJson, corsPreflight } from '../cors';
/** Polled by the Chromium extension every few seconds, and by the AutoBuyer page.
* `pendingLocate` rides along so the extension learns about queued clicks without
* a second request on every tick. */
export async function GET() {
return corsJson({
enabled: getSetting('autobuyer_enabled') === '1',
pendingLocate: countPendingLocate(),
});
}
/** Flipped by the ON/OFF button on the AutoBuyer page. */
export async function PATCH(req: NextRequest) {
try {
const body = await req.json() as { enabled?: unknown };
if (typeof body.enabled !== 'boolean') {
return corsJson({ error: '`enabled` must be a boolean' }, { status: 400 });
}
setSetting('autobuyer_enabled', body.enabled ? '1' : '0');
return corsJson({ enabled: body.enabled });
} catch (err: any) {
return corsJson({ error: err?.message ?? 'Failed to set status' }, { status: 500 });
}
}
export async function OPTIONS() {
return corsPreflight();
}
+271 -2
View File
@@ -1,11 +1,280 @@
'use client';
import { useCallback, useEffect, useRef, useState } from 'react';
interface ElementPosition {
index: number;
tag: string;
id: string | null;
text: string;
visible: boolean;
inViewport: boolean;
viewport: { x: number; y: number; width: number; height: number };
page: { x: number; y: number };
screen: { x: number; y: number };
}
interface ViewportMetrics {
scrollX: number;
scrollY: number;
innerWidth: number;
innerHeight: number;
screenX: number;
screenY: number;
chromeHeight: number;
devicePixelRatio: number;
}
interface Capture {
url: string;
title: string;
html: string;
viewport: Partial<ViewportMetrics>;
elements: ElementPosition[];
capturedAt: number;
}
const POLL_MS = 2000;
/** Point relative URLs (stylesheets, images, fonts) at the origin the snapshot came
* from, otherwise the render comes out unstyled. The HTML parser hoists a leading
* <base> into <head>, so prepending works even if the markup has no explicit head. */
function withBaseTag(html: string, url: string): string {
const base = `<base href="${url.replace(/"/g, '&quot;')}">`;
const head = html.match(/<head\b[^>]*>/i);
if (!head || head.index === undefined) return base + html;
const at = head.index + head[0].length;
return html.slice(0, at) + base + html.slice(at);
}
export default function AutoBuyer() {
const [enabled, setEnabled] = useState(false);
const [capture, setCapture] = useState<Capture | null>(null);
const [busy, setBusy] = useState(false);
const [error, setError] = useState<string | null>(null);
const [view, setView] = useState<'rendered' | 'source'>('source');
// The extension posts a new capture every few seconds, and re-feeding srcDoc would
// reload the frame that often. So the rendered view shows a snapshot pinned when you
// opened it — clicking Rendered again re-pins to the latest capture.
const [pinned, setPinned] = useState<Capture | null>(null);
// Tracks the newest capture we already hold, so the poll can skip re-downloading it.
const lastAtRef = useRef(0);
const poll = useCallback(async () => {
try {
const s = await fetch('/api/autobuyer/status', { cache: 'no-store' }).then((r) => r.json());
setEnabled(!!s.enabled);
if (!s.enabled) return;
const url = lastAtRef.current
? `/api/autobuyer/capture?since=${lastAtRef.current}`
: '/api/autobuyer/capture';
const data = await fetch(url, { cache: 'no-store' }).then((r) => r.json());
if (data.unchanged || !data.capture) return;
lastAtRef.current = data.capture.capturedAt;
setCapture(data.capture);
setError(null);
} catch (err: any) {
setError(err?.message ?? 'Poll failed');
}
}, []);
useEffect(() => {
poll();
const id = setInterval(poll, POLL_MS);
return () => clearInterval(id);
}, [poll]);
async function toggle() {
setBusy(true);
try {
const res = await fetch('/api/autobuyer/status', {
method: 'PATCH',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ enabled: !enabled }),
});
const data = await res.json();
setEnabled(!!data.enabled);
setError(null);
} catch (err: any) {
setError(err?.message ?? 'Failed to toggle');
} finally {
setBusy(false);
}
}
function showRendered() {
setView('rendered');
if (capture) setPinned(capture);
}
async function clearCapture() {
await fetch('/api/autobuyer/capture', { method: 'DELETE' });
lastAtRef.current = 0;
setCapture(null);
setPinned(null);
}
const stale = capture ? Date.now() - capture.capturedAt > 15000 : false;
return (
<div className="min-h-screen bg-slate-50 p-8">
<div className="max-w-7xl mx-auto">
<h1 className="text-2xl font-bold text-slate-900 mb-6">AutoBuyer</h1>
<div className="bg-white border border-slate-200 rounded-xl shadow-sm p-8 text-center text-slate-400">
Coming soon
{/* ── Capture switch ── */}
<div className="bg-white border border-slate-200 rounded-xl shadow-sm mb-6">
<div className="flex items-center justify-between px-4 py-3">
<div>
<p className="text-sm font-medium text-slate-800">Page capture</p>
<p className="text-xs text-slate-400 mt-0.5">
While on, the AutoFirmer Capture extension posts the target tab&apos;s HTML here every few seconds
</p>
</div>
<button
onClick={toggle}
disabled={busy}
className={`text-sm px-4 py-1.5 rounded-lg font-medium transition-colors disabled:opacity-50 ${
enabled
? 'bg-green-500 hover:bg-green-600 text-white'
: 'bg-slate-200 hover:bg-slate-300 text-slate-700'
}`}
>
{busy ? '…' : enabled ? 'ON' : 'OFF'}
</button>
</div>
</div>
{error && (
<div className="mb-6 rounded-lg border border-red-200 bg-red-50 px-4 py-2 text-sm text-red-700">
{error}
</div>
)}
{/* ── Latest capture ── */}
<h2 className="text-sm font-semibold uppercase tracking-wider text-slate-500 mb-3">
Latest capture
</h2>
{!enabled ? (
<div className="bg-white border border-slate-200 rounded-xl shadow-sm p-8 text-center text-slate-400">
Capture is off
</div>
) : !capture ? (
<div className="bg-white border border-slate-200 rounded-xl shadow-sm p-8 text-center text-slate-400">
Waiting for the extension open a page in Chrome with the extension installed
</div>
) : (
<div className="bg-white border border-slate-200 rounded-xl shadow-sm overflow-hidden">
<div className="flex flex-wrap items-baseline gap-x-4 gap-y-1 px-4 py-3 border-b border-slate-100">
<span className="text-sm font-medium text-slate-800">{capture.title || '(untitled)'}</span>
<span className="text-xs font-mono text-slate-400 break-all">{capture.url}</span>
<span className="ml-auto flex items-center gap-3 text-xs text-slate-400">
<span>{(capture.html.length / 1024).toFixed(1)} KB</span>
<span className={stale ? 'text-amber-600' : ''}>
{new Date(capture.capturedAt).toLocaleTimeString()}
{stale && ' (stale)'}
</span>
<span className="flex rounded-md border border-slate-200 overflow-hidden">
{(['rendered', 'source'] as const).map((v) => (
<button
key={v}
onClick={() => (v === 'rendered' ? showRendered() : setView('source'))}
className={`px-2 py-0.5 capitalize transition-colors ${
view === v
? 'bg-slate-700 text-white'
: 'text-slate-500 hover:bg-slate-50'
}`}
>
{v}
</button>
))}
</span>
<button
onClick={clearCapture}
className="text-slate-400 hover:text-red-500 transition-colors"
>
Clear
</button>
</span>
</div>
{capture.viewport?.innerWidth != null && (
<div className="px-4 py-2 border-b border-slate-100 flex flex-wrap gap-x-5 gap-y-1 text-xs font-mono text-slate-500">
<span>viewport {capture.viewport.innerWidth}×{capture.viewport.innerHeight}</span>
<span>scroll {Math.round(capture.viewport.scrollX ?? 0)},{Math.round(capture.viewport.scrollY ?? 0)}</span>
<span>window@screen {capture.viewport.screenX},{capture.viewport.screenY}</span>
<span>chrome {capture.viewport.chromeHeight}px</span>
<span>dpr {capture.viewport.devicePixelRatio}</span>
</div>
)}
{capture.elements?.length > 0 && (
<div className="px-4 py-3 border-b border-slate-100">
<p className="text-xs uppercase tracking-wider text-slate-400 mb-2">
Matched elements ({capture.elements.length})
</p>
<div className="overflow-x-auto">
<table className="text-xs font-mono text-slate-600">
<thead className="text-slate-400">
<tr>
<th className="text-left pr-4 font-normal">tag</th>
<th className="text-left pr-4 font-normal">text</th>
<th className="text-right pr-4 font-normal">window x,y</th>
<th className="text-right pr-4 font-normal">size</th>
<th className="text-right pr-4 font-normal">page x,y</th>
<th className="text-right pr-4 font-normal">screen x,y</th>
<th className="text-left font-normal">vis</th>
</tr>
</thead>
<tbody>
{capture.elements.map((el) => (
<tr key={el.index} className="border-t border-slate-50">
<td className="pr-4 py-1">{el.tag}{el.id ? `#${el.id}` : ''}</td>
<td className="pr-4 py-1 max-w-xs truncate">{el.text}</td>
<td className="pr-4 py-1 text-right">{Math.round(el.viewport.x)},{Math.round(el.viewport.y)}</td>
<td className="pr-4 py-1 text-right">{Math.round(el.viewport.width)}×{Math.round(el.viewport.height)}</td>
<td className="pr-4 py-1 text-right">{Math.round(el.page.x)},{Math.round(el.page.y)}</td>
<td className="pr-4 py-1 text-right">{Math.round(el.screen.x)},{Math.round(el.screen.y)}</td>
<td className="py-1">{el.inViewport ? '✓' : el.visible ? 'off-screen' : 'hidden'}</td>
</tr>
))}
</tbody>
</table>
</div>
</div>
)}
{view === 'rendered' ? (
<>
{pinned && pinned.capturedAt !== capture.capturedAt && (
<div className="px-4 py-1.5 border-b border-slate-100 text-xs text-amber-600">
Frozen snapshot from {new Date(pinned.capturedAt).toLocaleTimeString()} click Rendered again to refresh
</div>
)}
{/* allow-scripts WITHOUT allow-same-origin: the frame runs the
page's own JS inside an opaque origin, so it cannot reach this
dashboard's DOM, storage, or same-origin API routes — /api/firms
serves firm credentials. Granting both flags together is what
would let a frame drop its own sandbox.
Scripts are needed because most sites ship content at opacity:0
and fade it in with JS — blocked, they render blank. */}
<iframe
sandbox="allow-scripts"
srcDoc={withBaseTag((pinned ?? capture).html, (pinned ?? capture).url)}
title="Captured page"
className="block w-full h-[70vh] bg-white"
/>
</>
) : (
<pre className="max-h-[70vh] overflow-auto bg-slate-900 text-slate-200 text-xs font-mono p-4 whitespace-pre-wrap break-all">
{capture.html}
</pre>
)}
</div>
)}
</div>
</div>
);