Add autobuyer page capture, browser extension, and desktop clicker
Builds the pipeline the autobuyer needs: see the page, find an element, click it. extension/ — MV3 Chromium extension. Polls /api/autobuyer/status and, while on, scrapes the target tab's HTML and posts it back. Also serves locate requests: focuses the window, scrolls the element into view, and reports its position. host_permissions is scoped to tradeify plus localhost so it cannot read other sites — an empty target pattern would otherwise capture whatever tab happened to be active, including banking or mail. app/api/autobuyer/ — status toggle, capture store, and the locate request queue. CORS is open because the extension's origin changes every time an unpacked extension is reloaded. app/autobuyer/page.tsx — ON switch, source view (default) and a rendered view. The render uses sandbox="allow-scripts" without allow-same-origin: the page's own JS is needed because sites ship content at opacity:0 and fade it in, but the frame must not reach the dashboard's same-origin API routes, which serve firm credentials. clicker/ — Python CLI. Asks the extension where a selector is, adds the element rect to the window's screen position and the browser chrome height to get desktop coordinates, then clicks with a human motion model (curved path, eased velocity, occasional overshoot, dwell before press). Raises the browser application first, since macOS consumes a click on an unfocused window rather than delivering it. Refuses to click when the element is covered by an overlay, when the coordinates fall off-screen, or when the browser cannot be confirmed frontmost. Verified: API round-trips, capture pruning, locate claim-once semantics, motion geometry and timing, and focus activation — the last two against stubs, since pyautogui and pyobjc are not installed here. NOT verified end to end: Chrome is still running a stale build of the extension, so a locate request has never completed against a real page and no real click has been sent. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Brandon Li
co-authored by
Claude Opus 5
Claude Opus 5
parent
24a54c6642
commit
54221bbc0c
+271
-2
@@ -1,11 +1,280 @@
|
||||
'use client';
|
||||
|
||||
import { useCallback, useEffect, useRef, useState } from 'react';
|
||||
|
||||
interface ElementPosition {
|
||||
index: number;
|
||||
tag: string;
|
||||
id: string | null;
|
||||
text: string;
|
||||
visible: boolean;
|
||||
inViewport: boolean;
|
||||
viewport: { x: number; y: number; width: number; height: number };
|
||||
page: { x: number; y: number };
|
||||
screen: { x: number; y: number };
|
||||
}
|
||||
|
||||
interface ViewportMetrics {
|
||||
scrollX: number;
|
||||
scrollY: number;
|
||||
innerWidth: number;
|
||||
innerHeight: number;
|
||||
screenX: number;
|
||||
screenY: number;
|
||||
chromeHeight: number;
|
||||
devicePixelRatio: number;
|
||||
}
|
||||
|
||||
interface Capture {
|
||||
url: string;
|
||||
title: string;
|
||||
html: string;
|
||||
viewport: Partial<ViewportMetrics>;
|
||||
elements: ElementPosition[];
|
||||
capturedAt: number;
|
||||
}
|
||||
|
||||
const POLL_MS = 2000;
|
||||
|
||||
/** Point relative URLs (stylesheets, images, fonts) at the origin the snapshot came
|
||||
* from, otherwise the render comes out unstyled. The HTML parser hoists a leading
|
||||
* <base> into <head>, so prepending works even if the markup has no explicit head. */
|
||||
function withBaseTag(html: string, url: string): string {
|
||||
const base = `<base href="${url.replace(/"/g, '"')}">`;
|
||||
const head = html.match(/<head\b[^>]*>/i);
|
||||
if (!head || head.index === undefined) return base + html;
|
||||
const at = head.index + head[0].length;
|
||||
return html.slice(0, at) + base + html.slice(at);
|
||||
}
|
||||
|
||||
export default function AutoBuyer() {
|
||||
const [enabled, setEnabled] = useState(false);
|
||||
const [capture, setCapture] = useState<Capture | null>(null);
|
||||
const [busy, setBusy] = useState(false);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
const [view, setView] = useState<'rendered' | 'source'>('source');
|
||||
// The extension posts a new capture every few seconds, and re-feeding srcDoc would
|
||||
// reload the frame that often. So the rendered view shows a snapshot pinned when you
|
||||
// opened it — clicking Rendered again re-pins to the latest capture.
|
||||
const [pinned, setPinned] = useState<Capture | null>(null);
|
||||
|
||||
// Tracks the newest capture we already hold, so the poll can skip re-downloading it.
|
||||
const lastAtRef = useRef(0);
|
||||
|
||||
const poll = useCallback(async () => {
|
||||
try {
|
||||
const s = await fetch('/api/autobuyer/status', { cache: 'no-store' }).then((r) => r.json());
|
||||
setEnabled(!!s.enabled);
|
||||
if (!s.enabled) return;
|
||||
|
||||
const url = lastAtRef.current
|
||||
? `/api/autobuyer/capture?since=${lastAtRef.current}`
|
||||
: '/api/autobuyer/capture';
|
||||
const data = await fetch(url, { cache: 'no-store' }).then((r) => r.json());
|
||||
if (data.unchanged || !data.capture) return;
|
||||
|
||||
lastAtRef.current = data.capture.capturedAt;
|
||||
setCapture(data.capture);
|
||||
setError(null);
|
||||
} catch (err: any) {
|
||||
setError(err?.message ?? 'Poll failed');
|
||||
}
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
poll();
|
||||
const id = setInterval(poll, POLL_MS);
|
||||
return () => clearInterval(id);
|
||||
}, [poll]);
|
||||
|
||||
async function toggle() {
|
||||
setBusy(true);
|
||||
try {
|
||||
const res = await fetch('/api/autobuyer/status', {
|
||||
method: 'PATCH',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ enabled: !enabled }),
|
||||
});
|
||||
const data = await res.json();
|
||||
setEnabled(!!data.enabled);
|
||||
setError(null);
|
||||
} catch (err: any) {
|
||||
setError(err?.message ?? 'Failed to toggle');
|
||||
} finally {
|
||||
setBusy(false);
|
||||
}
|
||||
}
|
||||
|
||||
function showRendered() {
|
||||
setView('rendered');
|
||||
if (capture) setPinned(capture);
|
||||
}
|
||||
|
||||
async function clearCapture() {
|
||||
await fetch('/api/autobuyer/capture', { method: 'DELETE' });
|
||||
lastAtRef.current = 0;
|
||||
setCapture(null);
|
||||
setPinned(null);
|
||||
}
|
||||
|
||||
const stale = capture ? Date.now() - capture.capturedAt > 15000 : false;
|
||||
|
||||
return (
|
||||
<div className="min-h-screen bg-slate-50 p-8">
|
||||
<div className="max-w-7xl mx-auto">
|
||||
<h1 className="text-2xl font-bold text-slate-900 mb-6">AutoBuyer</h1>
|
||||
<div className="bg-white border border-slate-200 rounded-xl shadow-sm p-8 text-center text-slate-400">
|
||||
Coming soon
|
||||
|
||||
{/* ── Capture switch ── */}
|
||||
<div className="bg-white border border-slate-200 rounded-xl shadow-sm mb-6">
|
||||
<div className="flex items-center justify-between px-4 py-3">
|
||||
<div>
|
||||
<p className="text-sm font-medium text-slate-800">Page capture</p>
|
||||
<p className="text-xs text-slate-400 mt-0.5">
|
||||
While on, the AutoFirmer Capture extension posts the target tab's HTML here every few seconds
|
||||
</p>
|
||||
</div>
|
||||
<button
|
||||
onClick={toggle}
|
||||
disabled={busy}
|
||||
className={`text-sm px-4 py-1.5 rounded-lg font-medium transition-colors disabled:opacity-50 ${
|
||||
enabled
|
||||
? 'bg-green-500 hover:bg-green-600 text-white'
|
||||
: 'bg-slate-200 hover:bg-slate-300 text-slate-700'
|
||||
}`}
|
||||
>
|
||||
{busy ? '…' : enabled ? 'ON' : 'OFF'}
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{error && (
|
||||
<div className="mb-6 rounded-lg border border-red-200 bg-red-50 px-4 py-2 text-sm text-red-700">
|
||||
{error}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* ── Latest capture ── */}
|
||||
<h2 className="text-sm font-semibold uppercase tracking-wider text-slate-500 mb-3">
|
||||
Latest capture
|
||||
</h2>
|
||||
|
||||
{!enabled ? (
|
||||
<div className="bg-white border border-slate-200 rounded-xl shadow-sm p-8 text-center text-slate-400">
|
||||
Capture is off
|
||||
</div>
|
||||
) : !capture ? (
|
||||
<div className="bg-white border border-slate-200 rounded-xl shadow-sm p-8 text-center text-slate-400">
|
||||
Waiting for the extension… open a page in Chrome with the extension installed
|
||||
</div>
|
||||
) : (
|
||||
<div className="bg-white border border-slate-200 rounded-xl shadow-sm overflow-hidden">
|
||||
<div className="flex flex-wrap items-baseline gap-x-4 gap-y-1 px-4 py-3 border-b border-slate-100">
|
||||
<span className="text-sm font-medium text-slate-800">{capture.title || '(untitled)'}</span>
|
||||
<span className="text-xs font-mono text-slate-400 break-all">{capture.url}</span>
|
||||
<span className="ml-auto flex items-center gap-3 text-xs text-slate-400">
|
||||
<span>{(capture.html.length / 1024).toFixed(1)} KB</span>
|
||||
<span className={stale ? 'text-amber-600' : ''}>
|
||||
{new Date(capture.capturedAt).toLocaleTimeString()}
|
||||
{stale && ' (stale)'}
|
||||
</span>
|
||||
<span className="flex rounded-md border border-slate-200 overflow-hidden">
|
||||
{(['rendered', 'source'] as const).map((v) => (
|
||||
<button
|
||||
key={v}
|
||||
onClick={() => (v === 'rendered' ? showRendered() : setView('source'))}
|
||||
className={`px-2 py-0.5 capitalize transition-colors ${
|
||||
view === v
|
||||
? 'bg-slate-700 text-white'
|
||||
: 'text-slate-500 hover:bg-slate-50'
|
||||
}`}
|
||||
>
|
||||
{v}
|
||||
</button>
|
||||
))}
|
||||
</span>
|
||||
<button
|
||||
onClick={clearCapture}
|
||||
className="text-slate-400 hover:text-red-500 transition-colors"
|
||||
>
|
||||
Clear
|
||||
</button>
|
||||
</span>
|
||||
</div>
|
||||
|
||||
{capture.viewport?.innerWidth != null && (
|
||||
<div className="px-4 py-2 border-b border-slate-100 flex flex-wrap gap-x-5 gap-y-1 text-xs font-mono text-slate-500">
|
||||
<span>viewport {capture.viewport.innerWidth}×{capture.viewport.innerHeight}</span>
|
||||
<span>scroll {Math.round(capture.viewport.scrollX ?? 0)},{Math.round(capture.viewport.scrollY ?? 0)}</span>
|
||||
<span>window@screen {capture.viewport.screenX},{capture.viewport.screenY}</span>
|
||||
<span>chrome {capture.viewport.chromeHeight}px</span>
|
||||
<span>dpr {capture.viewport.devicePixelRatio}</span>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{capture.elements?.length > 0 && (
|
||||
<div className="px-4 py-3 border-b border-slate-100">
|
||||
<p className="text-xs uppercase tracking-wider text-slate-400 mb-2">
|
||||
Matched elements ({capture.elements.length})
|
||||
</p>
|
||||
<div className="overflow-x-auto">
|
||||
<table className="text-xs font-mono text-slate-600">
|
||||
<thead className="text-slate-400">
|
||||
<tr>
|
||||
<th className="text-left pr-4 font-normal">tag</th>
|
||||
<th className="text-left pr-4 font-normal">text</th>
|
||||
<th className="text-right pr-4 font-normal">window x,y</th>
|
||||
<th className="text-right pr-4 font-normal">size</th>
|
||||
<th className="text-right pr-4 font-normal">page x,y</th>
|
||||
<th className="text-right pr-4 font-normal">screen x,y</th>
|
||||
<th className="text-left font-normal">vis</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{capture.elements.map((el) => (
|
||||
<tr key={el.index} className="border-t border-slate-50">
|
||||
<td className="pr-4 py-1">{el.tag}{el.id ? `#${el.id}` : ''}</td>
|
||||
<td className="pr-4 py-1 max-w-xs truncate">{el.text}</td>
|
||||
<td className="pr-4 py-1 text-right">{Math.round(el.viewport.x)},{Math.round(el.viewport.y)}</td>
|
||||
<td className="pr-4 py-1 text-right">{Math.round(el.viewport.width)}×{Math.round(el.viewport.height)}</td>
|
||||
<td className="pr-4 py-1 text-right">{Math.round(el.page.x)},{Math.round(el.page.y)}</td>
|
||||
<td className="pr-4 py-1 text-right">{Math.round(el.screen.x)},{Math.round(el.screen.y)}</td>
|
||||
<td className="py-1">{el.inViewport ? '✓' : el.visible ? 'off-screen' : 'hidden'}</td>
|
||||
</tr>
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{view === 'rendered' ? (
|
||||
<>
|
||||
{pinned && pinned.capturedAt !== capture.capturedAt && (
|
||||
<div className="px-4 py-1.5 border-b border-slate-100 text-xs text-amber-600">
|
||||
Frozen snapshot from {new Date(pinned.capturedAt).toLocaleTimeString()} — click Rendered again to refresh
|
||||
</div>
|
||||
)}
|
||||
{/* allow-scripts WITHOUT allow-same-origin: the frame runs the
|
||||
page's own JS inside an opaque origin, so it cannot reach this
|
||||
dashboard's DOM, storage, or same-origin API routes — /api/firms
|
||||
serves firm credentials. Granting both flags together is what
|
||||
would let a frame drop its own sandbox.
|
||||
Scripts are needed because most sites ship content at opacity:0
|
||||
and fade it in with JS — blocked, they render blank. */}
|
||||
<iframe
|
||||
sandbox="allow-scripts"
|
||||
srcDoc={withBaseTag((pinned ?? capture).html, (pinned ?? capture).url)}
|
||||
title="Captured page"
|
||||
className="block w-full h-[70vh] bg-white"
|
||||
/>
|
||||
</>
|
||||
) : (
|
||||
<pre className="max-h-[70vh] overflow-auto bg-slate-900 text-slate-200 text-xs font-mono p-4 whitespace-pre-wrap break-all">
|
||||
{capture.html}
|
||||
</pre>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
|
||||
Reference in New Issue
Block a user