Add autobuyer page capture, browser extension, and desktop clicker
Builds the pipeline the autobuyer needs: see the page, find an element, click it. extension/ — MV3 Chromium extension. Polls /api/autobuyer/status and, while on, scrapes the target tab's HTML and posts it back. Also serves locate requests: focuses the window, scrolls the element into view, and reports its position. host_permissions is scoped to tradeify plus localhost so it cannot read other sites — an empty target pattern would otherwise capture whatever tab happened to be active, including banking or mail. app/api/autobuyer/ — status toggle, capture store, and the locate request queue. CORS is open because the extension's origin changes every time an unpacked extension is reloaded. app/autobuyer/page.tsx — ON switch, source view (default) and a rendered view. The render uses sandbox="allow-scripts" without allow-same-origin: the page's own JS is needed because sites ship content at opacity:0 and fade it in, but the frame must not reach the dashboard's same-origin API routes, which serve firm credentials. clicker/ — Python CLI. Asks the extension where a selector is, adds the element rect to the window's screen position and the browser chrome height to get desktop coordinates, then clicks with a human motion model (curved path, eased velocity, occasional overshoot, dwell before press). Raises the browser application first, since macOS consumes a click on an unfocused window rather than delivering it. Refuses to click when the element is covered by an overlay, when the coordinates fall off-screen, or when the browser cannot be confirmed frontmost. Verified: API round-trips, capture pruning, locate claim-once semantics, motion geometry and timing, and focus activation — the last two against stubs, since pyautogui and pyobjc are not installed here. NOT verified end to end: Chrome is still running a stale build of the extension, so a locate request has never completed against a real page and no real click has been sent. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
24a54c6642
commit
54221bbc0c
@@ -0,0 +1,140 @@
|
||||
"""Bring the browser to the front before clicking.
|
||||
|
||||
macOS (and Windows, to a lesser degree) treats a click on an unfocused window as
|
||||
an activation gesture: the click raises the window and is swallowed there, never
|
||||
reaching the control underneath. So an automated click against a background Chrome
|
||||
does nothing at all the first time, then works on the second attempt — which looks
|
||||
like a flaky clicker but is really the window manager doing its job.
|
||||
|
||||
The extension already calls chrome.windows.update({focused: true}), but that only
|
||||
orders windows *within* Chrome. If the frontmost application is your terminal —
|
||||
which it is, because that's where this script was launched — Chrome as a whole is
|
||||
still in the background. This module raises the application itself.
|
||||
"""
|
||||
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
# Chrome ships under several bundle ids; accept whichever is installed.
|
||||
MAC_BUNDLES = (
|
||||
"com.google.Chrome",
|
||||
"com.google.Chrome.beta",
|
||||
"com.google.Chrome.dev",
|
||||
"com.google.Chrome.canary",
|
||||
"com.brave.Browser",
|
||||
"com.microsoft.edgemac",
|
||||
)
|
||||
|
||||
SETTLE = 0.20 # let the window manager finish raising before measuring or clicking
|
||||
|
||||
|
||||
class FocusResult:
|
||||
def __init__(self, ok: bool, detail: str):
|
||||
self.ok = ok
|
||||
self.detail = detail
|
||||
|
||||
def __bool__(self) -> bool:
|
||||
return self.ok
|
||||
|
||||
|
||||
def _mac_workspace():
|
||||
"""NSWorkspace via pyobjc. Unlike AppleScript this needs no Automation
|
||||
permission — activating an app is not scripting it."""
|
||||
try:
|
||||
from AppKit import NSWorkspace
|
||||
except ImportError:
|
||||
return None
|
||||
return NSWorkspace.sharedWorkspace()
|
||||
|
||||
|
||||
def frontmost() -> str | None:
|
||||
"""Bundle id (macOS) or process name of the frontmost application."""
|
||||
if sys.platform == "darwin":
|
||||
ws = _mac_workspace()
|
||||
if ws is None:
|
||||
return None
|
||||
app = ws.frontmostApplication()
|
||||
return app.bundleIdentifier() if app else None
|
||||
return None
|
||||
|
||||
|
||||
def _mac_activate() -> FocusResult:
|
||||
ws = _mac_workspace()
|
||||
if ws is None:
|
||||
# pyobjc's AppKit isn't present. osascript works but may prompt for
|
||||
# Automation permission the first time.
|
||||
try:
|
||||
subprocess.run(
|
||||
["osascript", "-e", 'tell application "Google Chrome" to activate'],
|
||||
check=True, capture_output=True, timeout=5,
|
||||
)
|
||||
return FocusResult(True, "activated via osascript")
|
||||
except Exception as exc:
|
||||
return FocusResult(False, f"could not activate Chrome ({exc})")
|
||||
|
||||
running = {a.bundleIdentifier(): a for a in ws.runningApplications()}
|
||||
for bundle in MAC_BUNDLES:
|
||||
app = running.get(bundle)
|
||||
if app is None:
|
||||
continue
|
||||
# NSApplicationActivateIgnoringOtherApps — take focus even though the
|
||||
# terminal currently owns it.
|
||||
app.activateWithOptions_(1 << 1)
|
||||
return FocusResult(True, f"activated {bundle}")
|
||||
|
||||
return FocusResult(False, "no Chrome-family browser is running")
|
||||
|
||||
|
||||
def _other_activate() -> FocusResult:
|
||||
"""Windows/Linux: pyautogui already depends on pygetwindow, so use it."""
|
||||
try:
|
||||
import pygetwindow
|
||||
except ImportError:
|
||||
return FocusResult(False, "pygetwindow unavailable — cannot raise the browser")
|
||||
|
||||
wins = [w for w in pygetwindow.getAllWindows()
|
||||
if w.title and "Chrome" in w.title and w.visible]
|
||||
if not wins:
|
||||
return FocusResult(False, "no Chrome window found")
|
||||
try:
|
||||
win = wins[0]
|
||||
if getattr(win, "isMinimized", False):
|
||||
win.restore()
|
||||
win.activate()
|
||||
return FocusResult(True, f"activated window {win.title[:40]!r}")
|
||||
except Exception as exc:
|
||||
return FocusResult(False, f"could not activate window ({exc})")
|
||||
|
||||
|
||||
def activate_browser() -> FocusResult:
|
||||
"""Raise the browser application above everything else."""
|
||||
result = _mac_activate() if sys.platform == "darwin" else _other_activate()
|
||||
if result.ok:
|
||||
time.sleep(SETTLE)
|
||||
return result
|
||||
|
||||
|
||||
def ensure_frontmost(timeout: float = 1.5) -> FocusResult:
|
||||
"""Raise the browser and, where we can check, confirm it actually came forward.
|
||||
|
||||
Returning ok=False does not mean the click will fail — only that we could not
|
||||
verify. The caller decides whether to proceed.
|
||||
"""
|
||||
result = activate_browser()
|
||||
if not result.ok:
|
||||
return result
|
||||
|
||||
if sys.platform != "darwin":
|
||||
return result # no cheap way to verify; activation call succeeded
|
||||
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
front = frontmost()
|
||||
if front is None:
|
||||
return FocusResult(True, result.detail + " (unverified)")
|
||||
if front in MAC_BUNDLES:
|
||||
return FocusResult(True, f"{front} is frontmost")
|
||||
time.sleep(0.05)
|
||||
|
||||
return FocusResult(False, f"browser did not come to the front (frontmost is {frontmost()})")
|
||||
Reference in New Issue
Block a user