Files
Brandon LiandClaude Opus 5 3ac9fe060f Raise the browser the extension reported, and add a diagnostics script
focus.py raised the first browser in its list that happened to be running. On a
machine with both Chrome and Edge installed that is a coin flip, and losing it is
silent: coordinates measured from a tab in one browser, the click delivered into
a window of the other. It presents as selectors failing for no reason. A VPS with
both installed hit exactly this.

The extension now reports which browser is hosting it, and that travels with the
measurement, so the clicker raises the browser the coordinates actually came
from. Asking for a browser that is not running now fails honestly instead of
quietly raising a different one, and the verification step rejects the wrong
browser coming forward. Chromium, Opera and Vivaldi are recognised alongside
Chrome, Edge and Brave, on both platforms.

diagnose.py answers the question a remote desktop makes hard: whether the mouse
is really moving or the viewer simply is not drawing it. It moves the cursor and
reads the position back from the OS, so the answer does not depend on anything
being rendered, and it reports DPI mode, screen size, whether this is an RDP
session, and whether the browser can be raised at all. Nothing is clicked.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-08-30 14:13:18 -05:00

211 lines
6.7 KiB
Python

"""Execution of a single automation step: locate the element, then click or type.
Shared by the CLI (clicker.py) and the automation runner (runner.py) so there is
one implementation of the safety checks. Callers differ only in how they report
progress and how they surface failures.
"""
import random
import focus
class StepError(Exception):
"""A step refused to run, or ran and could not be verified.
`code` mirrors the CLI's exit codes: 3 = refused before acting,
4 = acted but verification failed, 5 = the session is signed out.
"""
def __init__(self, message: str, code: int = 3):
super().__init__(message)
self.code = code
class SignedOutError(StepError):
"""The tab is on the login page, so the session has expired.
Distinct from an ordinary failure because nothing is wrong with the
automation — it just cannot proceed until a human signs in.
"""
def __init__(self, url: str):
super().__init__(
f"signed out — the tab is on {url}. Log in to the site, then run this again.",
code=5,
)
def check_signed_in(found: dict, signed_out: str) -> None:
"""Abort if the page redirected to a login screen.
Worth doing before every action, not just at the start: a session can expire
mid-run, and once it does, every selector resolves against a login form. A
click or a keystroke aimed at the old page would land on whatever that form
happens to render in the same place.
"""
if not signed_out:
return
url = found.get("url") or ""
if signed_out in url:
raise SignedOutError(url)
# Display scaling factors that actually exist. Anything else means the two sides
# are describing different things (usually a multi-monitor desktop) rather than a
# scaled single display.
KNOWN_SCALES = (1.0, 1.25, 1.5, 1.75, 2.0, 2.5, 3.0)
def scale_factor(found: dict, override: float | None, report=None) -> float:
"""CSS pixels and desktop pixels match on macOS and unscaled Windows, but
Windows display scaling breaks that.
Deliberately conservative: a raw ratio is only trusted when it lands on a real
scaling factor. On a multi-monitor desktop the browser reports the display it
is on while pyautogui reports the primary one, and the resulting ratio is
meaningless — halving every coordinate would put clicks far from the target.
Fall back to 1:1 and say so, rather than silently scaling.
"""
if override is not None:
return override
try:
import pyautogui
except ImportError:
return 1.0
css_width = (found.get("screenSize") or {}).get("width")
if not css_width:
return 1.0
os_width = pyautogui.size().width
ratio = os_width / css_width
for known in KNOWN_SCALES:
if abs(ratio - known) < 0.02:
return known
if report:
report(
f"screen size disagrees (browser {css_width}px, OS {os_width}px, "
f"ratio {ratio:.3f}) — assuming 1:1. If clicks land off, pass --scale."
)
return 1.0
def perform(
dash,
action: str,
selector: str,
*,
index: int = 0,
url: str = "",
open_url: str = "",
signed_out: str = "",
text: str | None = None,
clear: bool = False,
scale: float | None = None,
timeout: float = 20.0,
rng: random.Random | None = None,
activate: bool = True,
verify: bool = True,
force: bool = False,
dry_run: bool = False,
robotic: bool = False,
report=None,
on_located=None,
) -> dict:
"""Run one step. Returns the located element's data.
Raises DashboardError if the extension can't find it, StepError if the step
is refused or fails verification.
"""
import pyautogui
import humanize
rng = rng or random.Random()
say = report or (lambda _m: None)
found = dash.locate(selector, index, url, timeout, open_url)
if on_located:
on_located(found)
# Before any check that assumes we are on the real page.
check_signed_in(found, signed_out)
if found.get("covered") and not force:
raise StepError(
f"{selector} is covered by {found.get('coveredBy')} — the click would hit that instead"
)
if action == "type" and not found.get("editable", True) and not force:
kind = found.get("inputType")
raise StepError(
f"<{found['tag']}{f' type={kind}' if kind else ''}> is not an editable field "
"(disabled, read-only, or not an input)"
)
factor = scale_factor(found, scale, report)
x = found["screen"]["x"] * factor
y = found["screen"]["y"] * factor
screen_w, screen_h = pyautogui.size()
if not (0 <= x < screen_w and 0 <= y < screen_h):
raise StepError(
f"target ({x:.0f}, {y:.0f}) is off-screen ({screen_w}x{screen_h}) — "
"is the Chrome window on another display?"
)
say(f"at {x:.0f},{y:.0f}")
# A click on a background window is consumed activating it and never reaches
# the page, so raise the browser immediately before pressing.
if activate:
# Raise the browser the extension actually measured from, not whichever
# one happens to be first in the list.
focused = focus.ensure_frontmost(browser=found.get("browser"))
say(focused.detail)
if not focused.ok:
raise StepError(
"the browser is not frontmost — the click would be consumed "
"activating its window instead of pressing the element"
)
pyautogui.FAILSAFE = True
if robotic:
pyautogui.moveTo(x, y, duration=0.25)
if not dry_run:
pyautogui.click()
else:
humanize.click(x, y, rng=rng, press=not dry_run)
if action == "click" or dry_run:
return found
# ── type ────────────────────────────────────────────────────────────────
import time
time.sleep(0.15) # let the field take focus and any JS handlers settle
if clear:
humanize.clear_field(rng)
humanize.type_text(text or "", rng)
if not verify:
return found
# Typing into a field that never took focus, or that ignores the input, fails
# silently and is otherwise indistinguishable from success.
after = dash.locate(selector, index, url, timeout, open_url)
check_signed_in(after, signed_out)
got = after.get("value")
if got is None:
say("element exposes no value to verify against")
return found
if (text or "") in got:
say(f"verified, field contains {got!r}")
return after
raise StepError(f"verification failed — field contains {got!r}", code=4)