diff --git a/README.md b/README.md index 7e40f23..c4b3430 100644 --- a/README.md +++ b/README.md @@ -166,6 +166,45 @@ the automation buttons when it is not there. A running Python process does not reload when the source changes, so restart it after pulling updates; the dashboard warns when its version is behind. +### 4. Captchas + +A run that hits a captcha solves it through the **2captcha API**, not by clicking +2captcha's own browser extension. The API key is hardcoded in +`app/api/autobuyer/captcha/route.ts` — to bill a different account, set +`TWOCAPTCHA_API_KEY`, or PATCH `twocaptcha_api_key` to `/api/settings`. Either way +the key stays on the dashboard: the runner asks this machine to solve and never +handles it. + +A `solveCaptcha` step then does three things, with no mouse involved: the +extension reads the widget's sitekey out of the page, the dashboard buys a token +from 2captcha, and the extension puts that token into the page's response field +and fires the page's own callback. reCAPTCHA v2 (including Enterprise), hCaptcha +and Turnstile are detected automatically. + +If the 2captcha *extension* is still installed, disable it or turn off its +auto-solve. Two solvers racing on one widget is a good way to burn balance and +end up with a token the page has already replaced. + +Solving costs money per captcha, and a failed solve is still billed. `timeoutSeconds` +on the step caps how long a run waits; 2captcha usually answers in 10–30s. + +**The solver proxy.** Solves go through 2captcha's residential proxy, so the +challenge is answered from a residential address rather than a datacentre one. +The credentials are hardcoded in `app/api/autobuyer/captcha/route.ts`; override +them without touching code via `TWOCAPTCHA_PROXY`, or by PATCHing +`twocaptcha_proxy` to `/api/settings`. Either accepts `none` to solve proxyless. +Neither has a field on the Settings page — both are hardcoded, so there is +nothing to fill in. The run log names the address that +solved each captcha — the first thing to check when a token comes back and the +site rejects it anyway. + +Two things that proxy does *not* do. It is not the browser's proxy: the token is +solved from the proxy's address but the form is submitted from this machine's, +and nothing makes those match. And the pool is not geo-pinned — despite the +`na.` hostname, test requests exited from Russia and Japan. For reCAPTCHA v2 and +Turnstile that mismatch is usually tolerated; if a site starts rejecting tokens +that 2captcha reports as solved, set the proxy to `none` and compare. + ### Running it Press an automation's button. Progress appears per step, and **Stop** halts a run diff --git a/app/api/autobuyer/captcha/route.ts b/app/api/autobuyer/captcha/route.ts new file mode 100644 index 0000000..7d7f013 --- /dev/null +++ b/app/api/autobuyer/captcha/route.ts @@ -0,0 +1,273 @@ +import { lookup as dnsLookup } from 'node:dns/promises'; +import { NextRequest } from 'next/server'; +import { getSetting } from '@/lib/db'; +import { corsJson, corsPreflight } from '../cors'; + +/** A thin proxy over api.2captcha.com, so the runner never handles the key. + * + * The runner is a desktop process that may not even be on this machine; keeping + * the key here means it lives in exactly one place — the settings table — and + * a captcha solve is one more dashboard call like every other step. + */ +const API_BASE = 'https://api.2captcha.com'; +const UPSTREAM_TIMEOUT_MS = 20_000; + +/** 2captcha's residential proxy, so the solve comes from a residential address + * rather than a datacentre one the site may already score badly. + * + * Hardcoded on request. Note what that means: unlike the sqlite database, this + * file IS in the repo, so these credentials travel with it. Override without + * editing code via TWOCAPTCHA_PROXY, or by PATCHing `twocaptcha_proxy` to + * /api/settings; either accepts `none` to go back to solving proxyless. + * + * Caveat worth knowing before trusting it: the token is scored partly on the + * address that solved it, while the form is submitted from THIS machine's + * address. The two only match if the browser goes out through the same proxy, + * which it does not. For reCAPTCHA v2 and Turnstile that is usually tolerated; + * for anything scoring the IP harder, a mismatched proxy can be worse than + * none. */ +const DEFAULT_PROXY = 'http://u3229993f561005c2-zone-custom:u3229993f561005c2@na.proxy.2captcha.com:2334'; + +interface Proxy { + type: string; + host: string; + port: number; + login: string; + password: string; +} + +/** Settings first, then the environment, then the hardcoded default. `none` + * anywhere in that chain means solve without a proxy. */ +function proxyConfig(): Proxy | null { + const raw = (getSetting('twocaptcha_proxy') || process.env.TWOCAPTCHA_PROXY || DEFAULT_PROXY).trim(); + if (!raw || /^(none|off|proxyless)$/i.test(raw)) return null; + + try { + const url = new URL(/^\w+:\/\//.test(raw) ? raw : `http://${raw}`); + const port = Number(url.port); + if (!url.hostname || !port) return null; + return { + type: url.protocol.replace(':', '').toLowerCase(), + host: url.hostname, + port, + login: decodeURIComponent(url.username), + password: decodeURIComponent(url.password), + }; + } catch { + return null; + } +} + +/** 2captcha rejects host names in `proxyAddress` — it wants a literal IP — so + * `na.proxy.2captcha.com` has to be resolved here. Cached: the gateway address + * is stable, and a DNS lookup per solve would be a needless round trip. */ +const IS_IP = /^(\d{1,3}\.){3}\d{1,3}$|:/; +const dnsCache = new Map(); +const DNS_TTL_MS = 5 * 60_000; + +async function proxyAddress(host: string): Promise { + if (IS_IP.test(host)) return host; + + const hit = dnsCache.get(host); + if (hit && Date.now() - hit.at < DNS_TTL_MS) return hit.ip; + + const { address } = await dnsLookup(host); + dnsCache.set(host, { ip: address, at: Date.now() }); + return address; +} + +type Provider = 'recaptcha' | 'hcaptcha' | 'turnstile'; + +/** The slice of 2captcha's answer we act on. Both endpoints share the shape. */ +interface UpstreamResult { + errorId?: number; + errorCode?: string; + errorDescription?: string; + taskId?: number | string; + status?: string; + solution?: { token?: string; gRecaptchaResponse?: string }; + cost?: string; +} + +function message(err: unknown, fallback: string): string { + return err instanceof Error ? err.message : fallback; +} + +/** The 2captcha account these solves are billed to. + * + * Hardcoded on request, with the same caveat as the proxy below: this file is in + * the repo, so the key travels with it. Anyone who ends up with a copy can spend + * the balance. To bill another account, set TWOCAPTCHA_API_KEY, or PATCH + * `twocaptcha_api_key` to /api/settings — the settings row still wins, it just + * has no field on the Settings page any more. */ +const DEFAULT_API_KEY = 'bde31ec7a337243e7331fbb882111a62'; + +/** Settings first, then the environment, then the hardcoded default. */ +function apiKey(): string { + return (getSetting('twocaptcha_api_key') || process.env.TWOCAPTCHA_API_KEY || DEFAULT_API_KEY).trim(); +} + +/** Build the task 2captcha expects for this widget. + * + * Each type comes in two spellings: `…TaskProxyless` solves from 2captcha's own + * address, `…Task` solves through the proxy named in `proxy`. Same fields + * otherwise, so the only difference is the suffix and the five proxy keys. + */ +function buildTask( + body: Record, + proxy: (Proxy & { address: string }) | null, +): { task?: Record; error?: string } { + const provider = String(body.provider || '') as Provider; + const websiteKey = String(body.sitekey || '').trim(); + const websiteURL = String(body.pageurl || '').trim(); + + if (!websiteKey) return { error: '`sitekey` is required' }; + if (!/^https?:\/\//.test(websiteURL)) return { error: '`pageurl` must be an http(s) URL' }; + + const suffix = proxy ? '' : 'Proxyless'; + const common: Record = { + websiteURL, + websiteKey, + ...(proxy ? { + proxyType: proxy.type, + proxyAddress: proxy.address, + proxyPort: proxy.port, + proxyLogin: proxy.login, + proxyPassword: proxy.password, + } : {}), + // The browser that will submit the form. Sent only alongside a proxy: + // a solve routed through a residential address is more convincing when + // it also claims the same browser, and 2captcha ignores it otherwise. + ...(proxy && body.userAgent ? { userAgent: String(body.userAgent) } : {}), + }; + + if (provider === 'turnstile') { + return { + task: { + type: `TurnstileTask${suffix}`, + ...common, + // Only present when the page rendered the widget programmatically + // with them. Sending empty ones makes 2captcha reject the task. + ...(body.action ? { action: String(body.action) } : {}), + ...(body.data ? { data: String(body.data) } : {}), + ...(body.pagedata ? { pagedata: String(body.pagedata) } : {}), + }, + }; + } + + if (provider === 'hcaptcha') { + return { + task: { + type: `HCaptchaTask${suffix}`, + ...common, + isInvisible: !!body.invisible, + }, + }; + } + + if (provider === 'recaptcha') { + return { + task: { + type: body.enterprise ? `RecaptchaV2EnterpriseTask${suffix}` : `RecaptchaV2Task${suffix}`, + ...common, + isInvisible: !!body.invisible, + }, + }; + } + + return { error: `unknown captcha provider ${provider || '(none)'}` }; +} + +async function callUpstream(path: string, payload: unknown): Promise { + const res = await fetch(`${API_BASE}/${path}`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(payload), + signal: AbortSignal.timeout(UPSTREAM_TIMEOUT_MS), + }); + if (!res.ok) throw new Error(`2captcha ${path} -> HTTP ${res.status}`); + return res.json(); +} + +/** Queue a solve. Returns the 2captcha task id to poll with GET. */ +export async function POST(req: NextRequest) { + const clientKey = apiKey(); + if (!clientKey) { + return corsJson( + { error: 'No 2captcha API key — set TWOCAPTCHA_API_KEY, or `twocaptcha_api_key` via /api/settings' }, + { status: 400 }, + ); + } + + let body: Record; + try { + body = await req.json(); + } catch { + return corsJson({ error: 'Invalid JSON body' }, { status: 400 }); + } + + const proxy = proxyConfig(); + let resolved: (Proxy & { address: string }) | null = null; + if (proxy) { + try { + resolved = { ...proxy, address: await proxyAddress(proxy.host) }; + } catch (err) { + return corsJson( + { error: `cannot resolve proxy host ${proxy.host}: ${message(err, 'DNS lookup failed')}` }, + { status: 502 }, + ); + } + } + + const { task, error } = buildTask(body, resolved); + if (!task) return corsJson({ error }, { status: 400 }); + + try { + const out = await callUpstream('createTask', { clientKey, task, softId: 0 }); + if (out.errorId) { + // A rejected task is 2captcha's verdict, not a broken pipeline — a bad + // key or an empty balance both land here, and the runner reports them + // as the step failure they are. + return corsJson({ error: `${out.errorCode}: ${out.errorDescription}` }, { status: 400 }); + } + return corsJson({ + taskId: out.taskId, + type: task.type, + // Echoed so the run log says which address solved it — the first + // thing to check when a token comes back and the site rejects it. + via: resolved ? `${proxy!.host}:${proxy!.port} (${resolved.address})` : 'no proxy', + }); + } catch (err) { + return corsJson({ error: message(err, 'createTask failed') }, { status: 502 }); + } +} + +/** Poll one task. `processing` means keep waiting; `failed` means this attempt + * is dead and a fresh task is the only way forward. */ +export async function GET(req: NextRequest) { + const clientKey = apiKey(); + if (!clientKey) return corsJson({ error: 'No 2captcha API key configured' }, { status: 400 }); + + const taskId = req.nextUrl.searchParams.get('taskId'); + if (!taskId) return corsJson({ error: '`taskId` is required' }, { status: 400 }); + + try { + const out = await callUpstream('getTaskResult', { clientKey, taskId: Number(taskId) || taskId }); + if (out.errorId) { + return corsJson({ status: 'failed', error: `${out.errorCode}: ${out.errorDescription}` }); + } + if (out.status !== 'ready') return corsJson({ status: 'processing' }); + + // Turnstile and hCaptcha answer with `token`, reCAPTCHA with + // `gRecaptchaResponse`. Same string, two names. + const token = out.solution?.token || out.solution?.gRecaptchaResponse || ''; + if (!token) return corsJson({ status: 'failed', error: 'solved, but the answer carried no token' }); + return corsJson({ status: 'ready', token, cost: out.cost ?? null }); + } catch (err) { + return corsJson({ error: message(err, 'getTaskResult failed') }, { status: 502 }); + } +} + +export async function OPTIONS() { + return corsPreflight(); +} diff --git a/app/api/settings/route.ts b/app/api/settings/route.ts index eeba523..688c81e 100644 --- a/app/api/settings/route.ts +++ b/app/api/settings/route.ts @@ -1,7 +1,7 @@ import { NextRequest, NextResponse } from 'next/server'; import { getSetting, setSetting } from '@/lib/db'; -const VALID_KEYS = ['max_concurrent_accounts', 'tick_interval_seconds', 'master_dashboard_url', 'instance_name', 'trading_hours'] as const; +const VALID_KEYS = ['max_concurrent_accounts', 'tick_interval_seconds', 'master_dashboard_url', 'instance_name', 'trading_hours', 'twocaptcha_api_key', 'twocaptcha_proxy'] as const; type SettingKey = typeof VALID_KEYS[number]; export async function GET() { diff --git a/clicker/captcha.py b/clicker/captcha.py new file mode 100644 index 0000000..d6278d0 --- /dev/null +++ b/clicker/captcha.py @@ -0,0 +1,161 @@ +"""Solving a page's captcha through the 2captcha API. + +The old flow clicked the 2captcha browser extension's own "solve" button and +waited for it to turn green. This replaces it: read the widget's parameters out +of the page, ask 2captcha for a token, put the token back into the page. + +Three round trips, all through the dashboard: + + runner --locate captchaInfo--> extension reads the widget's sitekey + runner --/api/autobuyer/captcha-> dashboard asks 2captcha, polls, returns a token + runner --locate captchaToken--> extension fills the field and fires the callback + +The API key lives on the dashboard, not here — see app/api/autobuyer/captcha. +""" + +import time + +import actions +from clicker import DashboardError + +# 2captcha's own guidance: nothing is ready sooner than this, so the first poll +# is a wasted round trip if it goes out immediately. +FIRST_POLL_SECONDS = 10.0 +POLL_SECONDS = 5.0 + +# A "task expired"/"unsolvable" verdict means this attempt is dead, not that the +# page is unsolvable — a fresh task usually goes through. Bounded so a site that +# genuinely cannot be solved fails the run instead of burning the balance. +MAX_ATTEMPTS = 3 + +# Timeouts on the two page ops. Reading the widget is a plain injection; posting +# the token runs the page's own callback, which can kick off a network request. +PAGE_TIMEOUT = 30.0 + + +def _page_op(dash, step, opts, options: dict, timeout: float) -> dict: + """Run one captcha op in the page. The selector is unused by these ops, but + the locate endpoint requires one, so `body` stands in for it.""" + return dash.locate( + "body", 0, + step.get("urlPattern", "") or opts.url, + timeout, + step.get("openUrl", "") or "", + options=options, + ) + + +def read_widget(dash, step, opts, provider: str, deadline: float, report) -> dict: + """Wait for a captcha to exist on the page, then describe it. + + Polls rather than reading once: the step that opens the modal returns as soon + as the click lands, and the widget's script still has to load and render. + """ + announced = False + last = "no captcha widget found" + + while True: + try: + return _page_op(dash, step, opts, { + "op": "captchaInfo", + "provider": provider, + "scopeSelector": step.get("selector", "") or "", + }, PAGE_TIMEOUT) + except DashboardError as exc: + # Includes "no captcha widget on the page yet", which is the normal + # answer while the widget is still loading. + last = str(exc) + + if time.time() >= deadline: + raise actions.StepError(f"no captcha to solve: {last}") + if not announced: + report("waiting for the captcha widget to render") + announced = True + time.sleep(2.0) + + +def _await_token(dash, task_id, deadline: float, report, cancelled) -> str | None: + """Poll one 2captcha task. Returns the token, None if the task failed and a + retry is worth it, or raises when the run is out of time.""" + time.sleep(min(FIRST_POLL_SECONDS, max(0.0, deadline - time.time()))) + + while True: + if cancelled and cancelled(): + raise actions.StepError("cancelled while waiting for the captcha") + + try: + row = dash._request(f"/api/autobuyer/captcha?taskId={task_id}") + except DashboardError as exc: + # A blip talking to our own dashboard should not throw away a task + # that 2captcha is already working on. + report(f"(poll failed, retrying: {exc})") + row = {"status": "processing"} + + if row.get("status") == "ready": + return row["token"] + if row.get("status") == "failed": + report(f"2captcha gave up on task {task_id}: {row.get('error')}") + return None + + if time.time() >= deadline: + raise actions.StepError(f"2captcha did not return a token in time (task {task_id})") + time.sleep(POLL_SECONDS) + + +def solve(dash, step, opts, report, cancelled=None) -> None: + """Run one `solveCaptcha` step. Raises StepError if the page is never solved.""" + deadline = time.time() + float(step.get("timeoutSeconds", 180)) + provider = step.get("provider") or "auto" + + info = read_widget(dash, step, opts, provider, deadline, report) + sitekey = info.get("sitekey") or "" + report(f"{info.get('provider')} widget, sitekey {sitekey[:16]}… (via {info.get('source')})") + + started = time.time() + token = None + for attempt in range(1, MAX_ATTEMPTS + 1): + if time.time() >= deadline: + break + + queued = dash._request("/api/autobuyer/captcha", "POST", { + "provider": info.get("provider"), + "sitekey": sitekey, + # The page's own URL, not the firm's configured one: 2captcha scores + # the token against the page it was asked for. + "pageurl": info.get("pageurl"), + "action": info.get("action") or "", + "data": info.get("data") or "", + "pagedata": info.get("pagedata") or "", + "invisible": bool(info.get("invisible")), + "enterprise": bool(info.get("enterprise")), + # The browser that will submit the form, so a proxied solve can claim + # the same one rather than 2captcha's default. + "userAgent": info.get("userAgent") or "", + }) + task_id = queued.get("taskId") + report(f"2captcha task {task_id} queued ({queued.get('type')} via {queued.get('via')})" + + (f", attempt {attempt}" if attempt > 1 else "")) + + token = _await_token(dash, task_id, deadline, report, cancelled) + if token: + break + + if not token: + raise actions.StepError("2captcha could not solve the captcha") + + report(f"token in hand after {time.time() - started:.0f}s ({len(token)} chars)") + + delivered = _page_op(dash, step, opts, { + "op": "captchaToken", + "provider": info.get("provider"), + "token": token, + }, PAGE_TIMEOUT) + + called = delivered.get("called") + report(f"filled {delivered.get('filled', 0)} response field(s)" + + (f", fired {called}" if called else "")) + if not called: + # The token is in the form, so a plain submit still works — but a page + # that gates its submit button on the callback will look unsolved, and + # that is worth saying out loud rather than failing three steps later. + report("no captcha callback found — if the next step fails, that is why") diff --git a/clicker/runner.py b/clicker/runner.py index cab2288..fe8cd5d 100644 --- a/clicker/runner.py +++ b/clicker/runner.py @@ -24,6 +24,7 @@ import threading import time import actions +import captcha import focus from clicker import Dashboard, DashboardError, NotFoundError, force_utf8_output @@ -33,7 +34,7 @@ HEARTBEAT_SECONDS = 2.0 # Bumped whenever the step vocabulary or the locate protocol changes. Reported in # the heartbeat so the dashboard can say "restart your runner" instead of letting # a stale process fail on a step type it has never heard of. -VERSION = "0.16.0" +VERSION = "0.17.0" # Shared with the heartbeat thread: whether a run is currently executing. _busy = threading.Event() @@ -119,6 +120,14 @@ def run_one_step(dash: Dashboard, step: dict, opts, rng, report, run_id: int | N announced = True time.sleep(2.0) + elif step["action"] == "solveCaptcha": + # No mouse: the extension reads the widget, the dashboard buys a token + # from 2captcha, and the extension hands it to the page's own callback. + captcha.solve( + dash, step, opts, report, + cancelled=(lambda: run_status(dash, run_id) == "cancelled") if run_id is not None else None, + ) + elif step["action"] == "scrollToLoad": # Lists that load progressively need walking to the bottom before the # steps that act on their items can see everything. diff --git a/extension/README.md b/extension/README.md index ae3b764..ad5decb 100644 --- a/extension/README.md +++ b/extension/README.md @@ -32,7 +32,7 @@ captures the tab you are on. ## Scope -Capture has no switch, so scope is what keeps it honest. Two limits, both +Capture has no switch, so scope is what keeps it honest. Three limits, all enforced by Chrome rather than by this code: 1. **`host_permissions` is deliberately narrow** — the firm sites plus localhost @@ -43,11 +43,20 @@ enforced by Chrome rather than by this code: last-focused window *and* a host_permissions match. A firm tab sitting in the background is not captured, and neither is anything else. Switch away and capture simply stops until you switch back. +3. **One content script, same hosts.** `captcha-hook.js` runs at document_start + on the firm sites only. It wraps `turnstile.render` / `grecaptcha.render` / + `hcaptcha.render` and remembers what the page passed them — the sitekey, and + above all the callback a solved token has to be handed to, which a + programmatically rendered widget puts nowhere else. It reads nothing else and + sends nothing anywhere; the remembered arguments stay in the page's own world + until a `solveCaptcha` step comes to collect them. To automate a different broker, add its pattern to `host_permissions` in `manifest.json` and reload the extension — that alone scopes capture to it. Avoid going back to ``: it would put every tab you visit in reach, and limit -1 is the only thing standing between this and scraping your whole browser. +1 is the only thing standing between this and scraping your whole browser. A new +firm needs its host in *both* lists — `host_permissions` and the `content_scripts` +matches — or captchas on that site will have no callback to fire. ## Notes diff --git a/extension/background.js b/extension/background.js index 0dd4251..ca27199 100644 --- a/extension/background.js +++ b/extension/background.js @@ -342,6 +342,187 @@ async function pageScrollToLoad(selector, containerSelector, maxScrolls, settleM }; } +// ── Captcha: read the widget, then hand a solved token back to the page ───── + +/** Runs in the page's MAIN world. Describes the captcha widget well enough for + * 2captcha to solve it, or reports that there isn't one yet. + * + * Three sources, best first: the render hook (the only one that sees + * programmatically-passed parameters), the widget's own data- attributes, and + * finally the challenge iframe, whose URL carries the sitekey when nothing else + * does. */ +function pageCaptchaInfo(scopeSelector, wanted) { + const root = scopeSelector ? document.querySelector(scopeSelector) : document; + if (!root) return { error: `No element matches ${scopeSelector}` }; + + const KINDS = { + turnstile: { cls: '.cf-turnstile', frame: 'challenges.cloudflare.com' }, + recaptcha: { cls: '.g-recaptcha', frame: '/recaptcha/' }, + hcaptcha: { cls: '.h-captcha', frame: 'hcaptcha.com' }, + }; + const want = (kind) => !wanted || wanted === 'auto' || wanted === kind; + const enterprise = !!document.querySelector('script[src*="recaptcha/enterprise"]'); + + // Cloudflare's iframe has no sitekey parameter — it is a path segment, and + // every Turnstile key starts with 0x. + const fromFrameUrl = (kind, src) => { + try { + const url = new URL(src, location.href); + if (kind === 'recaptcha') return url.searchParams.get('k'); + if (kind === 'hcaptcha') return url.searchParams.get('sitekey'); + return url.pathname.split('/').find((part) => /^0x[A-Za-z0-9_-]{8,}$/.test(part)) || null; + } catch { + return null; + } + }; + + for (const widget of (window.__autofirmerCaptcha?.widgets || [])) { + const params = widget.params || {}; + if (!want(widget.kind) || !params.sitekey) continue; + return { + provider: widget.kind, + sitekey: params.sitekey, + pageurl: location.href, + action: params.action || '', + data: params.cData || '', + pagedata: params.chlPageData || '', + invisible: params.size === 'invisible', + enterprise, + userAgent: navigator.userAgent, + source: 'render hook', + }; + } + + for (const kind of Object.keys(KINDS)) { + if (!want(kind)) continue; + const el = root.querySelector(`${KINDS[kind].cls}[data-sitekey]`); + if (!el) continue; + return { + provider: kind, + sitekey: el.getAttribute('data-sitekey'), + pageurl: location.href, + action: el.getAttribute('data-action') || '', + data: el.getAttribute('data-cdata') || '', + pagedata: '', + invisible: el.getAttribute('data-size') === 'invisible', + enterprise, + userAgent: navigator.userAgent, + source: 'data-sitekey', + }; + } + + const frames = Array.from(root.querySelectorAll('iframe')); + for (const kind of Object.keys(KINDS)) { + if (!want(kind)) continue; + for (const frame of frames) { + const src = frame.getAttribute('src') || ''; + if (!src.includes(KINDS[kind].frame)) continue; + const sitekey = fromFrameUrl(kind, src); + if (!sitekey) continue; + return { + provider: kind, + sitekey, + pageurl: location.href, + action: '', + data: '', + pagedata: '', + invisible: false, + enterprise, + userAgent: navigator.userAgent, + source: 'challenge iframe', + }; + } + } + + return { error: 'No captcha widget on the page yet' }; +} + +/** Runs in the page's MAIN world. Puts a solved token where the page looks for + * it, then tells the page it arrived. + * + * Both halves matter. The hidden response field is what the form posts, but + * most pages enable their submit button from the widget's callback and never + * read the field until then — so filling it alone leaves the page looking + * unsolved. Only the FIRST callback found is fired: they are the page's own + * submit handlers, and calling several would risk sending the form twice. */ +function pageCaptchaSubmit(token, provider) { + const FIELDS = { + turnstile: ['cf-turnstile-response', 'g-recaptcha-response'], + recaptcha: ['g-recaptcha-response'], + hcaptcha: ['h-captcha-response', 'g-recaptcha-response'], + }; + + const targets = new Set(); + for (const name of (FIELDS[provider] || FIELDS.recaptcha)) { + document.querySelectorAll(`[name="${name}"], #${name}`).forEach((el) => targets.add(el)); + } + // Turnstile names its hidden input per widget: cf-chl-widget-_response. + document.querySelectorAll('input[name^="cf-chl-widget-"]').forEach((el) => targets.add(el)); + + let filled = 0; + for (const el of targets) { + el.value = token; + el.dispatchEvent(new Event('input', { bubbles: true })); + el.dispatchEvent(new Event('change', { bubbles: true })); + filled++; + } + + let called = null; + const invoke = (fn, where) => { + if (called || typeof fn !== 'function') return false; + try { + fn(token); + called = where; + return true; + } catch (err) { + // The page's own handler threw. That is its business, but the token + // did reach it, so don't go looking for another one to call. + called = `${where} (threw: ${err.message})`; + return true; + } + }; + + for (const widget of (window.__autofirmerCaptcha?.widgets || [])) { + if (called) break; + if (provider && provider !== 'auto' && widget.kind !== provider) continue; + const cb = (widget.params || {}).callback; + if (typeof cb === 'function') invoke(cb, `${widget.kind} render callback`); + else if (typeof cb === 'string') invoke(window[cb], cb); + } + + if (!called) { + for (const el of document.querySelectorAll('[data-callback]')) { + const name = el.getAttribute('data-callback'); + if (invoke(window[name], `data-callback ${name}`)) break; + } + } + + // reCAPTCHA rendered declaratively keeps its callback in a private config + // object and nowhere else. The keys around it are minified; `callback` is not. + if (!called && window.___grecaptcha_cfg?.clients) { + const seen = new Set(); + const queue = Object.values(window.___grecaptcha_cfg.clients); + let budget = 2000; + while (queue.length && !called && budget-- > 0) { + const node = queue.shift(); + if (!node || typeof node !== 'object' || seen.has(node)) continue; + seen.add(node); + for (const [key, value] of Object.entries(node)) { + if (typeof value === 'function') { + if (/callback/i.test(key) && invoke(value, `grecaptcha ${key}`)) break; + } else if (value && typeof value === 'object') { + queue.push(value); + } + } + } + } + + if (!filled && !called) { + return { error: 'nowhere to put the token — no response field and no callback on the page' }; + } + return { filled, called, provider, url: location.href }; +} + async function resolveLocateTab(cfg, request) { // Every step carries its firm's pattern; the manifest hosts are the fallback // for a bare request (the CLI's locate without --url). @@ -368,6 +549,15 @@ async function resolveLocateTab(cfg, request) { return { tab, opened: false }; } +/** Hand an answer — or a failure — back to whoever queued the request. */ +async function postLocateResult(cfg, id, result, error) { + await fetch(`${cfg.apiBase}/api/autobuyer/locate/result`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ id, result, error }), + }); +} + /** Returns true if a request was handled, false if the queue was empty. */ async function serveLocateRequest(cfg) { const claim = await fetch(`${cfg.apiBase}/api/autobuyer/locate/claim`, { method: 'POST' }); @@ -421,11 +611,36 @@ async function serveLocateRequest(cfg) { if (out.error) throw new Error(out.error); result = { ...out, tabId: tab.id, windowId: tab.windowId, browser: detectBrowser() }; - await fetch(`${cfg.apiBase}/api/autobuyer/locate/result`, { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ id: request.id, result, error: null }), + await postLocateResult(cfg, request.id, result, null); + return true; + } + + // Captcha ops are the same shape: no element is measured, the page is + // read for its widget parameters or handed a token 2captcha solved. + // Both run in the page's MAIN world — the isolated world cannot see the + // page's own `turnstile` / `grecaptcha` objects, nor the render hook's + // registry, and those hold the callbacks a token has to be given to. + // allFrames because the widget is often in a same-origin sub-frame; + // Chrome skips frames outside host_permissions, so the challenge iframe + // itself is never touched. + if (request.options && (request.options.op === 'captchaInfo' || request.options.op === 'captchaToken')) { + const isInfo = request.options.op === 'captchaInfo'; + const injections = await chrome.scripting.executeScript({ + target: { tabId: tab.id, allFrames: true }, + world: 'MAIN', + func: isInfo ? pageCaptchaInfo : pageCaptchaSubmit, + args: isInfo + ? [request.options.scopeSelector || '', request.options.provider || 'auto'] + : [String(request.options.token || ''), request.options.provider || 'auto'], }); + const answers = injections.map((i) => i?.result).filter(Boolean); + const found = answers.find((a) => !a.error); + if (!found) { + throw new Error(answers[0]?.error || 'Captcha injection returned nothing'); + } + result = { ...found, tabId: tab.id, windowId: tab.windowId, browser: detectBrowser() }; + + await postLocateResult(cfg, request.id, result, null); return true; } @@ -457,11 +672,7 @@ async function serveLocateRequest(cfg) { error = err.message; } - await fetch(`${cfg.apiBase}/api/autobuyer/locate/result`, { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ id: request.id, result, error }), - }); + await postLocateResult(cfg, request.id, result, error); return true; } diff --git a/extension/captcha-hook.js b/extension/captcha-hook.js new file mode 100644 index 0000000..a4407f1 --- /dev/null +++ b/extension/captcha-hook.js @@ -0,0 +1,69 @@ +/* AutoFirmer — captcha render hook. + * + * Runs in the page's MAIN world at document_start, before any captcha script + * loads. It wraps `turnstile.render` / `grecaptcha.render` / `hcaptcha.render` + * and records what the page passed to them on `window.__autofirmerCaptcha`. + * + * Why bother, when the sitekey is usually a `data-sitekey` attribute: a widget + * rendered programmatically puts NOTHING in the DOM — no sitekey, and more + * importantly no callback. The callback is the whole point. Filling the hidden + * response field with a solved token does not by itself tell the page a token + * arrived; the page only reacts when its own callback runs with it. Turnstile in + * particular hands that callback straight to render() and never exposes it + * anywhere else, so a hook installed before the widget renders is the only way + * to hold on to it. + * + * Reads nothing and sends nothing. It only remembers arguments, in the page's + * own world, for background.js to use when a token comes back from 2captcha. + */ + +(() => { + const registry = (window.__autofirmerCaptcha = window.__autofirmerCaptcha || { widgets: [] }); + + /** Wrap `render` on a captcha API object, once. */ + function wrap(kind, api) { + if (!api || typeof api !== 'object' || api.__autofirmerWrapped) return api; + + if (typeof api.render === 'function') { + const original = api.render.bind(api); + api.render = function (container, params) { + const widgetId = original(container, params); + // `params` holds the sitekey, the callback, and for Turnstile the + // action / cData / chlPageData that a solve has to be asked for. + registry.widgets.push({ kind, params: params || {}, widgetId }); + return widgetId; + }; + try { + Object.defineProperty(api, '__autofirmerWrapped', { value: true }); + } catch { /* frozen API object — wrapping still took */ } + } + + // reCAPTCHA Enterprise is a second render() hanging off the same object. + if (api.enterprise) wrap(kind, api.enterprise); + return api; + } + + /** Wrap whatever the page eventually assigns to `window[name]`. + * + * The script that defines it loads after this one, so there is nothing to + * wrap yet — the accessor waits for the assignment instead. It also catches + * the reCAPTCHA pattern of replacing a `{ready}` placeholder later on. */ + function intercept(name, kind) { + let current = wrap(kind, window[name]); + try { + Object.defineProperty(window, name, { + configurable: true, + enumerable: true, + get: () => current, + set: (value) => { current = wrap(kind, value); }, + }); + } catch { + // The page made the property non-configurable. Detection falls back + // to the DOM, which covers every declaratively rendered widget. + } + } + + intercept('turnstile', 'turnstile'); + intercept('grecaptcha', 'recaptcha'); + intercept('hcaptcha', 'hcaptcha'); +})(); diff --git a/extension/manifest.json b/extension/manifest.json index 3881022..7639d0c 100644 --- a/extension/manifest.json +++ b/extension/manifest.json @@ -1,7 +1,7 @@ { "manifest_version": 3, "name": "AutoFirmer Capture", - "version": "0.10.0", + "version": "0.11.0", "description": "Scrapes the HTML of the firm tab you are viewing and posts it to the AutoFirmer dashboard.", "permissions": ["scripting", "tabs", "storage", "alarms"], "host_permissions": [ @@ -13,6 +13,18 @@ "background": { "service_worker": "background.js" }, + "content_scripts": [ + { + "matches": [ + "https://*.tradeify.co/*", + "https://*.lucidtrading.com/*" + ], + "js": ["captcha-hook.js"], + "run_at": "document_start", + "all_frames": true, + "world": "MAIN" + } + ], "action": { "default_popup": "popup.html", "default_title": "AutoFirmer Capture" diff --git a/lib/automations.ts b/lib/automations.ts index 94aa0a3..13c334f 100644 --- a/lib/automations.ts +++ b/lib/automations.ts @@ -39,6 +39,28 @@ export type AutomationStep = settleMs?: number; label?: string; } + // Solve the page's captcha through the 2captcha API. + // + // Replaces the old pair of steps that clicked the 2captcha *extension's* own + // "solve" button and waited for it to go green. That needed a second + // extension installed and signed in on the machine, put a real mouse click on + // a button whose position and markup are 2captcha's to change, and gave the + // run no way to tell "still solving" from "the extension is not there". + // + // This reads the widget's parameters out of the page, has the dashboard ask + // 2captcha for a token, and hands the token back to the page's own callback. + // No mouse involved. + // + // `provider` is normally left off: the widget is detected. Set it when a page + // carries more than one kind. `selector` likewise scopes detection to a + // container when several widgets are on the page at once. + | { + action: 'solveCaptcha'; + provider?: 'turnstile' | 'recaptcha' | 'hcaptcha'; + selector?: string; + timeoutSeconds?: number; + label?: string; + } // Run `steps` several times over. `times` fixes the count here; `timesFrom` // takes it from an input the user fills in on the dashboard. The block is // unrolled before the runner ever sees it — see resolveSteps. @@ -142,16 +164,9 @@ export const FIRMS: Firm[] = [ // every site build, so they are not safe to select on. { action: 'click', selector: 'a.add_account_btn', label: 'Open Add Account' }, - { action: 'waitFor', selector: 'div.account_types:nth-child(3) > div[role="radiogroup"] > div > div:nth-child(2)', timeoutSeconds: 30, label: 'Wait for form' }, - { action: 'click', selector: 'div.account_types:nth-child(3) > div[role="radiogroup"] > div > div:nth-child(2)'}, - { action: 'click', selector: 'div.account_types:nth-child(7) span:last-child'}, - { action: 'click', selector: 'div.summary_section div.MuiTextField-root input'}, - { action: 'type', selector: 'div.summary_section div.MuiTextField-root input', text: 'MX8'}, - { action: 'click', selector: 'div.summary_section div.MuiTextField-root + button'}, - { action: 'click', selector: 'div.captcha-solver[data-state="ready"]'}, - { action: 'waitFor', selector: "div.captcha-solver[data-state='solved']", timeoutSeconds: 120, - label: 'Wait for the challenge' }, - { action: 'click', selector: 'div.summary_section > button:last-child', label: 'Open Add Account' }, + + { action: 'solveCaptcha', timeoutSeconds: 180, label: 'Solve the challenge' }, + // TODO: the rest of the purchase flow. Confirm each selector with // `clicker.py locate` before adding it here. @@ -172,9 +187,7 @@ export const FIRMS: Firm[] = [ { action: 'repeat', timesFrom: 'count', steps: [ // one purchase — the steps you already have { action: 'click', selector: '.status-failed button.reset_btn', label: 'Reset Account' }, - { action: 'waitFor', selector: "div.captcha-solver[data-state='ready']", timeoutSeconds: 30, label: 'Wait for the solver' }, - { action: 'click', selector: 'div.captcha-solver[data-state="ready"]'}, - { action: 'waitFor', selector: "div.captcha-solver[data-state='solved']", timeoutSeconds: 120, label: 'Wait for the challenge' }, + { action: 'solveCaptcha', timeoutSeconds: 180, label: 'Solve the challenge' }, { action: 'click', selector: 'button.cancelBtn + button.modalActionBtn' }, { action: 'waitFor', selector: '.reset_acc_modal', absent: true, timeoutSeconds: 30, label: 'Wait for the modal to close' }, @@ -286,6 +299,11 @@ function resolveOne(firm: Firm, step: AutomationStep): ResolvedStep { // Acts on nothing, so no signed-out guard — same treatment as waitFor. return { ...step, urlPattern: firm.urlPattern, openUrl: firm.url }; } + if (step.action === 'solveCaptcha') { + // No guard either, and deliberately: a login form is exactly where a + // captcha shows up, so auth steps have to be able to solve one. + return { ...step, urlPattern: firm.urlPattern, openUrl: firm.url }; + } if (step.action === 'repeat') { // expand() peels these off first; reaching here means a caller bypassed it. throw new Error('repeat steps must be expanded, not resolved directly'); @@ -367,6 +385,7 @@ export function describeStep(step: AutomationStep): string { ? `wait for ${step.selector} to disappear` : `wait for ${step.selector}`; case 'scrollToLoad': return `scroll to load all ${step.selector}`; + case 'solveCaptcha': return `solve the ${step.provider ?? 'page'} captcha with 2captcha`; case 'repeat': { const inner = step.steps.length; const count = step.timesFrom ? `{${step.timesFrom}}` : `${step.times ?? 1}`; diff --git a/lib/db.ts b/lib/db.ts index 1806d06..1ad63f9 100644 --- a/lib/db.ts +++ b/lib/db.ts @@ -185,6 +185,13 @@ seedSetting.run('tick_interval_seconds', '60'); seedSetting.run('master_dashboard_url', 'https://master.juicerroom.com'); seedSetting.run('instance_name', ''); seedSetting.run('trading_hours', 'full_cme'); +// Solves the captchas the AutoBuyer runs into. Blank uses the key hardcoded in +// app/api/autobuyer/captcha; set this (or TWOCAPTCHA_API_KEY) to bill another +// account. +seedSetting.run('twocaptcha_api_key', ''); +// Blank falls back to the proxy hardcoded in app/api/autobuyer/captcha; `none` +// solves without one. +seedSetting.run('twocaptcha_proxy', ''); // Mirrors the in-memory scheduler state so a restart can resume it. seedSetting.run('scheduler_running', '0'); seedSetting.run('scheduler_action', 'Buy'); @@ -620,7 +627,7 @@ export const RUNNER_TIMEOUT_MS = 7000; /** The runner version this server's step vocabulary requires. A running process * doesn't reload when the source changes, so an older one silently fails on * steps it predates — the dashboard warns instead. */ -export const RUNNER_EXPECTED_VERSION = '0.16.0'; +export const RUNNER_EXPECTED_VERSION = '0.17.0'; export interface RunnerHeartbeat { at: number;