senofy/autofirmer-expanded
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add frame-aware locate for cross-origin iframe steps

Browse Source 
Selectors were only resolved against the top document, so steps targeting
the Lucid WooCommerce checkout (rendered in a lucidtrading.com iframe under
the dash.lucidtrading.com app) never matched — e.g. input#coupon_code_field.

Add an optional `frame` on click/type/waitFor naming the <iframe> element.
The extension measures the iframe in the top frame (pageFrameOffset), finds
the element across child frames (pageLocate `relative` mode), and composes
the two into desktop coordinates; the Python side is unchanged since the
result keeps the same shape. Without `frame`, every path behaves as before.

Point the Lucid checkout steps at the frame and harden the coupon selectors
(the exact field id is still unconfirmed against the live iframe).

Extension 0.13.0, runner 0.19.0.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Brandon Li
2026-09-02 13:22:10 -05:00
co-authored by Claude Opus 4.8
parent 016f121f40
commit 66c99c5499
5 changed files with 197 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files
extension/background.js
+155 -4
View File
@@ -188,8 +188,14 @@ function detectBrowser() {
// ── Locate: turn a CSS selector into desktop coordinates ────────────────────
/** Runs in the page. Scrolls the element into view, then reports where it ended up. */
function pageLocate(selector, index) {
/** Runs in the page. Scrolls the element into view, then reports where it ended up.
* `relative` is set when this runs inside a child frame: the frame's own
* window.screenX and getBoundingClientRect are relative to that frame, not the
* OS screen, so the desktop coordinate cannot be computed here — a cross-origin
* child cannot read where it sits inside its parent. In that mode it returns the
* element's position within THIS frame's viewport and leaves the composition to
* the background script, which adds the parent <iframe>'s offset (pageFrameOffset). */
function pageLocate(selector, index, relative) {
const matches = document.querySelectorAll(selector);
if (matches.length === 0) return { error: `No element matches ${selector}` };
const el = matches[index];
@@ -234,7 +240,7 @@ function pageLocate(selector, index) {
const isField = el.tagName === 'INPUT' || el.tagName === 'TEXTAREA';
const value = isField ? el.value : (el.isContentEditable ? el.innerText : null);
return {
const common = {
tag: el.tagName.toLowerCase(),
text: (el.textContent || '').trim().slice(0, 80),
matchCount: matches.length,
@@ -244,6 +250,18 @@ function pageLocate(selector, index) {
covered: !!covered,
coveredBy: covered ? describeNode(atPoint) : null,
viewport: { x: r.left, y: r.top, width: r.width, height: r.height },
url: location.href,
};
// Inside a child frame the desktop coordinate isn't ours to compute — hand
// back the element's centre within this frame's viewport and let the caller
// add the parent <iframe>'s on-screen offset.
if (relative) {
return { ...common, frameCenter: { x: cx, y: cy } };
}
return {
...common,
// Desktop coordinates of the element's centre, in CSS pixels.
screen: { x: window.screenX + cx, y: window.screenY + chromeHeight + cy },
// Lets the Python side work out whether the OS uses a different pixel
@@ -251,7 +269,36 @@ function pageLocate(selector, index) {
screenSize: { width: window.screen.width, height: window.screen.height },
devicePixelRatio: window.devicePixelRatio,
chromeHeight,
url: location.href,
};
}
/** Runs in the TOP frame. Measures where a child <iframe> sits and reports the
* top window's screen origin, so the background script can turn a frame-relative
* element position (pageLocate's `relative` mode) into desktop coordinates. This
* cannot be done from inside the child: a cross-origin frame can read neither its
* own offset within the parent nor the top window's screenX/screenY. */
function pageFrameOffset(frameSelector) {
const frame = document.querySelector(frameSelector);
if (!frame) return { error: `No element matches ${frameSelector}` };
if (frame.tagName !== 'IFRAME') {
return { error: `${frameSelector} is a <${frame.tagName.toLowerCase()}>, not an <iframe>` };
}
frame.scrollIntoView({ block: 'center', inline: 'center', behavior: 'instant' });
const r = frame.getBoundingClientRect();
if (r.width === 0 || r.height === 0) {
return { error: `Frame ${frameSelector} has zero size (hidden or display:none)` };
}
return {
rect: { left: r.left, top: r.top, width: r.width, height: r.height },
src: frame.getAttribute('src') || frame.src || '',
name: frame.getAttribute('name') || '',
screenX: window.screenX,
screenY: window.screenY,
chromeHeight: window.outerHeight - window.innerHeight,
screenSize: { width: window.screen.width, height: window.screen.height },
devicePixelRatio: window.devicePixelRatio,
innerWidth: window.innerWidth,
innerHeight: window.innerHeight,
};
}
@@ -758,6 +805,110 @@ async function serveLocateRequest(cfg) {
return true;
}
// A step can target an element inside a cross-origin <iframe> — the
// WooCommerce checkout modal is one. A selector run against the top
// document never matches it, so resolve it in two parts: measure where the
// <iframe> sits (top frame, pageFrameOffset) and find the element inside
// whichever child frame holds it (all frames, pageLocate in `relative`
// mode), then add the two together to reach desktop coordinates.
if (request.options && request.options.frame) {
const frameSel = request.options.frame;
// A frame-scoped visibility probe: no coordinates needed, just report
// whether the element is there and rendered in some child frame.
if (request.options.op === 'visible') {
const injections = await chrome.scripting.executeScript({
target: { tabId: tab.id, allFrames: true },
func: pageVisible,
args: [request.selector, request.index || 0],
});
const hits = injections.filter((i) => i.frameId !== 0 && i.result);
const present = hits.find((i) => i.result.present);
const chosen = present ? present.result
: (hits[0]?.result
|| { op: 'visible', present: false, visible: false, matchCount: 0, reason: 'no subframe matched' });
result = { ...chosen, tabId: tab.id, windowId: tab.windowId, browser: detectBrowser() };
await postLocateResult(cfg, request.id, result, null);
return true;
}
// Where the iframe sits, and the top window's screen origin.
const [offsetInj] = await chrome.scripting.executeScript({
target: { tabId: tab.id },
func: pageFrameOffset,
args: [frameSel],
});
const offset = offsetInj?.result;
if (!offset) throw new Error('Frame-offset injection returned nothing');
if (offset.error) throw new Error(offset.error);
// The element, measured inside whatever child frame holds it. Retry
// while the checkout iframe is still mounting — same budget as the
// top-frame path below.
const frameDeadline = Date.now() + (opened ? 8000 : 2500);
let hit;
for (;;) {
const injections = await chrome.scripting.executeScript({
target: { tabId: tab.id, allFrames: true },
func: pageLocate,
args: [request.selector, request.index || 0, true],
});
// Only child frames; the top frame is handled by the normal path.
const hits = injections.filter((i) => i.frameId !== 0 && i.result && !i.result.error);
if (hits.length === 1) { hit = hits[0].result; break; }
if (hits.length > 1) {
throw new Error(
`Selector ${request.selector} matched in ${hits.length} subframes `
+ `(${hits.map((h) => h.result.url).join(', ')}) — narrow it`);
}
// Phrased with "No element matches" so the runner treats it as
// absent (and skipIfNotFound can skip it), same as the top frame.
if (Date.now() >= frameDeadline) {
throw new Error(`No element matches ${request.selector} in frame ${frameSel}`);
}
await new Promise((r) => setTimeout(r, 350));
}
// Compose: the element's centre in the top viewport is where the
// iframe is plus where the element is inside it. From there the screen
// origin and browser-chrome height (both the top window's) give desktop
// pixels, exactly as pageLocate does for a top-level element.
const tx = offset.rect.left + hit.frameCenter.x;
const ty = offset.rect.top + hit.frameCenter.y;
if (tx < 0 || ty < 0 || tx >= offset.innerWidth || ty >= offset.innerHeight) {
throw new Error(
`Element ${request.selector} resolves outside the viewport `
+ `(frame ${frameSel} is partly off-screen)`);
}
result = {
tag: hit.tag,
text: hit.text,
matchCount: hit.matchCount,
value: hit.value,
editable: hit.editable,
inputType: hit.inputType,
covered: hit.covered,
coveredBy: hit.coveredBy,
url: hit.url,
viewport: {
x: offset.rect.left + hit.viewport.x,
y: offset.rect.top + hit.viewport.y,
width: hit.viewport.width,
height: hit.viewport.height,
},
screen: { x: offset.screenX + tx, y: offset.screenY + offset.chromeHeight + ty },
screenSize: offset.screenSize,
devicePixelRatio: offset.devicePixelRatio,
chromeHeight: offset.chromeHeight,
tabId: tab.id,
windowId: tab.windowId,
openedTab: opened,
browser: detectBrowser(),
};
await postLocateResult(cfg, request.id, result, null);
return true;
}
// A visibility probe is a read, not a lookup: it reports on the element
// instead of failing when there isn't one, so it never retries and never
// errors on a missing selector — being missing is the answer.