Model/scripts/hyde/elmhurst_explore.py

"""Dump the real DOM of an Elmhurst RdSAP-10-Online page (dev tool).

The entry tool is an ASP.NET WebForms app: field IDs are server-generated
(``ctl00_ctl00_ContentBody_...``) and many controls postback on change. We
can't guess selectors, so this reads them off the live page. Reuses the saved
persistent session (no login needed). Run it on the viewer display:

    DISPLAY=:99 python scripts/hyde/elmhurst_explore.py "<assessment-url>"

Writes, next to this file:
  * elmhurst_dom/<page>.png   — screenshot (confirms we're authenticated)
  * elmhurst_dom/<page>.json  — every input/select/textarea + the left-nav links

It navigates, dumps, and exits (releasing the profile lock for the next step).
"""

from __future__ import annotations

import json
import re
import sys
from pathlib import Path
from typing import Any

from playwright.sync_api import sync_playwright

HERE = Path(__file__).parent
SESSION_DIR = HERE / ".elmhurst-session"
OUT_DIR = HERE / "elmhurst_dom"

# Runs in the page to harvest every form control + the section nav, with the
# label text an assessor reads — so we can tie markdown fields to server IDs.
_DUMP_JS = r"""
() => {
  const labelFor = (el) => {
    if (el.id) {
      const l = document.querySelector(`label[for="${el.id}"]`);
      if (l) return l.innerText.trim();
    }
    // WebForms often puts the caption in a sibling cell, not a <label>.
    const row = el.closest('tr');
    if (row) {
      const firstCell = row.querySelector('td, th');
      if (firstCell && !firstCell.contains(el)) return firstCell.innerText.trim();
    }
    return '';
  };
  const ctrl = (el) => {
    const base = {
      tag: el.tagName.toLowerCase(),
      type: el.type || null,
      id: el.id || null,
      name: el.getAttribute('name') || null,
      label: labelFor(el),
      value: el.value ?? null,
      disabled: el.disabled || false,
      autopostback: /__doPostBack/.test(el.getAttribute('onchange') || ''),
    };
    if (el.tagName.toLowerCase() === 'select') {
      base.options = Array.from(el.options).map(o => ({
        value: o.value, text: o.text.trim(), selected: o.selected,
      }));
    }
    if (el.type === 'checkbox' || el.type === 'radio') base.checked = el.checked;
    return base;
  };
  const controls = Array.from(
    document.querySelectorAll('input, select, textarea')
  ).filter(el => el.type !== 'hidden').map(ctrl);

  // Capture every link/clickable with its RAW href + onclick so we can see
  // how page nav works (URL vs __doPostBack) and the real .aspx page names.
  const nav = Array.from(
    document.querySelectorAll('a, input[type=button], input[type=submit]')
  ).map(a => ({
    text: (a.innerText || a.value || '').trim(),
    href: a.getAttribute('href') || null,
    onclick: a.getAttribute('onclick') || null,
    id: a.id || null,
  })).filter(a => a.text || a.href || a.onclick);

  return { url: location.href, title: document.title, controls, nav };
}
"""


def _slug(url: str) -> str:
    m = re.search(r"WebForm(\w+)\.aspx", url)
    return (m.group(1) if m else "page").lower()


def explore(url: str, click_id: str | None = None) -> dict[str, Any]:
    OUT_DIR.mkdir(parents=True, exist_ok=True)
    with sync_playwright() as p:
        context = p.chromium.launch_persistent_context(
            user_data_dir=str(SESSION_DIR),
            headless=False,
            accept_downloads=True,
            viewport={"width": 1400, "height": 1000},
        )
        page = context.pages[0] if context.pages else context.new_page()
        page.goto(url, wait_until="networkidle", timeout=60_000)
        if click_id:
            # Left-rail nav is a full-form __doPostBack: it navigates to
            # rdsap10.elmhurstenergy.co.uk/Processing/WebForm<Section>.aspx
            # (no Guid — the assessment is bound to the server session). Must
            # wait for the navigation, not just networkidle.
            with page.expect_navigation(wait_until="networkidle", timeout=60_000):
                page.click(f"#{click_id}", timeout=15_000)
        slug = _slug(page.url)
        if click_id:
            slug = click_id.split("buttonAction")[-1].split("_")[0].lower() or slug
        shot = OUT_DIR / f"{slug}.png"
        page.screenshot(path=str(shot))
        dump: dict[str, Any] = page.evaluate(_DUMP_JS)
        (OUT_DIR / f"{slug}.json").write_text(json.dumps(dump, indent=2))
        context.close()

    controls = dump["controls"]
    print(f"landed on : {dump['url']}")
    print(f"title     : {dump['title']}")
    print(f"screenshot: {shot}")
    print(f"controls  : {len(controls)} (json: {OUT_DIR / (slug + '.json')})")
    if "Login" in dump["title"] or "login" in dump["url"].lower():
        print("!! looks like a LOGIN page — session may have expired; re-run login.")
    return dump


# Sections that aren't data-entry (or are terminal) — skip when dumping all.
_SKIP_SECTIONS = {
    "EPCAddress", "Recommendations", "Addenda", "TechnicalAdvice",
    "Results", "Summary",
}


def explore_all(url: str) -> None:
    """Enter once and click through every data section, dumping each. Read-only
    (we don't change fields), though navigating re-saves the current page."""
    OUT_DIR.mkdir(parents=True, exist_ok=True)
    with sync_playwright() as p:
        context = p.chromium.launch_persistent_context(
            user_data_dir=str(SESSION_DIR),
            headless=False,
            accept_downloads=True,
            viewport={"width": 1400, "height": 1000},
        )
        page = context.pages[0] if context.pages else context.new_page()
        page.goto(url, wait_until="networkidle", timeout=60_000)

        # Discover the section nav links in rail order.
        links: list[dict[str, str]] = page.eval_on_selector_all(
            "a[id^='ContentBody_buttonAction']",
            "els => els.map(e => ({id: e.id, text: e.innerText.trim()}))",
        )
        targets = [
            (re.sub(r"ContentBody_buttonAction(\w+)_Link", r"\1", lk["id"]), lk["id"])
            for lk in links
            if re.sub(r"ContentBody_buttonAction(\w+)_Link", r"\1", lk["id"])
            not in _SKIP_SECTIONS
        ]
        print(f"dumping {len(targets)} sections: {[t[0] for t in targets]}", flush=True)

        for name, link_id in targets:
            try:
                with page.expect_navigation(wait_until="networkidle", timeout=60_000):
                    page.click(f"#{link_id}", timeout=15_000)
                page.wait_for_timeout(500)
                slug = name.lower()
                page.screenshot(path=str(OUT_DIR / f"{slug}.png"))
                dump: dict[str, Any] = page.evaluate(_DUMP_JS)
                (OUT_DIR / f"{slug}.json").write_text(json.dumps(dump, indent=2))
                n = len([c for c in dump["controls"] if c.get("type") != "hidden"])
                print(f"  {name:22s} -> {n:3d} controls", flush=True)
            except Exception as exc:  # keep going; one bad page shouldn't abort
                print(f"  {name:22s} -> FAILED: {repr(exc)[:80]}", flush=True)
        context.close()


if __name__ == "__main__":
    if len(sys.argv) < 2:
        print("usage: elmhurst_explore.py <assessment-url> [nav-link-id | --all]")
        raise SystemExit(2)
    if sys.argv[-1] == "--all":
        explore_all(sys.argv[1])
    else:
        explore(sys.argv[1], sys.argv[2] if len(sys.argv) > 2 else None)