feat(scripts): add --from-db re-model path + raise EPC API timeout

- run_modelling_e2e --from-db re-models from already-persisted inputs (reads
  each Property's Effective EPC + planning protections + solar from the DB) and
  skips every live fetcher — zero gov-API calls. With --persist it re-writes the
  Plan and, for lodged-EPC Properties, the Baseline. Self-contained loop; the
  live-fetch path is untouched. Makes local re-runs instant and avoids tripping
  the gov API's per-IP rate limit (6000 req / 5 min) during iteration.
- EpcClientService.REQUEST_TIMEOUT 10s -> 30s: a cold per-UPRN search can exceed
  10s and the old timeout turned it into a timeout-then-retry; 30s rides it out.

Note: an open perf question remains — modelling is fast in isolation (<0.5s/
property) but a long-lived --persist run shows ~1 min/property; suspected in the
persist path (plan.save / baseline) or connection handling, NOT the API. Left
mid-diagnosis for handover.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

infrastructure/epc_client/epc_client_service.py

@@ -18,7 +18,11 @@ from datatypes.epc.search import EpcSearchResult
 
 class EpcClientService:
     BASE_URL = "https://api.get-energy-performance-data.communities.gov.uk"
-    REQUEST_TIMEOUT = 10.0
+    # The gov API's per-UPRN search latency is variable: usually ~0.2s but with
+    # intermittent slow spells. 10s was low enough that a slow spell timed out and
+    # call_with_retry then re-issued it (compounding the cost); 30s rides out the
+    # spell instead. Rate-limit (429) handling stays with call_with_retry.
+    REQUEST_TIMEOUT = 30.0
 
     def __init__(self, auth_token: str) -> None:
         self._headers = {

scripts/run_modelling_e2e.py

@@ -109,6 +109,7 @@ from repositories.geospatial.geospatial_s3_repository import (  # noqa: E402
 from repositories.product.composite_product_repository import (  # noqa: E402
     catalogue_with_off_catalogue_overrides,
 )
+from repositories.product.product_repository import ProductRepository  # noqa: E402
 from repositories.property.override_backed_prediction_attributes_reader import (  # noqa: E402
     OverrideBackedPredictionAttributesReader,
 )
@@ -430,6 +431,161 @@ def _predict_epc(
     return predicted
 
 
+def _run_from_db(
+    args: argparse.Namespace,
+    *,
+    engine: Engine,
+    products: ProductRepository,
+    scenario: Optional[Scenario],
+    considered: Optional[frozenset[MeasureType]],
+    baseline_orchestrator: Optional[PropertyBaselineOrchestrator],
+    md_path: Path,
+    csv_path: Path,
+    candidates_path: Path,
+    target: str,
+    measures_note: str,
+) -> None:
+    """Re-model from already-persisted inputs — **zero gov-API calls**.
+
+    Reads each Property's Effective EPC (lodged-or-predicted, overrides folded),
+    planning protections and solar straight from the DB (a prior ``--persist``
+    ingestion must have stored them), runs the same modelling, and — with
+    ``--persist`` — re-writes the Plan and, for lodged-EPC Properties, the
+    Baseline. A predicted Property has no lodged figures, so it gets no baseline
+    row (same rule as the live path). One bad property is logged and skipped.
+    """
+    md_lines: list[str] = [f"# Modelling recommendations ({target}, {measures_note})\n"]
+    csv_rows: list[str] = [
+        "property_id,uprn,api_sap,baseline_sap,sap_delta,post_sap,measures,"
+        "measure_types,cost_of_works"
+    ]
+    candidate_csv_rows: list[str] = [
+        "property_id,uprn,surface,measure_type,cost_total,contingency_rate,"
+        "selected,description"
+    ]
+    total = len(args.property_ids)
+    run_start = time.monotonic()
+    errors = 0
+    for index, property_id in enumerate(args.property_ids, start=1):
+        elapsed = time.monotonic() - run_start
+        eta = (elapsed / (index - 1)) * (total - index + 1) if index > 1 else 0.0
+        print(
+            f"[{index}/{total}] · {errors} err · elapsed {elapsed / 60:.1f}m "
+            f"· ETA {eta / 60:.1f}m · property {property_id} (from DB)",
+            flush=True,
+        )
+        try:
+            with PostgresUnitOfWork(lambda: Session(engine)) as uow:
+                prop = uow.property.get(property_id)
+                effective_epc: EpcPropertyData = prop.effective_epc
+                restrictions: PlanningRestrictions = prop.planning_restrictions
+                uprn: Optional[int] = prop.identity.uprn
+                epc: Optional[EpcPropertyData] = prop.epc
+                solar_insights: Optional[dict[str, Any]] = (
+                    uow.solar.get(uprn) if uprn is not None else None
+                )
+            predicted = epc is None
+            plan: Plan = run_modelling(
+                effective_epc,
+                goal_band=args.goal,
+                planning_restrictions=restrictions,
+                solar_insights=solar_insights,
+                considered_measures=considered,
+                products=products,
+                scenario=scenario,
+                print_table=False,
+            )
+            candidates: list[Recommendation] = candidate_recommendations(
+                epc if epc is not None else effective_epc,
+                planning_restrictions=restrictions,
+                solar_insights=solar_insights,
+                considered_measures=considered,
+                products=products,
+            )
+            if args.persist:
+                assert scenario is not None  # guaranteed by the --persist guard
+                with PostgresUnitOfWork(lambda: Session(engine)) as uow:
+                    uow.plan.save(
+                        plan,
+                        property_id=property_id,
+                        scenario_id=scenario.id,
+                        portfolio_id=args.portfolio_id,
+                        is_default=scenario.is_default,
+                    )
+                    uow.property.mark_modelled(
+                        property_id, has_recommendations=bool(plan.measures)
+                    )
+                    uow.commit()
+                # Lodged EPC also gets its Baseline Performance re-established from
+                # the persisted EPC; predicted Properties have no lodged figures.
+                if epc is not None:
+                    assert baseline_orchestrator is not None
+                    baseline_orchestrator.run([property_id])
+        except Exception as error:  # noqa: BLE001 — one bad property must not stop the run
+            errors += 1
+            line = f"property {property_id}: ERROR — {type(error).__name__}: {error}"
+            print(line + "\n")
+            md_lines.append(f"## Property {property_id}\n\n`{line}`\n")
+            csv_rows.append(f"{property_id},,,,,,,ERROR,")
+            continue
+
+        measure_types = [m.measure_type for m in plan.measures]
+        selected: set[MeasureType] = {m.measure_type for m in plan.measures}
+        flags = [
+            name
+            for name, on in (
+                ("conservation", restrictions.in_conservation_area),
+                ("listed", restrictions.is_listed),
+                ("heritage", restrictions.is_heritage),
+            )
+            if on
+        ]
+        context = (
+            f"{', '.join(flags) if flags else 'unrestricted'}; "
+            f"{'solar ✓' if solar_insights is not None else 'no solar'}"
+        )
+        source_tag = " · ⚠ PREDICTED (no lodged EPC)" if predicted else ""
+        candidate_lines = _candidate_lines(candidates, selected)
+        print(
+            f"=== Property {property_id} (uprn {uprn}) === "
+            f"SAP {plan.baseline.sap_continuous:.1f} -> {plan.post_sap_continuous:.1f} "
+            f"· {len(plan.measures)} measure(s) · £{plan.cost_of_works:,.0f} "
+            f"· {context}{source_tag}"
+        )
+        print(format_plan_table(plan))
+        md_lines.append(f"## Property {property_id} (uprn {uprn}){source_tag}\n")
+        md_lines.append(
+            f"SAP {plan.baseline.sap_continuous:.1f} → {plan.post_sap_continuous:.1f} "
+            f"· {len(plan.measures)} measure(s) · cost £{plan.cost_of_works:,.0f} "
+            f"· {context}\n"
+        )
+        md_lines.append("**Selected Plan**\n")
+        md_lines.extend(_measure_summary(m) for m in plan.measures)
+        md_lines.append("")
+        md_lines.append("**All candidate measures (cost per measure)**\n")
+        md_lines.extend(candidate_lines)
+        md_lines.append("")
+        api_sap: Optional[int] = epc.energy_rating_current if epc is not None else None
+        calc_sap: float = plan.baseline.sap_continuous
+        api_cell = "" if api_sap is None else str(api_sap)
+        delta_cell = "" if api_sap is None else f"{calc_sap - api_sap:.2f}"
+        csv_rows.append(
+            f"{property_id},{uprn},{api_cell},{calc_sap:.2f},{delta_cell},"
+            f"{plan.post_sap_continuous:.2f},{len(plan.measures)},"
+            f"{'|'.join(measure_types)},{plan.cost_of_works:.0f}"
+        )
+        candidate_csv_rows.extend(
+            _candidate_csv_rows(property_id, uprn, candidates, selected)
+        )
+
+    md_path.write_text("\n".join(md_lines) + "\n", encoding="utf-8")
+    csv_path.write_text("\n".join(csv_rows) + "\n", encoding="utf-8")
+    candidates_path.write_text("\n".join(candidate_csv_rows) + "\n", encoding="utf-8")
+    print(f"wrote {md_path.resolve()}")
+    print(f"wrote {csv_path.resolve()}")
+    print(f"wrote {candidates_path.resolve()}")
+
+
 def main() -> None:
     parser = argparse.ArgumentParser(description=__doc__)
     parser.add_argument(
@@ -478,6 +634,16 @@ def main() -> None:
         "(parent dirs created) instead of ./modelling_e2e.*; lets batched runs "
         "keep separate, durable output files",
     )
+    parser.add_argument(
+        "--from-db",
+        action="store_true",
+        default=False,
+        help="re-model from already-persisted inputs: read each Property's "
+        "Effective EPC + planning protections + solar from the DB and skip the "
+        "live EPC/spatial/solar fetch entirely (zero gov-API calls). Requires a "
+        "prior --persist ingestion run; with --persist it re-writes the Plan "
+        "(and Baseline for lodged-EPC Properties) without re-fetching.",
+    )
     args = parser.parse_args()
 
     if args.persist and (args.scenario_id is None or args.portfolio_id is None):
@@ -563,11 +729,31 @@ def main() -> None:
     )
     measures_note = ",".join(sorted(considered)) if considered else "all measures"
     mode = "PERSISTING to DB" if args.persist else "no DB writes"
+    source = "persisted DB inputs" if args.from_db else "live EPC/solar"
     print(
         f"modelling {len(args.property_ids)} propertie(s) · {target} · {measures_note} · "
-        f"{mode} (DB material catalogue, live EPC/solar)...\n"
+        f"{mode} (DB material catalogue, {source})...\n"
     )
 
+    if args.from_db:
+        # Read inputs from the DB and skip every live fetcher (no gov-API calls).
+        # Self-contained loop + file writing; the live path below is left as-is.
+        _run_from_db(
+            args,
+            engine=engine,
+            products=products,
+            scenario=scenario,
+            considered=considered,
+            baseline_orchestrator=baseline_orchestrator,
+            md_path=md_path,
+            csv_path=csv_path,
+            candidates_path=candidates_path,
+            target=target,
+            measures_note=measures_note,
+        )
+        catalogue_session.close()
+        return
+
     md_lines: list[str] = [f"# Modelling recommendations ({target}, {measures_note})\n"]
     csv_rows: list[str] = [
         "property_id,uprn,api_sap,baseline_sap,sap_delta,post_sap,measures,"