From efaff228ac8b63f2e2c226ed6595d0f6525bd954 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Tue, 23 Jun 2026 11:05:06 +0000 Subject: [PATCH] feat(scripts): add --from-db re-model path + raise EPC API timeout MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - run_modelling_e2e --from-db re-models from already-persisted inputs (reads each Property's Effective EPC + planning protections + solar from the DB) and skips every live fetcher — zero gov-API calls. With --persist it re-writes the Plan and, for lodged-EPC Properties, the Baseline. Self-contained loop; the live-fetch path is untouched. Makes local re-runs instant and avoids tripping the gov API's per-IP rate limit (6000 req / 5 min) during iteration. - EpcClientService.REQUEST_TIMEOUT 10s -> 30s: a cold per-UPRN search can exceed 10s and the old timeout turned it into a timeout-then-retry; 30s rides it out. Note: an open perf question remains — modelling is fast in isolation (<0.5s/ property) but a long-lived --persist run shows ~1 min/property; suspected in the persist path (plan.save / baseline) or connection handling, NOT the API. Left mid-diagnosis for handover. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../epc_client/epc_client_service.py | 6 +- scripts/run_modelling_e2e.py | 188 +++++++++++++++++- 2 files changed, 192 insertions(+), 2 deletions(-) diff --git a/infrastructure/epc_client/epc_client_service.py b/infrastructure/epc_client/epc_client_service.py index 16cd4d2f..fe9d130f 100644 --- a/infrastructure/epc_client/epc_client_service.py +++ b/infrastructure/epc_client/epc_client_service.py @@ -18,7 +18,11 @@ from datatypes.epc.search import EpcSearchResult class EpcClientService: BASE_URL = "https://api.get-energy-performance-data.communities.gov.uk" - REQUEST_TIMEOUT = 10.0 + # The gov API's per-UPRN search latency is variable: usually ~0.2s but with + # intermittent slow spells. 10s was low enough that a slow spell timed out and + # call_with_retry then re-issued it (compounding the cost); 30s rides out the + # spell instead. Rate-limit (429) handling stays with call_with_retry. + REQUEST_TIMEOUT = 30.0 def __init__(self, auth_token: str) -> None: self._headers = { diff --git a/scripts/run_modelling_e2e.py b/scripts/run_modelling_e2e.py index c99e023b..6625aa49 100644 --- a/scripts/run_modelling_e2e.py +++ b/scripts/run_modelling_e2e.py @@ -109,6 +109,7 @@ from repositories.geospatial.geospatial_s3_repository import ( # noqa: E402 from repositories.product.composite_product_repository import ( # noqa: E402 catalogue_with_off_catalogue_overrides, ) +from repositories.product.product_repository import ProductRepository # noqa: E402 from repositories.property.override_backed_prediction_attributes_reader import ( # noqa: E402 OverrideBackedPredictionAttributesReader, ) @@ -430,6 +431,161 @@ def _predict_epc( return predicted +def _run_from_db( + args: argparse.Namespace, + *, + engine: Engine, + products: ProductRepository, + scenario: Optional[Scenario], + considered: Optional[frozenset[MeasureType]], + baseline_orchestrator: Optional[PropertyBaselineOrchestrator], + md_path: Path, + csv_path: Path, + candidates_path: Path, + target: str, + measures_note: str, +) -> None: + """Re-model from already-persisted inputs — **zero gov-API calls**. + + Reads each Property's Effective EPC (lodged-or-predicted, overrides folded), + planning protections and solar straight from the DB (a prior ``--persist`` + ingestion must have stored them), runs the same modelling, and — with + ``--persist`` — re-writes the Plan and, for lodged-EPC Properties, the + Baseline. A predicted Property has no lodged figures, so it gets no baseline + row (same rule as the live path). One bad property is logged and skipped. + """ + md_lines: list[str] = [f"# Modelling recommendations ({target}, {measures_note})\n"] + csv_rows: list[str] = [ + "property_id,uprn,api_sap,baseline_sap,sap_delta,post_sap,measures," + "measure_types,cost_of_works" + ] + candidate_csv_rows: list[str] = [ + "property_id,uprn,surface,measure_type,cost_total,contingency_rate," + "selected,description" + ] + total = len(args.property_ids) + run_start = time.monotonic() + errors = 0 + for index, property_id in enumerate(args.property_ids, start=1): + elapsed = time.monotonic() - run_start + eta = (elapsed / (index - 1)) * (total - index + 1) if index > 1 else 0.0 + print( + f"[{index}/{total}] · {errors} err · elapsed {elapsed / 60:.1f}m " + f"· ETA {eta / 60:.1f}m · property {property_id} (from DB)", + flush=True, + ) + try: + with PostgresUnitOfWork(lambda: Session(engine)) as uow: + prop = uow.property.get(property_id) + effective_epc: EpcPropertyData = prop.effective_epc + restrictions: PlanningRestrictions = prop.planning_restrictions + uprn: Optional[int] = prop.identity.uprn + epc: Optional[EpcPropertyData] = prop.epc + solar_insights: Optional[dict[str, Any]] = ( + uow.solar.get(uprn) if uprn is not None else None + ) + predicted = epc is None + plan: Plan = run_modelling( + effective_epc, + goal_band=args.goal, + planning_restrictions=restrictions, + solar_insights=solar_insights, + considered_measures=considered, + products=products, + scenario=scenario, + print_table=False, + ) + candidates: list[Recommendation] = candidate_recommendations( + epc if epc is not None else effective_epc, + planning_restrictions=restrictions, + solar_insights=solar_insights, + considered_measures=considered, + products=products, + ) + if args.persist: + assert scenario is not None # guaranteed by the --persist guard + with PostgresUnitOfWork(lambda: Session(engine)) as uow: + uow.plan.save( + plan, + property_id=property_id, + scenario_id=scenario.id, + portfolio_id=args.portfolio_id, + is_default=scenario.is_default, + ) + uow.property.mark_modelled( + property_id, has_recommendations=bool(plan.measures) + ) + uow.commit() + # Lodged EPC also gets its Baseline Performance re-established from + # the persisted EPC; predicted Properties have no lodged figures. + if epc is not None: + assert baseline_orchestrator is not None + baseline_orchestrator.run([property_id]) + except Exception as error: # noqa: BLE001 — one bad property must not stop the run + errors += 1 + line = f"property {property_id}: ERROR — {type(error).__name__}: {error}" + print(line + "\n") + md_lines.append(f"## Property {property_id}\n\n`{line}`\n") + csv_rows.append(f"{property_id},,,,,,,ERROR,") + continue + + measure_types = [m.measure_type for m in plan.measures] + selected: set[MeasureType] = {m.measure_type for m in plan.measures} + flags = [ + name + for name, on in ( + ("conservation", restrictions.in_conservation_area), + ("listed", restrictions.is_listed), + ("heritage", restrictions.is_heritage), + ) + if on + ] + context = ( + f"{', '.join(flags) if flags else 'unrestricted'}; " + f"{'solar ✓' if solar_insights is not None else 'no solar'}" + ) + source_tag = " · ⚠ PREDICTED (no lodged EPC)" if predicted else "" + candidate_lines = _candidate_lines(candidates, selected) + print( + f"=== Property {property_id} (uprn {uprn}) === " + f"SAP {plan.baseline.sap_continuous:.1f} -> {plan.post_sap_continuous:.1f} " + f"· {len(plan.measures)} measure(s) · £{plan.cost_of_works:,.0f} " + f"· {context}{source_tag}" + ) + print(format_plan_table(plan)) + md_lines.append(f"## Property {property_id} (uprn {uprn}){source_tag}\n") + md_lines.append( + f"SAP {plan.baseline.sap_continuous:.1f} → {plan.post_sap_continuous:.1f} " + f"· {len(plan.measures)} measure(s) · cost £{plan.cost_of_works:,.0f} " + f"· {context}\n" + ) + md_lines.append("**Selected Plan**\n") + md_lines.extend(_measure_summary(m) for m in plan.measures) + md_lines.append("") + md_lines.append("**All candidate measures (cost per measure)**\n") + md_lines.extend(candidate_lines) + md_lines.append("") + api_sap: Optional[int] = epc.energy_rating_current if epc is not None else None + calc_sap: float = plan.baseline.sap_continuous + api_cell = "" if api_sap is None else str(api_sap) + delta_cell = "" if api_sap is None else f"{calc_sap - api_sap:.2f}" + csv_rows.append( + f"{property_id},{uprn},{api_cell},{calc_sap:.2f},{delta_cell}," + f"{plan.post_sap_continuous:.2f},{len(plan.measures)}," + f"{'|'.join(measure_types)},{plan.cost_of_works:.0f}" + ) + candidate_csv_rows.extend( + _candidate_csv_rows(property_id, uprn, candidates, selected) + ) + + md_path.write_text("\n".join(md_lines) + "\n", encoding="utf-8") + csv_path.write_text("\n".join(csv_rows) + "\n", encoding="utf-8") + candidates_path.write_text("\n".join(candidate_csv_rows) + "\n", encoding="utf-8") + print(f"wrote {md_path.resolve()}") + print(f"wrote {csv_path.resolve()}") + print(f"wrote {candidates_path.resolve()}") + + def main() -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( @@ -478,6 +634,16 @@ def main() -> None: "(parent dirs created) instead of ./modelling_e2e.*; lets batched runs " "keep separate, durable output files", ) + parser.add_argument( + "--from-db", + action="store_true", + default=False, + help="re-model from already-persisted inputs: read each Property's " + "Effective EPC + planning protections + solar from the DB and skip the " + "live EPC/spatial/solar fetch entirely (zero gov-API calls). Requires a " + "prior --persist ingestion run; with --persist it re-writes the Plan " + "(and Baseline for lodged-EPC Properties) without re-fetching.", + ) args = parser.parse_args() if args.persist and (args.scenario_id is None or args.portfolio_id is None): @@ -563,11 +729,31 @@ def main() -> None: ) measures_note = ",".join(sorted(considered)) if considered else "all measures" mode = "PERSISTING to DB" if args.persist else "no DB writes" + source = "persisted DB inputs" if args.from_db else "live EPC/solar" print( f"modelling {len(args.property_ids)} propertie(s) · {target} · {measures_note} · " - f"{mode} (DB material catalogue, live EPC/solar)...\n" + f"{mode} (DB material catalogue, {source})...\n" ) + if args.from_db: + # Read inputs from the DB and skip every live fetcher (no gov-API calls). + # Self-contained loop + file writing; the live path below is left as-is. + _run_from_db( + args, + engine=engine, + products=products, + scenario=scenario, + considered=considered, + baseline_orchestrator=baseline_orchestrator, + md_path=md_path, + csv_path=csv_path, + candidates_path=candidates_path, + target=target, + measures_note=measures_note, + ) + catalogue_session.close() + return + md_lines: list[str] = [f"# Modelling recommendations ({target}, {measures_note})\n"] csv_rows: list[str] = [ "property_id,uprn,api_sap,baseline_sap,sap_delta,post_sap,measures,"