mirror of
https://github.com/Hestia-Homes/Model.git
synced 2026-07-12 13:29:04 +00:00
101 lines
3.9 KiB
Python
101 lines
3.9 KiB
Python
"""Harvest raw EPC certificates into a JSONL corpus for mapper tests.
|
|
|
|
Source: the bulk EPC dumps in downloads/certificates-YYYY.json. Each line is
|
|
|
|
{"certificate_number": "...", "document": "<json string>", ...}
|
|
|
|
where ``document`` is the cert in the exact shape
|
|
``EpcClientService._fetch_certificate`` returns and
|
|
``EpcPropertyDataMapper.from_api_response`` consumes (it has ``schema_type``,
|
|
``roofs``, ``walls`` ... and matches the committed json_samples).
|
|
|
|
We want a balanced sample per schema so we can build out and regression-test
|
|
the mappers (notably the incomplete ``RdSapSchema20.0.0``). Schema version
|
|
tracks the dump year, so we read each target schema from a year that's rich in
|
|
it and stop once its cap is full — no need to stream whole multi-GB files.
|
|
|
|
Year -> dominant schema (see downloads/README.txt):
|
|
2026 -> RdSAP-Schema-21.0.1
|
|
2021-2024 -> RdSAP-Schema-20.0.0
|
|
|
|
SAP-Schema-18.0.0 is a minority schema (~12% of the 2021 dump) but each year
|
|
holds ~1.6M lines, so 2021 still yields well over 1000 — it just scans deeper
|
|
before the cap fills. SAP-Schema-17.1 is richest in the 2019 dump (~20%).
|
|
|
|
21.0.0 is skipped — it's effectively absent from these dumps.
|
|
|
|
Run cell by cell. No API token needed — this is pure local streaming.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from pathlib import Path
|
|
|
|
import pandas as pd
|
|
|
|
DOWNLOADS = Path("downloads")
|
|
SAMPLES = Path("backend/epc_api/json_samples")
|
|
|
|
# One corpus per schema, written into that schema's own json_samples folder
|
|
# (alongside its epc.json) as corpus.jsonl. Each schema is read from a year
|
|
# where it dominates, so we hit the cap within the first few-thousand lines.
|
|
SOURCES: list[tuple[str, str, int]] = [
|
|
# ("certificates-2026.json", "RdSAP-Schema-21.0.1", 1000),
|
|
# ("certificates-2022.json", "RdSAP-Schema-20.0.0", 1000),
|
|
# pre-SAP10 RdSAP family — NOT the SAP-Schema-* full/design-SAP family.
|
|
# schema_type scan (first 300k lines of each dump):
|
|
# 18.0 ~82% of certificates-2018.json
|
|
# 17.1 dominant in 2017
|
|
# 19.0 dominant in certificates-2020.json (~59%); only ~21% in 2019
|
|
# (behind 18.0), so harvest from 2020.
|
|
# 17.0 dominant in certificates-2015.json (~89%); 2016 a fallback.
|
|
# ("certificates-2018.json", "RdSAP-Schema-18.0", 1000),
|
|
# ("certificates-2017.json", "RdSAP-Schema-17.1", 1000),
|
|
# ("certificates-2020.json", "RdSAP-Schema-19.0", 1000),
|
|
("certificates-2015.json", "RdSAP-Schema-17.0", 1000),
|
|
]
|
|
|
|
|
|
def corpus_path(schema: str) -> Path:
|
|
return SAMPLES / schema / "corpus.jsonl"
|
|
|
|
|
|
# %%
|
|
def harvest_one(filename: str, schema: str, cap: int) -> list[dict[str, object]]:
|
|
"""Stream `filename`, returning up to `cap` cert docs of `schema`."""
|
|
path = DOWNLOADS / filename
|
|
docs: list[dict[str, object]] = []
|
|
scanned = 0
|
|
with path.open() as fh:
|
|
for line in fh:
|
|
if len(docs) >= cap:
|
|
break
|
|
scanned += 1
|
|
try:
|
|
doc = json.loads(json.loads(line)["document"])
|
|
except (json.JSONDecodeError, KeyError):
|
|
continue
|
|
if doc.get("schema_type") == schema:
|
|
docs.append(doc)
|
|
print(f"{schema}: {len(docs)}/{cap} from {filename} (scanned {scanned} lines)")
|
|
return docs
|
|
|
|
|
|
# %%
|
|
# Build one corpus per schema, into that schema's json_samples folder.
|
|
# Overwrites each run — deterministic and cheap.
|
|
for filename, schema, cap in SOURCES:
|
|
out_path = corpus_path(schema)
|
|
out_path.parent.mkdir(parents=True, exist_ok=True)
|
|
with out_path.open("w") as out:
|
|
for doc in harvest_one(filename, schema, cap):
|
|
out.write(json.dumps(doc) + "\n")
|
|
print(f"wrote {out_path}")
|
|
|
|
# %%
|
|
# Sanity-check each corpus: line count per schema.
|
|
for _, schema, _ in SOURCES:
|
|
path = corpus_path(schema)
|
|
n = sum(1 for line in path.read_text().splitlines() if line.strip())
|
|
print(f"{schema}: {n} ({path})")
|