"""Harvest raw EPC certificates into a JSONL corpus for mapper tests.

Source: the bulk EPC dumps in downloads/certificates-YYYY.json. Each line is

    {"certificate_number": "...", "document": "<json string>", ...}

where ``document`` is the cert in the exact shape
``EpcClientService._fetch_certificate`` returns and
``EpcPropertyDataMapper.from_api_response`` consumes (it has ``schema_type``,
``roofs``, ``walls`` ... and matches the committed json_samples).

We want a balanced sample per schema so we can build out and regression-test
the mappers (notably the incomplete ``RdSapSchema20.0.0``). Schema version
tracks the dump year, so we read each target schema from a year that's rich in
it and stop once its cap is full — no need to stream whole multi-GB files.

Year -> dominant schema (see downloads/README.txt):
    2026 -> RdSAP-Schema-21.0.1
    2021-2024 -> RdSAP-Schema-20.0.0

SAP-Schema-18.0.0 is a minority schema (~12% of the 2021 dump) but each year
holds ~1.6M lines, so 2021 still yields well over 1000 — it just scans deeper
before the cap fills. SAP-Schema-17.1 is richest in the 2019 dump (~20%).

21.0.0 is skipped — it's effectively absent from these dumps.

Run cell by cell. No API token needed — this is pure local streaming.
"""

from __future__ import annotations

import json
from pathlib import Path

import pandas as pd

DOWNLOADS = Path("downloads")
SAMPLES = Path("backend/epc_api/json_samples")

# One corpus per schema, written into that schema's own json_samples folder
# (alongside its epc.json) as corpus.jsonl. Each schema is read from a year
# where it dominates, so we hit the cap within the first few-thousand lines.
SOURCES: list[tuple[str, str, int]] = [
    # ("certificates-2026.json", "RdSAP-Schema-21.0.1", 1000),
    # ("certificates-2022.json", "RdSAP-Schema-20.0.0", 1000),
    # pre-SAP10 RdSAP family — NOT the SAP-Schema-* full/design-SAP family.
    # schema_type scan (first 300k lines of each dump):
    #   18.0  ~82% of certificates-2018.json
    #   17.1  dominant in 2017
    #   19.0  dominant in certificates-2020.json (~59%); only ~21% in 2019
    #         (behind 18.0), so harvest from 2020.
    #   17.0  dominant in certificates-2015.json (~89%); 2016 a fallback.
    # ("certificates-2018.json", "RdSAP-Schema-18.0", 1000),
    # ("certificates-2017.json", "RdSAP-Schema-17.1", 1000),
    # ("certificates-2020.json", "RdSAP-Schema-19.0", 1000),
    ("certificates-2015.json", "RdSAP-Schema-17.0", 1000),
]


def corpus_path(schema: str) -> Path:
    return SAMPLES / schema / "corpus.jsonl"


# %%
def harvest_one(filename: str, schema: str, cap: int) -> list[dict[str, object]]:
    """Stream `filename`, returning up to `cap` cert docs of `schema`."""
    path = DOWNLOADS / filename
    docs: list[dict[str, object]] = []
    scanned = 0
    with path.open() as fh:
        for line in fh:
            if len(docs) >= cap:
                break
            scanned += 1
            try:
                doc = json.loads(json.loads(line)["document"])
            except (json.JSONDecodeError, KeyError):
                continue
            if doc.get("schema_type") == schema:
                docs.append(doc)
        print(f"{schema}: {len(docs)}/{cap} from {filename} (scanned {scanned} lines)")
    return docs


# %%
# Build one corpus per schema, into that schema's json_samples folder.
# Overwrites each run — deterministic and cheap.
for filename, schema, cap in SOURCES:
    out_path = corpus_path(schema)
    out_path.parent.mkdir(parents=True, exist_ok=True)
    with out_path.open("w") as out:
        for doc in harvest_one(filename, schema, cap):
            out.write(json.dumps(doc) + "\n")
    print(f"wrote {out_path}")

# %%
# Sanity-check each corpus: line count per schema.
for _, schema, _ in SOURCES:
    path = corpus_path(schema)
    n = sum(1 for line in path.read_text().splitlines() if line.strip())
    print(f"{schema}: {n} ({path})")