mirror of
https://github.com/Hestia-Homes/Model.git
synced 2026-06-08 11:17:27 +00:00
Fixed outputs of export test
This commit is contained in:
Khalim Conn-Kowlessar
parent
1df4fb7815
commit
e5d7a3ff14
3 changed files with 23 additions and 69 deletions
|
|
@ -37,8 +37,7 @@ logger = setup_logger()
|
|||
load_dotenv(dotenv_path="../backend/.env")
|
||||
|
||||
# OpenAI API Key (set this in your environment variables for security)
|
||||
OPENAI_API_KEY = os.environ.get("OPENAI_API_KEY",
|
||||
"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")
|
||||
OPENAI_API_KEY = os.environ.get("OPENAI_API_KEY")
|
||||
|
||||
|
||||
class DataRemapper:
|
||||
|
|
|
|||
|
|
@ -13,10 +13,10 @@ from utils.logger import setup_logger
|
|||
logger = setup_logger()
|
||||
|
||||
|
||||
def choose_group_keys(payload: ExportRequest) -> List[int]:
|
||||
def choose_group_keys(payload: ExportRequest) -> List[Union[int, str]]:
|
||||
if payload.default_plans_only:
|
||||
return [] # Single export, no scenario grouping
|
||||
return payload.scenario_ids or []
|
||||
return ["default_plans"] # Single export, no scenario grouping
|
||||
return payload.scenario_ids
|
||||
|
||||
|
||||
def process_export(payload: ExportRequest, session: Session) -> Dict[Union[str, int], pd.DataFrame]:
|
||||
|
|
@ -42,6 +42,8 @@ def process_export(payload: ExportRequest, session: Session) -> Dict[Union[str,
|
|||
plan_ids: List[int] = plans_df["id"].tolist()
|
||||
recommendations_df: pd.DataFrame = db_methods.get_recommendations(plan_ids)
|
||||
|
||||
logger.info("Retrieved %s recommendations for export", len(recommendations_df))
|
||||
|
||||
recommendations_df = db_methods.attach_materials(recommendations_df)
|
||||
|
||||
group_keys: List[Union[str, int]] = choose_group_keys(payload)
|
||||
|
|
@ -50,14 +52,13 @@ def process_export(payload: ExportRequest, session: Session) -> Dict[Union[str,
|
|||
|
||||
if payload.default_plans_only:
|
||||
scenario_recs = recommendations_df
|
||||
export_label: Union[str, int] = "default_plans"
|
||||
else:
|
||||
scenario_recs = recommendations_df[
|
||||
recommendations_df["scenario_id"] == group_key
|
||||
]
|
||||
export_label = group_key
|
||||
|
||||
if scenario_recs.empty:
|
||||
logger.info("No recommendations found for group_key %s - skipping export for this group", group_key)
|
||||
continue
|
||||
|
||||
measures_df: pd.DataFrame = scenario_recs[
|
||||
|
|
@ -90,7 +91,7 @@ def process_export(payload: ExportRequest, session: Session) -> Dict[Union[str,
|
|||
df["predicted_post_works_sap"] = df["current_sap_points"] + df["sap_points"]
|
||||
df["predicted_post_works_epc"] = df["predicted_post_works_sap"].apply(sap_to_epc)
|
||||
|
||||
export_files[export_label] = df
|
||||
export_files[group_key] = df
|
||||
|
||||
return export_files
|
||||
|
||||
|
|
|
|||
|
|
@ -27,38 +27,10 @@ def test_default_export_integration(db_session):
|
|||
t0 = time.perf_counter()
|
||||
portfolio_df = load_csv("portfolio_569.csv")
|
||||
properties_df = load_csv("properties_569.csv")
|
||||
# properties_df = properties_df.head(10)
|
||||
|
||||
property_details_epc_df = load_csv("property_details_epc_569.csv")
|
||||
# property_details_epc_df = property_details_epc_df[property_details_epc_df["property_id"].isin(properties_df[
|
||||
# "id"])]
|
||||
|
||||
plans_df = load_csv("plans_569.csv")
|
||||
# plans_df = plans_df[plans_df["property_id"].isin(properties_df["id"])]
|
||||
|
||||
plan_recs_df = load_csv("plan_recs_569.csv")
|
||||
# plan_recs_df = plan_recs_df[plan_recs_df["plan_id"].isin(plans_df["id"])]
|
||||
recommendations_df = load_csv("recommendations_569.csv")
|
||||
# recommendations_df = recommendations_df[recommendations_df["id"].isin(plan_recs_df["recommendation_id"])]
|
||||
|
||||
# Shrink down recommendations_df to speed up the data load. For this test, we only need
|
||||
# default recommendations so let's focus on those. We filter on where default is true
|
||||
# recommendations_df = recommendations_df[
|
||||
# recommendations_df["default"]
|
||||
# ]
|
||||
# valid_rec_ids = recommendations_df["id"].unique()
|
||||
#
|
||||
# plan_recs_df = plan_recs_df[
|
||||
# plan_recs_df["recommendation_id"].isin(valid_rec_ids)
|
||||
# ]
|
||||
|
||||
# Save all of this:
|
||||
# portfolio_df.to_csv("portfolio_569.csv")
|
||||
# properties_df.to_csv("properties_569.csv")
|
||||
# property_details_epc_df.to_csv("property_details_epc_569.csv")
|
||||
# plans_df.to_csv("plans_569.csv")
|
||||
# plan_recs_df.to_csv("plan_recs_569.csv")
|
||||
# recommendations_df.to_csv("recommendations_569.csv")
|
||||
|
||||
logger.info(
|
||||
"Loaded CSVs in %.2f seconds | properties=%s plans=%s recs=%s",
|
||||
|
|
@ -129,22 +101,11 @@ def test_default_export_integration(db_session):
|
|||
# 4) Insert property details - EPC
|
||||
# ----------------------------------------
|
||||
|
||||
property_lookup = {
|
||||
prop.uprn: prop
|
||||
for prop in db_session.query(PropertyModel).all()
|
||||
}
|
||||
|
||||
epc_rows = []
|
||||
|
||||
for row in property_details_epc_df.itertuples(index=False):
|
||||
row_dict = row._asdict()
|
||||
|
||||
uprn = int(row_dict["uprn"]) if row_dict.get("uprn") else None
|
||||
property_obj = property_lookup.get(uprn)
|
||||
|
||||
if not property_obj:
|
||||
continue # skip if property not found
|
||||
|
||||
# Build only fields that exist on the model
|
||||
epc_data = {
|
||||
col.name: row_dict[col.name]
|
||||
|
|
@ -153,8 +114,8 @@ def test_default_export_integration(db_session):
|
|||
}
|
||||
|
||||
epc = PropertyDetailsEpcModel(
|
||||
property_id=property_obj.id,
|
||||
portfolio_id=property_obj.portfolio_id,
|
||||
property_id=row.property_id,
|
||||
portfolio_id=row.portfolio_id,
|
||||
**epc_data,
|
||||
)
|
||||
|
||||
|
|
@ -250,13 +211,18 @@ def test_default_export_integration(db_session):
|
|||
)
|
||||
|
||||
logger.info(
|
||||
"Default + not installed count: %s",
|
||||
db_session.query(Recommendation)
|
||||
.filter(
|
||||
Recommendation.default.is_(True),
|
||||
Recommendation.already_installed.is_(False)
|
||||
)
|
||||
.count()
|
||||
"Property EPC in DB: %s",
|
||||
db_session.query(PropertyDetailsEpcModel).count()
|
||||
)
|
||||
|
||||
logger.info(
|
||||
"Plan count in DB: %s",
|
||||
db_session.query(PlanModel).count()
|
||||
)
|
||||
|
||||
logger.info(
|
||||
"PlanRecommendatons count in DB: %s",
|
||||
db_session.query(PlanModel).count()
|
||||
)
|
||||
|
||||
logger.info("Starting process_export")
|
||||
|
|
@ -276,21 +242,9 @@ def test_default_export_integration(db_session):
|
|||
|
||||
assert df.shape[0] == 10, "Expected 10 properties in the export, got {}".format(df.shape[0])
|
||||
|
||||
# This test was generated on a real portfolio and so we check the things we expect to do
|
||||
|
||||
# 1) All packages are "compliant", where in this case, the properties should get to EPC C
|
||||
|
||||
failed = df[df["predicted_post_works_sap"] < 69]
|
||||
failed_property_types = failed["property_type"].value_counts().to_dict()
|
||||
assert failed_property_types["Flat"] == 113
|
||||
assert failed_property_types["House"] == 8
|
||||
assert failed_property_types["Bungalow"] == 4
|
||||
assert failed_property_types["Maisonette"] == 1
|
||||
assert failed_property_types["Flat"] == 2
|
||||
# Check the houses
|
||||
|
||||
assert failed.shape[0]
|
||||
|
||||
# Errors for me:
|
||||
# - should get to EPC C: https://ara.domna.homes/portfolio/569/building-passport/661051/plans
|
||||
# - Why doesn't this get to a C, under the plan?:
|
||||
# https://ara.domna.homes/portfolio/569/building-passport/660447/plans/1603913
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue