diff --git a/asset_list/AssetList.py b/asset_list/AssetList.py index 5f354a27..dede3162 100644 --- a/asset_list/AssetList.py +++ b/asset_list/AssetList.py @@ -37,8 +37,7 @@ logger = setup_logger() load_dotenv(dotenv_path="../backend/.env") # OpenAI API Key (set this in your environment variables for security) -OPENAI_API_KEY = os.environ.get("OPENAI_API_KEY", - "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") +OPENAI_API_KEY = os.environ.get("OPENAI_API_KEY") class DataRemapper: diff --git a/backend/export/property_scenarios/main.py b/backend/export/property_scenarios/main.py index 00787040..b7c61df5 100644 --- a/backend/export/property_scenarios/main.py +++ b/backend/export/property_scenarios/main.py @@ -13,10 +13,10 @@ from utils.logger import setup_logger logger = setup_logger() -def choose_group_keys(payload: ExportRequest) -> List[int]: +def choose_group_keys(payload: ExportRequest) -> List[Union[int, str]]: if payload.default_plans_only: - return [] # Single export, no scenario grouping - return payload.scenario_ids or [] + return ["default_plans"] # Single export, no scenario grouping + return payload.scenario_ids def process_export(payload: ExportRequest, session: Session) -> Dict[Union[str, int], pd.DataFrame]: @@ -42,6 +42,8 @@ def process_export(payload: ExportRequest, session: Session) -> Dict[Union[str, plan_ids: List[int] = plans_df["id"].tolist() recommendations_df: pd.DataFrame = db_methods.get_recommendations(plan_ids) + logger.info("Retrieved %s recommendations for export", len(recommendations_df)) + recommendations_df = db_methods.attach_materials(recommendations_df) group_keys: List[Union[str, int]] = choose_group_keys(payload) @@ -50,14 +52,13 @@ def process_export(payload: ExportRequest, session: Session) -> Dict[Union[str, if payload.default_plans_only: scenario_recs = recommendations_df - export_label: Union[str, int] = "default_plans" else: scenario_recs = recommendations_df[ recommendations_df["scenario_id"] == group_key ] - export_label = group_key if scenario_recs.empty: + logger.info("No recommendations found for group_key %s - skipping export for this group", group_key) continue measures_df: pd.DataFrame = scenario_recs[ @@ -90,7 +91,7 @@ def process_export(payload: ExportRequest, session: Session) -> Dict[Union[str, df["predicted_post_works_sap"] = df["current_sap_points"] + df["sap_points"] df["predicted_post_works_epc"] = df["predicted_post_works_sap"].apply(sap_to_epc) - export_files[export_label] = df + export_files[group_key] = df return export_files diff --git a/backend/export/tests/test_export.py b/backend/export/tests/test_export.py index a0b0e77f..be3c5dae 100644 --- a/backend/export/tests/test_export.py +++ b/backend/export/tests/test_export.py @@ -27,38 +27,10 @@ def test_default_export_integration(db_session): t0 = time.perf_counter() portfolio_df = load_csv("portfolio_569.csv") properties_df = load_csv("properties_569.csv") - # properties_df = properties_df.head(10) - property_details_epc_df = load_csv("property_details_epc_569.csv") - # property_details_epc_df = property_details_epc_df[property_details_epc_df["property_id"].isin(properties_df[ - # "id"])] - plans_df = load_csv("plans_569.csv") - # plans_df = plans_df[plans_df["property_id"].isin(properties_df["id"])] - plan_recs_df = load_csv("plan_recs_569.csv") - # plan_recs_df = plan_recs_df[plan_recs_df["plan_id"].isin(plans_df["id"])] recommendations_df = load_csv("recommendations_569.csv") - # recommendations_df = recommendations_df[recommendations_df["id"].isin(plan_recs_df["recommendation_id"])] - - # Shrink down recommendations_df to speed up the data load. For this test, we only need - # default recommendations so let's focus on those. We filter on where default is true - # recommendations_df = recommendations_df[ - # recommendations_df["default"] - # ] - # valid_rec_ids = recommendations_df["id"].unique() - # - # plan_recs_df = plan_recs_df[ - # plan_recs_df["recommendation_id"].isin(valid_rec_ids) - # ] - - # Save all of this: - # portfolio_df.to_csv("portfolio_569.csv") - # properties_df.to_csv("properties_569.csv") - # property_details_epc_df.to_csv("property_details_epc_569.csv") - # plans_df.to_csv("plans_569.csv") - # plan_recs_df.to_csv("plan_recs_569.csv") - # recommendations_df.to_csv("recommendations_569.csv") logger.info( "Loaded CSVs in %.2f seconds | properties=%s plans=%s recs=%s", @@ -129,22 +101,11 @@ def test_default_export_integration(db_session): # 4) Insert property details - EPC # ---------------------------------------- - property_lookup = { - prop.uprn: prop - for prop in db_session.query(PropertyModel).all() - } - epc_rows = [] for row in property_details_epc_df.itertuples(index=False): row_dict = row._asdict() - uprn = int(row_dict["uprn"]) if row_dict.get("uprn") else None - property_obj = property_lookup.get(uprn) - - if not property_obj: - continue # skip if property not found - # Build only fields that exist on the model epc_data = { col.name: row_dict[col.name] @@ -153,8 +114,8 @@ def test_default_export_integration(db_session): } epc = PropertyDetailsEpcModel( - property_id=property_obj.id, - portfolio_id=property_obj.portfolio_id, + property_id=row.property_id, + portfolio_id=row.portfolio_id, **epc_data, ) @@ -250,13 +211,18 @@ def test_default_export_integration(db_session): ) logger.info( - "Default + not installed count: %s", - db_session.query(Recommendation) - .filter( - Recommendation.default.is_(True), - Recommendation.already_installed.is_(False) - ) - .count() + "Property EPC in DB: %s", + db_session.query(PropertyDetailsEpcModel).count() + ) + + logger.info( + "Plan count in DB: %s", + db_session.query(PlanModel).count() + ) + + logger.info( + "PlanRecommendatons count in DB: %s", + db_session.query(PlanModel).count() ) logger.info("Starting process_export") @@ -276,21 +242,9 @@ def test_default_export_integration(db_session): assert df.shape[0] == 10, "Expected 10 properties in the export, got {}".format(df.shape[0]) - # This test was generated on a real portfolio and so we check the things we expect to do - - # 1) All packages are "compliant", where in this case, the properties should get to EPC C - failed = df[df["predicted_post_works_sap"] < 69] failed_property_types = failed["property_type"].value_counts().to_dict() - assert failed_property_types["Flat"] == 113 - assert failed_property_types["House"] == 8 - assert failed_property_types["Bungalow"] == 4 - assert failed_property_types["Maisonette"] == 1 + assert failed_property_types["Flat"] == 2 # Check the houses assert failed.shape[0] - - # Errors for me: - # - should get to EPC C: https://ara.domna.homes/portfolio/569/building-passport/661051/plans - # - Why doesn't this get to a C, under the plan?: - # https://ara.domna.homes/portfolio/569/building-passport/660447/plans/1603913