working on export issues

.idea/watcherTasks.xml

@@ -1,7 +1,7 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <project version="4">
   <component name="ProjectTasksOptions">
-    <TaskOptions isEnabled="true">
+    <TaskOptions isEnabled="false">
       <option name="arguments" value="$FilePath$" />
       <option name="checkSyntaxErrors" value="true" />
       <option name="description" />

backend/export/property_scenarios/main.py

@@ -37,6 +37,7 @@ def process_export(payload: ExportRequest, session: Session) -> Dict[Union[str,
     logger.info("Retrieved %s plans for export", len(plans_df))
 
     if plans_df.empty:
+        logger.info("Empty plans dataframe - no plans to export. Returning empty export.")
         return export_files
     plan_ids: List[int] = plans_df["id"].tolist()
     recommendations_df: pd.DataFrame = db_methods.get_recommendations(plan_ids)

backend/export/tests/test_export.py

@@ -27,21 +27,38 @@ def test_default_export_integration(db_session):
     t0 = time.perf_counter()
     portfolio_df = load_csv("portfolio_569.csv")
     properties_df = load_csv("properties_569.csv")
+    # properties_df = properties_df.head(10)
+
     property_details_epc_df = load_csv("property_details_epc_569.csv")
+    # property_details_epc_df = property_details_epc_df[property_details_epc_df["property_id"].isin(properties_df[
+    # "id"])]
+
     plans_df = load_csv("plans_569.csv")
+    # plans_df = plans_df[plans_df["property_id"].isin(properties_df["id"])]
+
     plan_recs_df = load_csv("plan_recs_569.csv")
+    # plan_recs_df = plan_recs_df[plan_recs_df["plan_id"].isin(plans_df["id"])]
     recommendations_df = load_csv("recommendations_569.csv")
+    # recommendations_df = recommendations_df[recommendations_df["id"].isin(plan_recs_df["recommendation_id"])]
 
     # Shrink down recommendations_df to speed up the data load. For this test, we only need
     # default recommendations so let's focus on those. We filter on where default is true
-    recommendations_df = recommendations_df[
-        recommendations_df["default"]
-    ]
-    valid_rec_ids = recommendations_df["id"].unique()
+    # recommendations_df = recommendations_df[
+    #     recommendations_df["default"]
+    # ]
+    # valid_rec_ids = recommendations_df["id"].unique()
+    #
+    # plan_recs_df = plan_recs_df[
+    #     plan_recs_df["recommendation_id"].isin(valid_rec_ids)
+    # ]
 
-    plan_recs_df = plan_recs_df[
-        plan_recs_df["recommendation_id"].isin(valid_rec_ids)
-    ]
+    # Save all of this:
+    # portfolio_df.to_csv("portfolio_569.csv")
+    # properties_df.to_csv("properties_569.csv")
+    # property_details_epc_df.to_csv("property_details_epc_569.csv")
+    # plans_df.to_csv("plans_569.csv")
+    # plan_recs_df.to_csv("plan_recs_569.csv")
+    # recommendations_df.to_csv("recommendations_569.csv")
 
     logger.info(
         "Loaded CSVs in %.2f seconds | properties=%s plans=%s recs=%s",
@@ -131,7 +148,7 @@ def test_default_export_integration(db_session):
         # Build only fields that exist on the model
         epc_data = {
             col.name: row_dict[col.name]
-            for col in PropertyDetailsEpcModel.__table__.columns
+            for col in PropertyDetailsEpcModel.__table__.columns.values()
             if col.name in row_dict and col.name not in ["id", "property_id", "portfolio_id"]
         }
 
@@ -227,6 +244,11 @@ def test_default_export_integration(db_session):
         db_session.query(Recommendation).count()
     )
 
+    logger.info(
+        "Property count in DB: %s",
+        db_session.query(PropertyModel).count()
+    )
+
     logger.info(
         "Default + not installed count: %s",
         db_session.query(Recommendation)
@@ -248,11 +270,11 @@ def test_default_export_integration(db_session):
     # 8) Assertions
     # ----------------------------------------
 
-    assert "default_plans" in result
+    assert "default_plans" in result, "Expected 'default_plans' in export result, got {}".format(result.keys())
 
     df = result["default_plans"]
 
-    assert not df.empty
+    assert df.shape[0] == 10, "Expected 10 properties in the export, got {}".format(df.shape[0])
 
     # This test was generated on a real portfolio and so we check the things we expect to do
 

sfr/principal_pitch/2_export_data.py

@@ -28,15 +28,16 @@ from sqlalchemy import func
 
 # PORTFOLIO_ID = 206
 # SCENARIOS = [389]
-PORTFOLIO_ID = 568
+PORTFOLIO_ID = 581
 SCENARIOS = [
-    1059,
+    1074, 1075
 ]
 scenario_names = {
-    1059: "EPC C - 10k budget",
+    1074: "EPC C",
+    1075: "EPC C Again",
 }
 
-project_name = "manchester"
+project_name = "???"
 
 
 def get_data(portfolio_id, scenario_ids):
@@ -234,7 +235,7 @@ for scenario_id in SCENARIOS:
     # Get recs for this scenario
     recommended_measures_df = recommendations_df[
         recommendations_df["scenario_id"] == scenario_id
-    ][["property_id", "measure_type", "estimated_cost", "default"]]
+        ][["property_id", "measure_type", "estimated_cost", "default"]]
     recommended_measures_df = recommended_measures_df[
         recommended_measures_df["default"]
     ]
@@ -242,7 +243,7 @@ for scenario_id in SCENARIOS:
 
     post_install_sap = recommendations_df[
         recommendations_df["scenario_id"] == scenario_id
-    ][["property_id", "default", "sap_points"]]
+        ][["property_id", "default", "sap_points"]]
     post_install_sap = post_install_sap[post_install_sap["default"]]
     # Sum up the sap points by property id
     post_install_sap = (
@@ -320,7 +321,7 @@ for scenario_id in SCENARIOS:
     z = df2[
         (df2["predicted_post_works_epc"] != "D")
         & (df2["post_epc_rating"].astype(str) == "Epc.D")
-    ]
+        ]
 
     df2["predicted_post_works_epc"].value_counts()
     df2["post_epc_rating"].astype(str).value_counts()