setting up data pull code

etl/customers/stonewater/shdf_3_clustering.py

@@ -73,3 +73,42 @@ def app():
 
     if pd.isnull(asset_list["full_address"]).sum():
         raise ValueError("Missing full addresses")
+
+    # TODO: Store in S3
+
+    # TODO: Move ths
+    # Pull in the data
+
+    import os
+    from dotenv import load_dotenv
+    from backend.SearchEpc import SearchEpc
+
+    load_dotenv(dotenv_path="backend/.env")
+    EPC_AUTH_TOKEN = os.getenv("EPC_AUTH_TOKEN")
+
+    # Perform an initial pull without ordnance survey data
+    epc_data = []
+    older_epc_data = {}
+    for row_number, asset in asset_list.iterrows():
+        searcher = SearchEpc(
+            address1=asset["address1"],
+            postcode=asset["postcode"],
+            auth_token=EPC_AUTH_TOKEN,
+            os_api_key="",
+            full_address=asset["full_address"],
+            uprn=asset.get("uprn", None),
+        )
+        searcher.find_property(skip_os=True)
+
+        if searcher.newest_epc is None:
+            continue
+
+        epc_data.append(
+            {
+                "internal_id": asset["internal_id"],
+                **searcher.newest_epc
+            }
+        )
+
+        if searcher.older_epcs is not None:
+            older_epc_data[asset["internal_id"]] = searcher.older_epcs