restructuring openUrpn code

.gitignore

@@ -170,7 +170,7 @@ cython_debug/
 .idea/**/shelf
 
 # AWS User-specific
-.idea/**/aws.xml
+.idea/misc.xml
 
 # Generated files
 .idea/**/contentModel.xml
@@ -245,3 +245,9 @@ infrastructure/terraform/.terraform*
 
 # Don't commit packages up serverless packages
 .serverless
+
+backend/node_modules
+node_modules/
+backend/.idea
+open_uprn/.idea/
+

.idea/misc.xml

@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <project version="4">
-  <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.10 fastapi" project-jdk-type="Python SDK" />
+  <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.10 (open_uprn)" project-jdk-type="Python SDK" />
   <component name="PythonCompatibilityInspectionAdvertiser">
     <option name="version" value="3" />
   </component>

backend/app/plan/router.py

@@ -22,28 +22,20 @@ async def trigger_plan(body: PlanTriggerRequest):
     # Read in the trigger file from s3
     bucket_name = get_settings().PLAN_TRIGGER_BUCKET
     plan_input = read_csv_from_s3(bucket_name=bucket_name, filepath=body.trigger_file_path)
-    print(plan_input)
+
     # TODO: Add validation to the file
 
-    print("What's the token")
-    print(get_settings().EPC_AUTH_TOKEN)
-
+    logger.info("Getting EPC data")
     epc_client = EpcClient(auth_token=get_settings().EPC_AUTH_TOKEN)
     input_properties = [
         Property(postcode=config['postcode'], address1=config['address'], epc_client=epc_client)
         for config in plan_input
     ]
 
-    logger.info("Getting EPC data")
     for p in input_properties:
         p.search_address_epc()
         p.set_year_built()
 
-    logger.info("Parsing and validating the file")
-
-    logger.info("properties")
-    logger.info(input_properties)
-
     # TODO: get co-ordinates
 
     logger.info("Reading in EPC data")

model_data/app.py

@@ -10,7 +10,7 @@ from model_data.config import EPC_AUTH_TOKEN
 from epc_api.client import EpcClient
 from model_data.downloader import pagenated_epc_download
 from model_data.EpcClean import EpcClean
-from model_data.OpenUprnClient import OpenUprnClient
+from open_uprn.OpenUprnClient import OpenUprnClient
 from model_data.analysis.UvalueEstimations import UvalueEstimations
 
 LAND_REGISTRY_PATHS = [
@@ -245,11 +245,6 @@ def handler():
 
     print(results.summary())
 
-    import matplotlib.pyplot as plt
-    import numpy as np
-
-    import numpy as np
-
     grouped_error = []
     groupby = ["mainheat-description"]
     for group, data in model_data.groupby(groupby, observed=True):

model_data/tests/test_property.py

@@ -3,7 +3,7 @@ import pandas as pd
 from unittest.mock import Mock
 from epc_api.client import EpcClient
 from model_data.Property import Property
-from model_data.OpenUprnClient import OpenUprnClient
+from open_uprn.OpenUprnClient import OpenUprnClient
 from model_data.EpcClean import EpcClean
 
 # Define some test data

open_uprn/OpenUprnClient.py

@@ -12,7 +12,7 @@ class OpenUprnClient:
 
     # TODO: Document this
 
-    def __init__(self, path, uprns):
+    def __init__(self, path, uprns=None):
         self.path = path
         self.uprns = [int(x) for x in uprns]
         self.data = None
@@ -25,6 +25,7 @@ class OpenUprnClient:
         logger.info("Reading in open uprn data")
 
         df = pd.read_csv(self.path)
-        df = df[df["UPRN"].isin(self.uprns)]
+        if self.uprns:
+            df = df[df["UPRN"].isin(self.uprns)]
 
         self.data = df

open_uprn/app.py

@@ -0,0 +1,18 @@
+"""
+This application reads in the open uprn data from a static location and loads it into
+our database for querying from other services
+"""
+
+import os
+from open_uprn.OpenUprnClient import OpenUprnClient
+
+
+def app():
+    open_uprn_client = OpenUprnClient(
+        path=os.path.abspath(
+            os.path.dirname(__file__)
+        ) + "/model_data/local_data/osopenuprn_202306_csv/osopenuprn_202305.csv",
+    )
+    open_uprn_client.read()
+
+    # TODO: Add a method to write to the database

open_uprn/requirements.txt

@@ -0,0 +1,6 @@
+numpy==1.25.1
+pandas==2.0.3
+python-dateutil==2.8.2
+pytz==2023.3
+six==1.16.0
+tzdata==2023.3

package.json

@@ -0,0 +1,6 @@
+{
+  "devDependencies": {
+    "serverless-domain-manager": "^7.1.0",
+    "serverless-python-requirements": "^6.0.0"
+  }
+}