restructuring repo to split out static data build processes

.idea/Model.iml

@@ -4,8 +4,9 @@
     <content url="file://$MODULE_DIR$">
       <sourceFolder url="file://$MODULE_DIR$/backend" isTestSource="false" />
       <sourceFolder url="file://$MODULE_DIR$/model_data" isTestSource="false" />
+      <sourceFolder url="file://$MODULE_DIR$/open_uprn" isTestSource="false" />
     </content>
-    <orderEntry type="jdk" jdkName="Python 3.10 fastapi" jdkType="Python SDK" />
+    <orderEntry type="jdk" jdkName="Python 3.10 (hestia-data)" jdkType="Python SDK" />
     <orderEntry type="sourceFolder" forTests="false" />
   </component>
 </module>

.idea/misc.xml

@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <project version="4">
-  <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.10 (open_uprn)" project-jdk-type="Python SDK" />
+  <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.10 (hestia-data)" project-jdk-type="Python SDK" />
   <component name="PythonCompatibilityInspectionAdvertiser">
     <option name="version" value="3" />
   </component>

backend/app/plan/router.py

@@ -15,6 +15,26 @@ router = APIRouter(
     responses={404: {"description": "Not found"}}
 )
 
+# TODO: Load this data from db
+open_uprn_data = [
+    {'UPRN': 6032920, 'X_COORDINATE': 535110.0, 'Y_COORDINATE': 181819.0, 'LATITUDE': 51.5191407,
+     'LONGITUDE': -0.0540506},
+    {'UPRN': 6038625, 'X_COORDINATE': 535374.0, 'Y_COORDINATE': 182784.0, 'LATITUDE': 51.5277492,
+     'LONGITUDE': -0.0498772},
+    {'UPRN': 34153991, 'X_COORDINATE': 523238.74, 'Y_COORDINATE': 178003.02, 'LATITUDE': 51.4875579,
+     'LONGITUDE': -0.226392},
+    {'UPRN': 10008299676, 'X_COORDINATE': 533285.0, 'Y_COORDINATE': 184711.0, 'LATITUDE': 51.5455629,
+     'LONGITUDE': -0.0792445},
+    {'UPRN': 10008299677, 'X_COORDINATE': 533285.0, 'Y_COORDINATE': 184711.0, 'LATITUDE': 51.5455629,
+     'LONGITUDE': -0.0792445},
+    {'UPRN': 100021039066, 'X_COORDINATE': 535506.0, 'Y_COORDINATE': 185624.0, 'LATITUDE': 51.5532385,
+     'LONGITUDE': -0.0468833},
+    {'UPRN': 100021226060, 'X_COORDINATE': 529247.0, 'Y_COORDINATE': 187959.0, 'LATITUDE': 51.5756908,
+     'LONGITUDE': -0.1362513},
+    {'UPRN': 200003489276, 'X_COORDINATE': 533210.0, 'Y_COORDINATE': 179442.0, 'LATITUDE': 51.4982309,
+     'LONGITUDE': -0.0823165}
+]
+
 
 @router.post("/trigger")
 async def trigger_plan(body: PlanTriggerRequest):
@@ -36,8 +56,10 @@ async def trigger_plan(body: PlanTriggerRequest):
         p.search_address_epc()
         p.set_year_built()
 
-    # TODO: get co-ordinates
+    logger.info("Getting coordinates")
+    # This is placeholder, until the full dataset is loaded into the database
+    for p in input_properties:
+        coordinate_data = [x for x in open_uprn_data if x['UPRN'] == int(p.data['uprn'])][0]
+        p.set_coordinates(coordinate_data)
 
-    logger.info("Reading in EPC data")
-
-    return {"message": "Plan triggered"}
+    return {"message": "Plan complete"}

model_data/Property.py

@@ -68,24 +68,12 @@ class Property(BaseUtility):
 
         self.data = response["rows"][0]
 
-    def get_coordinates(self, open_uprn_client):
+    def set_coordinates(self, coordinates):
         """
-        This method utlises the OpenOprnClient to get the coordinates of the property
-        The OpenOprnClient interfactes with the Ordinance Survey Open UPRN database to extract
-        property coordinates. This database holds lookups between UPRN and coordinates.
-        :param open_uprn_client:    Instance of OpenOprnClient. This method expects the client to have already read
-                                    the data
+        This method sets the coordinates of the property, given the open uprn data
+        :param coordinates: dictionary
         """
-
-        if open_uprn_client.data is None:
-            raise ValueError("OpenUprnClient has not read data")
-
-        self.coordinates = (
-            open_uprn_client.data[open_uprn_client.data["UPRN"] == int(self.data["uprn"])]
-            .to_dict("records")[0]
-        )
-
-        self.coordinates = {key.lower(): value for key, value in self.coordinates.items()}
+        self.coordinates = {key.lower(): value for key, value in coordinates.items()}
 
     def get_components(self, cleaner):
         """

open_uprn/OpenUprnClient.py

@@ -14,7 +14,7 @@ class OpenUprnClient:
 
     def __init__(self, path, uprns=None):
         self.path = path
-        self.uprns = [int(x) for x in uprns]
+        self.uprns = [int(x) for x in uprns] if uprns else None
         self.data = None
 
     def read(self):

open_uprn/app.py

@@ -15,4 +15,12 @@ def app():
     )
     open_uprn_client.read()
 
+    uprns = [
+        int(x) for x in
+        ['34153991', '6038625', '100021039066', '100021226060', '10008299676', '10008299677', '6032920', '200003489276']
+    ]
+
+    open_uprn_client.data[open_uprn_client.data["UPRN"].isin(
+        uprns
+    )].to_dict("records")
     # TODO: Add a method to write to the database

open_uprn/requirements.txt

@@ -4,3 +4,10 @@ python-dateutil==2.8.2
 pytz==2023.3
 six==1.16.0
 tzdata==2023.3
+click==8.1.6
+joblib==1.3.1
+nltk==3.8.1
+regex==2023.6.3
+textblob==0.17.1
+tqdm==4.65.0
+