Merge pull request #78 from Hestia-Homes/main

Restructuring backend to include model_data repo

.dockerignore

@@ -0,0 +1 @@
+model_data/local_data/*

.idea/Model.iml

@@ -1,8 +1,11 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <module type="PYTHON_MODULE" version="4">
   <component name="NewModuleRootManager">
-    <content url="file://$MODULE_DIR$" />
-    <orderEntry type="jdk" jdkName="Python 3.10 (hestia-data)" jdkType="Python SDK" />
+    <content url="file://$MODULE_DIR$">
+      <sourceFolder url="file://$MODULE_DIR$/backend" isTestSource="false" />
+      <sourceFolder url="file://$MODULE_DIR$/model_data" isTestSource="false" />
+    </content>
+    <orderEntry type="jdk" jdkName="Python 3.10 (BackendApi)" jdkType="Python SDK" />
     <orderEntry type="sourceFolder" forTests="false" />
   </component>
 </module>

.idea/misc.xml

@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <project version="4">
-  <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.10 (hestia-data)" project-jdk-type="Python SDK" />
+  <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.10 (BackendApi)" project-jdk-type="Python SDK" />
   <component name="PythonCompatibilityInspectionAdvertiser">
     <option name="version" value="3" />
   </component>

backend/app/config.py

@@ -8,9 +8,10 @@ class Settings(BaseSettings):
     SECRET_KEY: str
     ENVIRONMENT: str
     PLAN_TRIGGER_BUCKET: str
+    EPC_AUTH_TOKEN: str
 
     class Config:
-        env_file = ".env"
+        env_file = "backend/.env"
 
 
 @lru_cache()

backend/app/dependencies.py

@@ -6,9 +6,10 @@ from cryptography.hazmat.primitives import hashes
 from cryptography.hazmat.backends import default_backend
 from typing import Any
 import json
-from app.config import get_settings
-from app.utils import logger
+from backend.app.config import get_settings
+from backend.app.utils import setup_logger
 
+logger = setup_logger()
 
 api_key_header = APIKeyHeader(name=get_settings().API_KEY_NAME, auto_error=False)
 oauth2_scheme = OAuth2PasswordBearer(tokenUrl="token")
@@ -94,7 +95,6 @@ def validate_jwt_token(token: str = Depends(oauth2_scheme)):
 
 
 async def validate_token(token: str = Depends(oauth2_scheme), request: Request = None):
-    print("VALIDATING - PRINT")
     logger.info("Validating token")
     logger.info(token)
     logger.info("Secret")

backend/app/main.py

@@ -1,19 +1,18 @@
 from fastapi import FastAPI, Depends
 from mangum import Mangum
-from app.portfolio import router as portfolio_router
-from app.plan import router as plan_router
-from app.dependencies import validate_api_key
-from app.config import get_settings
-
+from backend.app.portfolio import router as portfolio_router
+from backend.app.plan import router as plan_router
+from backend.app.dependencies import validate_api_key
+from backend.app.config import get_settings
 
 app = FastAPI(dependencies=[Depends(validate_api_key)])
 
-
 app.include_router(portfolio_router.router, prefix="/v1")
 app.include_router(plan_router.router, prefix="/v1")
 
 if get_settings().ENVIRONMENT == "local":
     from app.local import router as local_router
+
     app.include_router(local_router.router)
 
 handler = Mangum(app)

backend/app/plan/router.py

@@ -1,9 +1,12 @@
 from fastapi import APIRouter, Depends
-from app.dependencies import validate_token
-from app.plan.schemas import PlanTriggerRequest
-from app.utils import read_csv_from_s3, logger
-from app.config import get_settings
+from backend.app.dependencies import validate_token
+from backend.app.plan.schemas import PlanTriggerRequest
+from backend.app.utils import read_csv_from_s3, setup_logger
+from backend.app.config import get_settings
+from model_data.Property import Property
+from epc_api.client import EpcClient
 
+logger = setup_logger()
 
 router = APIRouter(
     prefix="/plan",
@@ -18,13 +21,25 @@ async def trigger_plan(body: PlanTriggerRequest):
     logger.info("Getting the inputs")
     # Read in the trigger file from s3
     bucket_name = get_settings().PLAN_TRIGGER_BUCKET
-    logger.info("bucket_name: ", bucket_name)
-    logger.info("body.trigger_file_path: ", body.trigger_file_path)
     plan_input = read_csv_from_s3(bucket_name=bucket_name, filepath=body.trigger_file_path)
-    logger.info("Got the inputs")
-    logger.info(plan_input)
+    print(plan_input)
 
-    # TODO: Parse the file
-    # TODO: Put messages on the queue
+    epc_client = EpcClient(auth_token=get_settings().EPC_AUTH_TOKEN)
+    input_properties = [
+        Property(postcode=config['postcode'], address1=config['address'], epc_client=epc_client)
+        for config in plan_input
+    ]
+
+    logger.info("Getting EPC data")
+    for p in input_properties:
+        p.search_address_epc()
+        p.set_year_built()
+
+    logger.info("Parsing and validating the file")
+    # TODO: Add validation
+    logger.info("properties")
+    logger.info(input_properties)
+
+    logger.info("Reading in EPC data")
 
     return {"message": "Plan triggered"}

backend/app/portfolio/router.py

@@ -1,5 +1,5 @@
 from fastapi import APIRouter, Depends
-from app.dependencies import validate_token
+from backend.app.dependencies import validate_token
 
 router = APIRouter(
     prefix="/portfolio",

backend/app/utils.py

@@ -6,8 +6,38 @@ import secrets
 import logging
 
 
-logger = logging.getLogger(__name__)
-logger.setLevel(logging.INFO)
+def setup_logger(log_file=None, level=logging.INFO, overwrite_handler=False):
+    # Create a logger and set the logging level
+    logger = logging.getLogger()
+    logger.setLevel(level)
+
+    # if logger already has handlers, just return it
+    if logger.hasHandlers() and not overwrite_handler:
+        return logger
+
+    # Define the log message format
+    log_format = "%(asctime)s [%(levelname)s] %(message)s"
+    date_format = "%Y-%m-%d %H:%M:%S"
+    formatter = logging.Formatter(log_format, datefmt=date_format)
+
+    # Create a file handler and set the file path and format
+    if log_file:
+        file_handler = logging.FileHandler(log_file)
+        file_handler.setLevel(level)
+        file_handler.setFormatter(formatter)
+        logger.addHandler(file_handler)
+
+    # Create a console handler and set the format
+    console_handler = logging.StreamHandler()
+    console_handler.setLevel(level)
+
+    # Set the formatter for the handlers
+    console_handler.setFormatter(formatter)
+
+    # Add the handlers to the logger
+    logger.addHandler(console_handler)
+
+    return logger
 
 
 def read_csv_from_s3(bucket_name, filepath):

backend/docker/Dockerfile

@@ -5,19 +5,22 @@ FROM python:3.10.12-slim-buster
 ENV PYTHONDONTWRITEBYTECODE 1
 ENV PYTHONUNBUFFERED 1
 
-# Set work directory
-WORKDIR /app
+# Set work directory to the root of your project
+WORKDIR /Model
 
 # Install system dependencies
 RUN apt-get update && apt-get install -y netcat-openbsd
 
 # Install python dependencies
-COPY ./requirements/base.txt ./requirements/base.txt
+COPY ./backend/requirements/base.txt ./backend/requirements/base.txt
+COPY ./model_data/requirements/requirements.txt ./model_data/requirements/requirements.txt
 RUN pip install --upgrade pip
-RUN pip install -r requirements/base.txt
+RUN pip install -r backend/requirements/base.txt
+RUN pip install -r model_data/requirements/requirements.txt
 
 # Copy project
-COPY . .
+COPY ./backend ./backend
+COPY ./model_data ./model_data
 
 # command to run on container start
-CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000"]
+CMD ["uvicorn", "backend.app.main:app", "--host", "0.0.0.0", "--port", "8000"]

backend/requirements/base.txt

@@ -1,6 +1,7 @@
 anyio==3.7.1
 cffi==1.15.1
 click==8.1.3
+cryptography==37.0.4
 ecdsa==0.18.0
 exceptiongroup==1.1.2
 fastapi==0.99.1
@@ -15,4 +16,13 @@ PyJWT==2.7.0
 python-dotenv==1.0.0
 python-jose==3.3.0
 PyYAML==6.0
-cryptography==37.0.4
+rsa==4.9
+six==1.16.0
+sniffio==1.3.0
+starlette==0.27.0
+typing_extensions==4.7.1
+uvicorn==0.22.0
+uvloop==0.17.0
+watchfiles==0.19.0
+websockets==11.0.3
+boto3

backend/requirements/lambda.txt

@@ -1 +0,0 @@
-cryptography==41.0.2

backend/requirements/local.txt

@@ -0,0 +1,28 @@
+anyio==3.7.1
+cffi==1.15.1
+click==8.1.3
+cryptography==37.0.4
+ecdsa==0.18.0
+exceptiongroup==1.1.2
+fastapi==0.99.1
+h11==0.14.0
+httptools==0.5.0
+idna==3.4
+mangum==0.17.0
+pyasn1==0.5.0
+pycparser==2.21
+pydantic==1.10.11
+PyJWT==2.7.0
+python-dotenv==1.0.0
+python-jose==3.3.0
+PyYAML==6.0
+rsa==4.9
+six==1.16.0
+sniffio==1.3.0
+starlette==0.27.0
+typing_extensions==4.7.1
+uvicorn==0.22.0
+uvloop==0.17.0
+watchfiles==0.19.0
+websockets==11.0.3
+boto3

backend/serverless.yml

@@ -17,6 +17,8 @@ package:
   individually: true
   include:
     - Model/backend/**
+    # Might need to refine the paths that are included
+    - Model/model_data/**
 
 plugins:
   - serverless-python-requirements

model_data/ConservationAreaClient.py

@@ -1,4 +1,4 @@
-import pandas as pd
+from enum import Enum
 import geopandas as gpd
 from shapely.geometry import Point
 from model_data.utils import setup_logger
@@ -39,6 +39,31 @@ class ConservationAreaClient:
         self.gov_data = gpd.read_file(self.gov_path)
         self.gov_data = self.gov_data.drop(columns=["dataset"])
 
+    def is_in_conservation_area(self, coordinates: dict):
+
+        if not coordinates:
+            raise ValueError("Coordinates have not been set, run get_coordinates() first")
+
+        is_in_conservation_area = self.is_in_conservation_area_historic_england(
+            x_bng=coordinates["x_coordinate"],
+            y_bng=coordinates["y_coordinate"]
+        )
+
+        if is_in_conservation_area != "unknown":
+            return is_in_conservation_area
+
+        if is_in_conservation_area == "unknown":
+            # We double check the secondary data source
+            backup = self.is_in_conservation_area_historic_gov(
+                longitude=coordinates["longitude"],
+                latitude=coordinates["latitude"]
+            )
+
+            if backup:
+                return ConservationAreaClient.IN_CONSERVATION_AREA
+            else:
+                return ConservationAreaClient.UNKNOWN
+
     def is_in_conservation_area_historic_england(self, x_bng: float, y_bng: float) -> str:
         """
         Check if a property is in a conservation area
@@ -103,3 +128,9 @@ class ConservationAreaClient:
         distance_meters = distances.min()
 
         return distance_meters
+
+
+class InConservationArea(Enum):
+    IN_CONSERVATION_AREA = ConservationAreaClient.IN_CONSERVATION_AREA
+    NOT_IN_CONSERVATION_AREA = ConservationAreaClient.NOT_IN_CONSERVATION_AREA
+    UNKNOWN = ConservationAreaClient.UNKNOWN

model_data/Property.py

@@ -5,7 +5,6 @@ from model_data.config import EPC_AUTH_TOKEN
 from model_data.OpenUprnClient import OpenUprnClient
 from model_data.EpcClean import EpcClean
 from model_data.BaseUtility import BaseUtility
-from model_data.ConservationAreaClient import ConservationAreaClient
 
 
 class Property(BaseUtility):
@@ -117,28 +116,12 @@ class Property(BaseUtility):
                 raise ValueError("Either No attributes or multiple found for %s" % description)
             setattr(self, self.ATTRIBUTE_MAP[description], attributes[0])
 
-    def set_is_in_conservation_area(self, conservation_area_client: ConservationAreaClient):
-
-        if not self.coordinates:
-            raise ValueError("Coordinates have not been set, run get_coordinates() first")
-
-        is_in_conservation_area = conservation_area_client.is_in_conservation_area_historic_england(
-            x_bng=self.coordinates["x_coordinate"],
-            y_bng=self.coordinates["y_coordinate"]
-        )
-
-        self.in_conservation_area = is_in_conservation_area
-        if is_in_conservation_area == "unknown":
-            # We double check the secondary data source
-            backup = conservation_area_client.is_in_conservation_area_historic_gov(
-                longitude=self.coordinates["longitude"],
-                latitude=self.coordinates["latitude"]
-            )
-
-            if backup:
-                self.in_conservation_area = ConservationAreaClient.IN_CONSERVATION_AREA
-            else:
-                self.in_conservation_area = ConservationAreaClient.UNKNOWN
+    def set_is_in_conservation_area(self, in_conservation_area):
+        """
+        Sets whether the property is in a conservation area given the output of the ConservationAreaClient
+        :param in_conservation_area:  string value, indicating whether the property is in a conservation area
+        """
+        self.in_conservation_area = in_conservation_area
 
     def set_year_built(self):
         """

model_data/app.py

@@ -67,7 +67,8 @@ def handler():
 
     # Check if the property is in a conversation area
     for p in input_properties:
-        p.set_is_in_conservation_area(conservation_area_client)
+        in_conservation_area = conservation_area_client.is_in_conservation_area(p.coordinates)
+        p.set_is_in_conservation_area(in_conservation_area)
 
     local_authorities = {p.data['local-authority'] for p in input_properties}
     # TODO: Do this at a constituency level

model_data/requirements/dev.txt

@@ -0,0 +1,6 @@
+python-dotenv
+pytest
+mock
+pytest-cov
+pytest-mock
+pip-check-reqs

model_data/requirements/requirements.txt

@@ -1,18 +1,15 @@
+pandas==2.0.3
+numpy==1.25.1
+pytz==2023.3
+tzdata==2023.3
 epc-api-python==1.0.2
-python-dotenv
 tqdm
-pandas
 mypy
-pytest
-mock
-pytest-cov
-pytest-mock
 fuzzywuzzy
 python-Levenshtein
 dbfread
 pyproj
 pint
-geopandas
 mip
 seaborn
 statsmodels

model_data/requirements/static.txt

@@ -0,0 +1 @@
+geopandas