Trying to get /backend and /model_data working together

.idea/Model.iml

@@ -1,8 +1,11 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <module type="PYTHON_MODULE" version="4">
   <component name="NewModuleRootManager">
-    <content url="file://$MODULE_DIR$" />
-    <orderEntry type="jdk" jdkName="Python 3.10 (hestia-data)" jdkType="Python SDK" />
+    <content url="file://$MODULE_DIR$">
+      <sourceFolder url="file://$MODULE_DIR$/backend" isTestSource="false" />
+      <sourceFolder url="file://$MODULE_DIR$/model_data" isTestSource="false" />
+    </content>
+    <orderEntry type="jdk" jdkName="Python 3.10 (BackendApi)" jdkType="Python SDK" />
     <orderEntry type="sourceFolder" forTests="false" />
   </component>
 </module>

.idea/misc.xml

@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <project version="4">
-  <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.10 (hestia-data)" project-jdk-type="Python SDK" />
+  <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.10 (BackendApi)" project-jdk-type="Python SDK" />
   <component name="PythonCompatibilityInspectionAdvertiser">
     <option name="version" value="3" />
   </component>

backend/app/config.py

@@ -8,9 +8,10 @@ class Settings(BaseSettings):
     SECRET_KEY: str
     ENVIRONMENT: str
     PLAN_TRIGGER_BUCKET: str
+    EPC_AUTH_TOKEN: str
 
     class Config:
-        env_file = ".env"
+        env_file = "backend/.env"
 
 
 @lru_cache()

backend/app/dependencies.py

@@ -6,9 +6,10 @@ from cryptography.hazmat.primitives import hashes
 from cryptography.hazmat.backends import default_backend
 from typing import Any
 import json
-from app.config import get_settings
-from app.utils import logger
+from backend.app.config import get_settings
+from backend.app.utils import setup_logger
 
+logger = setup_logger()
 
 api_key_header = APIKeyHeader(name=get_settings().API_KEY_NAME, auto_error=False)
 oauth2_scheme = OAuth2PasswordBearer(tokenUrl="token")
@@ -94,7 +95,6 @@ def validate_jwt_token(token: str = Depends(oauth2_scheme)):
 
 
 async def validate_token(token: str = Depends(oauth2_scheme), request: Request = None):
-    print("VALIDATING - PRINT")
     logger.info("Validating token")
     logger.info(token)
     logger.info("Secret")

backend/app/main.py

@@ -1,19 +1,18 @@
 from fastapi import FastAPI, Depends
 from mangum import Mangum
-from app.portfolio import router as portfolio_router
-from app.plan import router as plan_router
-from app.dependencies import validate_api_key
-from app.config import get_settings
-
+from backend.app.portfolio import router as portfolio_router
+from backend.app.plan import router as plan_router
+from backend.app.dependencies import validate_api_key
+from backend.app.config import get_settings
 
 app = FastAPI(dependencies=[Depends(validate_api_key)])
 
-
 app.include_router(portfolio_router.router, prefix="/v1")
 app.include_router(plan_router.router, prefix="/v1")
 
 if get_settings().ENVIRONMENT == "local":
     from app.local import router as local_router
+
     app.include_router(local_router.router)
 
 handler = Mangum(app)

backend/app/plan/router.py

@@ -1,9 +1,12 @@
 from fastapi import APIRouter, Depends
-from app.dependencies import validate_token
-from app.plan.schemas import PlanTriggerRequest
-from app.utils import read_csv_from_s3, logger
-from app.config import get_settings
+from backend.app.dependencies import validate_token
+from backend.app.plan.schemas import PlanTriggerRequest
+from backend.app.utils import read_csv_from_s3, setup_logger
+from backend.app.config import get_settings
+from model_data.Property import Property
+from epc_api.client import EpcClient
 
+logger = setup_logger()
 
 router = APIRouter(
     prefix="/plan",
@@ -18,13 +21,25 @@ async def trigger_plan(body: PlanTriggerRequest):
     logger.info("Getting the inputs")
     # Read in the trigger file from s3
     bucket_name = get_settings().PLAN_TRIGGER_BUCKET
-    logger.info("bucket_name: ", bucket_name)
-    logger.info("body.trigger_file_path: ", body.trigger_file_path)
     plan_input = read_csv_from_s3(bucket_name=bucket_name, filepath=body.trigger_file_path)
-    logger.info("Got the inputs")
-    logger.info(plan_input)
+    print(plan_input)
 
-    # TODO: Parse the file
-    # TODO: Put messages on the queue
+    epc_client = EpcClient(auth_token=get_settings().EPC_AUTH_TOKEN)
+    input_properties = [
+        Property(postcode=config['postcode'], address1=config['address'], epc_client=epc_client)
+        for config in plan_input
+    ]
+
+    logger.info("Getting EPC data")
+    for p in input_properties:
+        p.search_address_epc()
+        p.set_year_built()
+
+    logger.info("Parsing and validating the file")
+    # TODO: Add validation
+    logger.info("properties")
+    logger.info(input_properties)
+
+    logger.info("Reading in EPC data")
 
     return {"message": "Plan triggered"}

backend/app/portfolio/router.py

@@ -1,5 +1,5 @@
 from fastapi import APIRouter, Depends
-from app.dependencies import validate_token
+from backend.app.dependencies import validate_token
 
 router = APIRouter(
     prefix="/portfolio",

backend/app/utils.py

@@ -6,8 +6,38 @@ import secrets
 import logging
 
 
-logger = logging.getLogger(__name__)
-logger.setLevel(logging.INFO)
+def setup_logger(log_file=None, level=logging.INFO, overwrite_handler=False):
+    # Create a logger and set the logging level
+    logger = logging.getLogger()
+    logger.setLevel(level)
+
+    # if logger already has handlers, just return it
+    if logger.hasHandlers() and not overwrite_handler:
+        return logger
+
+    # Define the log message format
+    log_format = "%(asctime)s [%(levelname)s] %(message)s"
+    date_format = "%Y-%m-%d %H:%M:%S"
+    formatter = logging.Formatter(log_format, datefmt=date_format)
+
+    # Create a file handler and set the file path and format
+    if log_file:
+        file_handler = logging.FileHandler(log_file)
+        file_handler.setLevel(level)
+        file_handler.setFormatter(formatter)
+        logger.addHandler(file_handler)
+
+    # Create a console handler and set the format
+    console_handler = logging.StreamHandler()
+    console_handler.setLevel(level)
+
+    # Set the formatter for the handlers
+    console_handler.setFormatter(formatter)
+
+    # Add the handlers to the logger
+    logger.addHandler(console_handler)
+
+    return logger
 
 
 def read_csv_from_s3(bucket_name, filepath):

backend/requirements/lambda.txt

@@ -1 +0,0 @@
-cryptography==41.0.2

backend/requirements/local.txt

@@ -0,0 +1,28 @@
+anyio==3.7.1
+cffi==1.15.1
+click==8.1.3
+cryptography==37.0.4
+ecdsa==0.18.0
+exceptiongroup==1.1.2
+fastapi==0.99.1
+h11==0.14.0
+httptools==0.5.0
+idna==3.4
+mangum==0.17.0
+pyasn1==0.5.0
+pycparser==2.21
+pydantic==1.10.11
+PyJWT==2.7.0
+python-dotenv==1.0.0
+python-jose==3.3.0
+PyYAML==6.0
+rsa==4.9
+six==1.16.0
+sniffio==1.3.0
+starlette==0.27.0
+typing_extensions==4.7.1
+uvicorn==0.22.0
+uvloop==0.17.0
+watchfiles==0.19.0
+websockets==11.0.3
+boto3

model_data/requirements/dev.txt

@@ -0,0 +1,5 @@
+python-dotenv
+pytest
+mock
+pytest-cov
+pytest-mock

model_data/requirements/requirements.txt

@@ -1,12 +1,10 @@
+pandas==2.0.3
+numpy==1.25.1
+pytz==2023.3
+tzdata==2023.3
 epc-api-python==1.0.2
-python-dotenv
 tqdm
-pandas
 mypy
-pytest
-mock
-pytest-cov
-pytest-mock
 fuzzywuzzy
 python-Levenshtein
 dbfread