Merge 3dc14480e9 into 133190f093

Feature/ml prediction buckets

.github/workflows/deploy_fastapi_backend.yml

@@ -77,6 +77,9 @@ jobs:
           echo "::set-output name=hotwater_kwh_predictions_bucket::${{ secrets[format('{0}_HOTWATER_KWH_PREDICTIONS_BUCKET', github.ref_name)] }}"
           echo "::set-output name=energy_asessments_bucket::${{ secrets[format('{0}_ENERGY_ASSESSMENTS_BUCKET', github.ref_name)] }}"
           echo "::set-output name=google_solar_api_key::${{ secrets[format('{0}_GOOGLE_SOLAR_API_KEY', github.ref_name)] }}"
+          echo "::set-output name=sap_baseline_predictions_bucket::${{ secrets[format('{0}_SAP_BASELINE_PREDICTIONS_BUCKET', github.ref_name)] }}"
+          echo "::set-output name=carbon_baseline_predictions_bucket::${{ secrets[format('{0}_CARBON_BASELINE_PREDICTIONS_BUCKET', github.ref_name)] }}"
+          echo "::set-output name=heat_baseline_predictions_bucket::${{ secrets[format('{0}_HEAT_BASELINE_PREDICTIONS_BUCKET', github.ref_name)] }}"
 
       - name: Setup Docker
         uses: docker/setup-buildx-action@v1
@@ -129,6 +132,9 @@ jobs:
           DB_NAME: ${{ steps.set_db_credentials.outputs.db_name }}
           ECR_URI: ${{ steps.set_ecr_credentials.outputs.ecr_uri }}
           GITHUB_SHA: ${{ github.sha }}
+          SAP_BASELINE_PREDICTIONS_BUCKET: ${{ steps.set_api_secrets.outputs.sap_baseline_predictions_bucket }}
+          CARBON_BASELINE_PREDICTIONS_BUCKET: ${{ steps.set_api_secrets.outputs.carbon_baseline_predictions_bucket }}
+          HEAT_BASELINE_PREDICTIONS_BUCKET: ${{ steps.set_api_secrets.outputs.heat_baseline_predictions_bucket }}
         run: |
           # Fetch database credentials from AWS Secrets Manager
           SECRET_VALUE=$(aws secretsmanager get-secret-value --secret-id ${{ github.ref_name }}/assessment_model/db_credentials --query SecretString)

backend/pashub_fetcher/handler/handler.py

@@ -1,54 +1,26 @@
-from typing import Any, Dict, Mapping
-
-from playwright.sync_api import sync_playwright, TimeoutError as PlaywrightTimeoutError
+from typing import Any, List, Mapping
 
+from backend.pashub_fetcher.cotality_client import CotalityClient
+from backend.pashub_fetcher.token_getter import get_token_from_local_storage
 from utils.logger import setup_logger
 
 
 logger = setup_logger()
 
 
-def handler(event: Mapping[str, Any], context: Any) -> Dict[str, str]:
-    logger.info("Starting Playwright flow")
+def handler(event: Mapping[str, Any], context: Any) -> None:
+    pas_hub_email = "random@test.com"
+    pas_hub_password = "my_fake_password"
 
-    email = "random@test.com"
-    password = "my_fake_password"
+    try:
+        token: str = get_token_from_local_storage(pas_hub_email, pas_hub_password)
+        logger.info(f"Token extracted successfully: {token}")
+    except:
+        logger.error("Error getting auth token from Pas Hub")
+        raise
 
-    with sync_playwright() as p:
-        browser = p.chromium.launch(
-            headless=True,
-            args=["--no-sandbox", "--disable-dev-shm-usage"],
-        )
-        page = browser.new_page()
+    client = CotalityClient(token=token)
+    uprn = "100061885568"  # TODO: get from request body
 
-        try:
-            logger.info("Navigating to site...")
-            page.goto("https://pashub.net/", timeout=30000)
-
-            logger.info("Filling login form...")
-            page.fill("#email", email)
-            page.fill("#password", password)
-
-            logger.info("Submitting login...")
-            page.click("#btn-login")
-
-            page.wait_for_timeout(3000)
-
-            if "login" in page.url.lower():
-                logger.error("Login failed (still on login page)")
-                return {"status": "error", "message": "Login failed"}
-
-            logger.info(f"Login likely successful. URL: {page.url}")
-
-            return {"status": "ok"}
-
-        except PlaywrightTimeoutError as e:
-            logger.error(f"Timeout during login flow: {str(e)}")
-            return {"status": "error", "message": "Timeout during login"}
-
-        except Exception as e:
-            logger.error(f"Unexpected error: {str(e)}")
-            return {"status": "error", "message": str(e)}
-
-        finally:
-            browser.close()
+    saved_files: List[str] = client.get_evidence_files_by_uprn(uprn=uprn)
+    print(f"saved {len(saved_files)} files")

backend/pashub_fetcher/token_getter.py

@@ -0,0 +1,54 @@
+from playwright.sync_api import sync_playwright, TimeoutError as PlaywrightTimeoutError
+
+from utils.logger import setup_logger
+
+logger = setup_logger()
+
+
+def get_token_from_local_storage(email: str, password: str) -> str:
+    logger.info("Starting Playwright flow")
+
+    with sync_playwright() as p:
+        browser = p.chromium.launch(
+            headless=True,
+            args=["--no-sandbox", "--disable-dev-shm-usage"],
+        )
+        page = browser.new_page()
+
+        try:
+            logger.info("Navigating to site...")
+            page.goto("https://pashub.net/", timeout=30000)
+
+            logger.info("Filling login form...")
+            page.fill("#email", email)
+            page.fill("#password", password)
+
+            logger.info("Submitting login...")
+            page.click("#btn-login")
+
+            page.wait_for_timeout(3000)
+
+            if "login" in page.url.lower():
+                raise Exception("Login failed (still on login page)")
+
+            logger.info(f"Login likely successful. URL: {page.url}")
+
+            token = page.evaluate(
+                """() => {
+                    return localStorage.getItem('token');
+                }"""
+            )
+
+            if not token:
+                raise Exception("Login succeeded but no token found")
+
+            return token
+
+        except PlaywrightTimeoutError as e:
+            raise Exception(f"Timeout during login flow: {str(e)}")
+
+        except Exception as e:
+            raise Exception(f"Unexpected error: {str(e)}")
+
+        finally:
+            browser.close()

infrastructure/terraform/lambda/fast-api/main.tf

@@ -5,7 +5,7 @@ data "terraform_remote_state" "shared" {
   backend = "s3"
   config = {
     bucket = "assessment-model-terraform-state"
-    key = "env:/${var.stage}/terraform.tfstate"
+    key    = "env:/${var.stage}/terraform.tfstate"
     region = "eu-west-2"
   }
 }
@@ -14,7 +14,7 @@ data "terraform_remote_state" "engine" {
   backend = "s3"
   config = {
     bucket = "ara-engine-terraform-state",
-    key = "env:/${var.stage}/terraform.tfstate"
+    key    = "env:/${var.stage}/terraform.tfstate"
     region = "eu-west-2"
   }
 }
@@ -23,7 +23,7 @@ data "terraform_remote_state" "categorisation" {
   backend = "s3"
   config = {
     bucket = "categorisation-terraform-state",
-    key = "env:/${var.stage}/terraform.tfstate"
+    key    = "env:/${var.stage}/terraform.tfstate"
     region = "eu-west-2"
   }
 }
@@ -43,26 +43,26 @@ locals {
 # FastAPI Lambda + API Gateway
 ############################################
 module "fastapi" {
-  source     = "../../modules/lambda_with_api_gateway"
+  source = "../../modules/lambda_with_api_gateway"
 
-  name        = "fastapi"
-  stage       = var.stage
-  source_dir  = "${path.root}/../../../../"
-  handler     = "backend.app.main.handler"
-  runtime     = "python3.11"
-  timeout     = 600
-  memory_size = 512
-  artifact_bucket = data.terraform_remote_state.shared.outputs.ara_fast_api_state_bucket
+  name              = "fastapi"
+  stage             = var.stage
+  source_dir        = "${path.root}/../../../../"
+  handler           = "backend.app.main.handler"
+  runtime           = "python3.11"
+  timeout           = 600
+  memory_size       = 512
+  artifact_bucket   = data.terraform_remote_state.shared.outputs.ara_fast_api_state_bucket
   requirements_file = "${path.root}/../../../../backend/app/requirements/requirements.txt"
 
   domain_name = "api.${var.domain_name}"
 
   environment = {
-    ENVIRONMENT    = var.stage
-    API_KEY        = var.api_key
-    SECRET_KEY     = var.secret_key
+    ENVIRONMENT = var.stage
+    API_KEY     = var.api_key
+    SECRET_KEY  = var.secret_key
     # DOMAIN_NAME    = var.domain_name
-    EPC_AUTH_TOKEN = var.epc_auth_token
+    EPC_AUTH_TOKEN       = var.epc_auth_token
     GOOGLE_SOLAR_API_KEY = var.google_solar_api_key
 
     DB_HOST     = var.db_host
@@ -71,14 +71,17 @@ module "fastapi" {
     DB_USERNAME = local.db_credentials.db_assessment_model_username
     DB_PASSWORD = local.db_credentials.db_assessment_model_password
 
-    PLAN_TRIGGER_BUCKET             = data.terraform_remote_state.shared.outputs.retrofit_plan_trigger_bucket_name
-    DATA_BUCKET                     = data.terraform_remote_state.shared.outputs.retrofit_sap_data_bucket_name
-    SAP_PREDICTIONS_BUCKET          = data.terraform_remote_state.shared.outputs.retrofit_sap_predictions_bucket_name
-    CARBON_PREDICTIONS_BUCKET       = data.terraform_remote_state.shared.outputs.retrofit_carbon_predictions_bucket_name
-    HEAT_PREDICTIONS_BUCKET         = data.terraform_remote_state.shared.outputs.retrofit_heat_predictions_bucket_name
-    HEATING_KWH_PREDICTIONS_BUCKET  = data.terraform_remote_state.shared.outputs.retrofit_heating_kwh_predictions_bucket_name
-    HOTWATER_KWH_PREDICTIONS_BUCKET = data.terraform_remote_state.shared.outputs.retrofit_hotwater_kwh_predictions_bucket_name
-    ENERGY_ASSESSMENTS_BUCKET       = data.terraform_remote_state.shared.outputs.retrofit_energy_assessments_bucket_name
+    PLAN_TRIGGER_BUCKET                = data.terraform_remote_state.shared.outputs.retrofit_plan_trigger_bucket_name
+    DATA_BUCKET                        = data.terraform_remote_state.shared.outputs.retrofit_sap_data_bucket_name
+    SAP_PREDICTIONS_BUCKET             = data.terraform_remote_state.shared.outputs.retrofit_sap_predictions_bucket_name
+    CARBON_PREDICTIONS_BUCKET          = data.terraform_remote_state.shared.outputs.retrofit_carbon_predictions_bucket_name
+    HEAT_PREDICTIONS_BUCKET            = data.terraform_remote_state.shared.outputs.retrofit_heat_predictions_bucket_name
+    HEATING_KWH_PREDICTIONS_BUCKET     = data.terraform_remote_state.shared.outputs.retrofit_heating_kwh_predictions_bucket_name
+    HOTWATER_KWH_PREDICTIONS_BUCKET    = data.terraform_remote_state.shared.outputs.retrofit_hotwater_kwh_predictions_bucket_name
+    ENERGY_ASSESSMENTS_BUCKET          = data.terraform_remote_state.shared.outputs.retrofit_energy_assessments_bucket_name
+    SAP_BASELINE_PREDICTIONS_BUCKET    = data.terraform_remote_state.shared.outputs.retrofit_sap_baseline_predictions_bucket_name
+    CARBON_BASELINE_PREDICTIONS_BUCKET = data.terraform_remote_state.shared.outputs.retrofit_carbon_baseline_predictions_bucket_name
+    HEAT_BASELINE_PREDICTIONS_BUCKET   = data.terraform_remote_state.shared.outputs.retrofit_heat_baseline_predictions_bucket_name
 
     ENGINE_SQS_URL         = data.terraform_remote_state.engine.outputs.ara_engine_queue_url
     CATEGORISATION_SQS_URL = data.terraform_remote_state.categorisation.outputs.categorisation_queue_url
@@ -121,4 +124,4 @@ resource "aws_iam_role_policy_attachment" "fastapi_sqs_send" {
 resource "aws_iam_role_policy_attachment" "fastapi_s3_read_and_write" {
   role       = module.fastapi.role_name
   policy_arn = data.terraform_remote_state.shared.outputs.fast_api_s3_read_and_write_arn
-}
+}

infrastructure/terraform/shared/main.tf

@@ -239,6 +239,33 @@ module "retrofit_sap_baseline_predictions" {
   allowed_origins = var.allowed_origins
 }
 
+output "retrofit_sap_baseline_predictions_bucket_name" {
+  value       = module.retrofit_sap_baseline_predictions.bucket_name
+  description = "Name of the retrofit SAP baseline predictions bucket"
+}
+
+module "retrofit_carbon_baseline_predictions" {
+  source          = "../modules/s3"
+  bucketname      = "retrofit-carbon-baseline-predictions-${var.stage}"
+  allowed_origins = var.allowed_origins
+}
+
+output "retrofit_carbon_baseline_predictions_bucket_name" {
+  value       = module.retrofit_carbon_baseline_predictions.bucket_name
+  description = "Name of the retrofit carbon baseline predictions bucket"
+}
+
+module "retrofit_heat_baseline_predictions" {
+  source          = "../modules/s3"
+  bucketname      = "retrofit-heat-baseline-predictions-${var.stage}"
+  allowed_origins = var.allowed_origins
+}
+
+output "retrofit_heat_baseline_predictions_bucket_name" {
+  value       = module.retrofit_heat_baseline_predictions.bucket_name
+  description = "Name of the retrofit heat baseline predictions bucket"
+}
+
 // We make this bucket presignable, because we want to generate download links for the frontend
 module "retrofit_energy_assessments" {
   source          = "../modules/s3_presignable_bucket"
@@ -526,7 +553,10 @@ module "engine_s3_read_and_write" {
     "arn:aws:s3:::${module.retrofit_heat_predictions.bucket_name}",
     "arn:aws:s3:::${module.retrofit_heating_kwh_predictions.bucket_name}",
     "arn:aws:s3:::${module.retrofit_hotwater_kwh_predictions.bucket_name}",
-    "arn:aws:s3:::${module.retrofit_energy_assessments.bucket_name}"
+    "arn:aws:s3:::${module.retrofit_energy_assessments.bucket_name}",
+    "arn:aws:s3:::${module.retrofit_sap_baseline_predictions.bucket_name}",
+    "arn:aws:s3:::${module.retrofit_carbon_baseline_predictions.bucket_name}",
+    "arn:aws:s3:::${module.retrofit_heat_baseline_predictions.bucket_name}"
   ]
   actions        = ["s3:*"]
   resource_paths = ["/*"]

serverless.yml

@@ -28,6 +28,9 @@ provider:
     HOTWATER_KWH_PREDICTIONS_BUCKET: ${env:HOTWATER_KWH_PREDICTIONS_BUCKET}
     ENERGY_ASSESSMENTS_BUCKET: ${env:ENERGY_ASSESSMENTS_BUCKET}
     GOOGLE_SOLAR_API_KEY: ${env:GOOGLE_SOLAR_API_KEY}
+    SAP_BASELINE_PREDICTIONS_BUCKET: ${env:SAP_BASELINE_PREDICTIONS_BUCKET}
+    CARBON_BASELINE_PREDICTIONS_BUCKET: ${env:CARBON_BASELINE_PREDICTIONS_BUCKET}
+    HEAT_BASELINE_PREDICTIONS_BUCKET: ${env:HEAT_BASELINE_PREDICTIONS_BUCKET}
     ENGINE_SQS_URL:
       Ref: EngineQueue
     # hardcode the categorisation queue for now as it's created in terraform
@@ -177,3 +180,9 @@ resources:
                     - arn:aws:s3:::${env:HEATING_KWH_PREDICTIONS_BUCKET}/*
                     - arn:aws:s3:::${env:HOTWATER_KWH_PREDICTIONS_BUCKET}
                     - arn:aws:s3:::${env:HOTWATER_KWH_PREDICTIONS_BUCKET}/*
+                    - arn:aws:s3:::${env:SAP_BASELINE_PREDICTIONS_BUCKET}
+                    - arn:aws:s3:::${env:SAP_BASELINE_PREDICTIONS_BUCKET}/*
+                    - arn:aws:s3:::${env:CARBON_BASELINE_PREDICTIONS_BUCKET}
+                    - arn:aws:s3:::${env:CARBON_BASELINE_PREDICTIONS_BUCKET}/*
+                    - arn:aws:s3:::${env:HEAT_BASELINE_PREDICTIONS_BUCKET}
+                    - arn:aws:s3:::${env:HEAT_BASELINE_PREDICTIONS_BUCKET}/*