start defining infrastructure including ecr

backend/app/db/models/uploaded_file.py

@@ -39,9 +39,21 @@ class UploadedFile(Base):
     hubspot_listing_id = Column(BigInteger, nullable=True)
 
     file_type = Column(
-        SqlEnum(FileTypeEnum, name="file_type", create_type=False), nullable=True
+        SqlEnum(
+            FileTypeEnum,
+            name="file_type",
+            create_type=False,
+            values_callable=lambda enum_cls: [e.value for e in enum_cls],
+        ),
+        nullable=True,
     )
 
     file_source = Column(
-        SqlEnum(FileSourceEnum, name="file_source", create_type=False), nullable=True
+        SqlEnum(
+            FileSourceEnum,
+            name="file_source",
+            create_type=False,
+            values_callable=lambda enum_cls: [e.value for e in enum_cls],
+        ),
+        nullable=True,
     )

backend/ecmk_fetcher/.~lock.hubspot-crm-exports-southern-ra-lite-programme-3103-2026-03-31-2.xlsx#

@@ -1 +0,0 @@
-,daniel,daniel-Dell-15-DC15250,07.04.2026 11:47,/home/daniel/snap/onlyoffice-desktopeditors/1067/.local/share/onlyoffice;

backend/ecmk_fetcher/handler/Dockerfile

@@ -0,0 +1,26 @@
+FROM mcr.microsoft.com/playwright/python:v1.58.0-jammy
+
+# Install AWS Lambda RIE
+ADD https://github.com/aws/aws-lambda-runtime-interface-emulator/releases/latest/download/aws-lambda-rie /usr/local/bin/aws-lambda-rie
+RUN chmod +x /usr/local/bin/aws-lambda-rie
+
+# Set working directory (Lambda task root)
+WORKDIR /var/task
+
+COPY backend/ecmk_fetcher/handler/requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+
+COPY utils/ utils/
+COPY backend/ backend/
+COPY datatypes/ datatypes/
+
+# Local lambda entrypoint 
+# ENTRYPOINT ["/usr/local/bin/aws-lambda-rie", "python", "-m", "awslambdaric"]
+
+#AWS lambda entrypoint
+ENTRYPOINT ["python", "-m", "awslambdaric"]
+
+# -----------------------------
+# Lambda handler
+# -----------------------------
+CMD ["backend.ecmk_fetcher.handler.handler.handler"]

backend/ecmk_fetcher/handler/handler.py

@@ -1,9 +1,13 @@
 from typing import Any, Mapping
 
 from backend.ecmk_fetcher.processor import run_job
+from utils.logger import setup_logger
+
+logger = setup_logger()
 
 
 def handler(event: Mapping[str, Any], context: Any) -> None:
+    logger.info("Entered handler")
     run_job()
 
 

backend/ecmk_fetcher/handler/requirements.txt

@@ -0,0 +1,12 @@
+awslambdaric
+playwright==1.58.0
+msal
+openpyxl
+sqlalchemy==2.0.36
+sqlmodel
+pytz==2024.2
+psycopg2-binary==2.9.10
+pydantic-settings==2.6.0
+boto3==1.35.44
+pandas==2.2.2
+numpy<2.0

backend/ecmk_fetcher/local_handler/docker-compose.yml

@@ -0,0 +1,11 @@
+version: "3.9"
+
+services:
+  ecmk-fetcher-lambda:
+    build:
+      context: ../../../
+      dockerfile: backend/ecmk_fetcher/handler/Dockerfile
+    ports:
+      - "9000:8080"
+    env_file:
+      - ../../../.env

backend/ecmk_fetcher/local_handler/invoke_local_lambda.py

@@ -0,0 +1,26 @@
+#!/usr/bin/env python3
+import json
+import requests
+
+HOST = "localhost"
+PORT = "9000"
+
+LAMBDA_URL = f"http://{HOST}:{PORT}/2015-03-31/functions/function/invocations"
+
+payload = {
+    "Records": [
+        {
+            "body": json.dumps(
+                {
+                    "test": 123456,
+                }
+            )
+        }
+    ]
+}
+
+response = requests.post(LAMBDA_URL, json=payload)
+
+print("Status code:", response.status_code)
+print("Response:")
+print(response.text)

backend/ecmk_fetcher/processor.py

@@ -41,7 +41,8 @@ logger = setup_logger()
 
 
 def run_job() -> None:
-    username: str = ""
+
+    username: str = ""  # TODO: get from github secrets
     password: str = ""
 
     property_list_file: str = (

backend/ecmk_fetcher/upload.py

@@ -31,7 +31,9 @@ def upload_file_to_sharepoint(
 def upload_file_to_s3_and_update_db(
     bucket: str, file_path: str, hubspot_listing_id: str, file_type: FileTypeEnum
 ) -> None:
-    key: str = f"documents/hubspot_listing_id/{hubspot_listing_id}"
+    filename: str = os.path.basename(file_path)
+    key: str = f"documents/hubspot_listing_id/{hubspot_listing_id}/{filename}"
+
     upload_file_to_s3(file_path, bucket, key)
 
     uploaded_file = UploadedFile(
@@ -40,7 +42,7 @@ def upload_file_to_s3_and_update_db(
         s3_upload_timestamp=datetime.now(timezone.utc),
         hubspot_listing_id=hubspot_listing_id,
         file_source=FileSourceEnum.ECMK.value,
-        file_type=file_type,
+        file_type=file_type.value,
     )
 
     with db_session() as session:

infrastructure/terraform/lambda/ecmk_to_ara/main.tf

@@ -0,0 +1,27 @@
+data "terraform_remote_state" "shared" {
+  backend = "s3"
+  config = {
+    bucket = "assessment-model-terraform-state"
+    key = "env:/${var.stage}/terraform.tfstate"
+    region = "eu-west-2"
+  }
+}
+
+module "lambda" {
+  source = "../../modules/lambda_with_sqs"
+
+  name  = "ecmk_to_ara" #"address2uprn" for example
+  stage = var.stage
+
+  image_uri = local.image_uri
+
+  # Optional: Set maximum_concurrency to limit concurrent SQS-triggered invocations (2-1000)
+  maximum_concurrency = var.maximum_concurrency
+
+  batch_size = var.batch_size
+
+  environment = {
+    STAGE = var.stage
+    LOG_LEVEL = "info"
+  }
+}

infrastructure/terraform/lambda/ecmk_to_ara/provider.tf

@@ -0,0 +1,16 @@
+terraform {
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = ">= 5.0"
+    }
+  }
+
+  backend "s3" {
+    bucket = "ecmk-to-ara-terraform-state"
+    key    = "terraform.tfstate"
+    region = "eu-west-2"
+  }
+
+  required_version = ">= 1.2.0"
+}

infrastructure/terraform/lambda/ecmk_to_ara/variables.tf

@@ -0,0 +1,37 @@
+variable "lambda_name" {
+  type        = string
+  description = "Logical name of the lambda (e.g. address2uprn)"
+}
+
+variable "stage" {
+  description = "Deployment stage (e.g. dev, prod)"
+  type        = string
+}
+variable "ecr_repo_url" {
+  type        = string
+  description = "ECR repository URL (no tag, no digest)"
+}
+
+variable "image_digest" {
+  type        = string
+  description = "Image digest (sha256:...)"
+}
+
+variable "maximum_concurrency" {
+  type        = number
+  default     = 2
+  description = "Maximum number of concurrent Lambda invocations from SQS (2-1000). null = no limit."
+}
+
+variable "batch_size" {
+  type    = number
+  default = 1
+}
+
+locals {
+  image_uri = "${var.ecr_repo_url}@${var.image_digest}"
+}
+
+output "resolved_image_uri" {
+  value = local.image_uri
+}

infrastructure/terraform/shared/main.tf

@@ -538,6 +538,20 @@ module "pashub_to_ara_registry" {
   stage  = var.stage
 }
 
+################################################
+# ECMK to Ara – Lambda
+################################################
+module "ecmk_to_ara_state_bucket" {
+  source      = "../modules/tf_state_bucket"
+  bucket_name = "ecmk-to-ara-terraform-state"
+}
+
+module "ecmk_to_ara_registry" {
+  source = "../modules/container_registry"
+  name   = "ecmk_to_ara"
+  stage  = var.stage
+}
+
 ################################################
 # Engine – Lambda ECR
 ################################################