Add MagicPlan Lambda Dockerfile, CI/CD jobs, and SQS IAM wiring in hubspot_deal_etl

.github/workflows/deploy_terraform.yml

@@ -537,11 +537,49 @@ jobs:
       AWS_SECRET_ACCESS_KEY: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }}
       AWS_REGION: ${{ secrets.DEV_AWS_REGION }}
 
+  # ============================================================
+  # Build MagicPlan Lambda image
+  # ============================================================
+  magic_plan_image:
+    needs: [determine_stage, shared_terraform]
+    uses: ./.github/workflows/_build_image.yml
+    with:
+      ecr_repo: magic-plan-${{ needs.determine_stage.outputs.stage }}
+      dockerfile_path: backend/magic_plan/handler/Dockerfile
+      build_context: .
+    secrets:
+      AWS_ACCESS_KEY_ID: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }}
+      AWS_REGION: ${{ secrets.DEV_AWS_REGION }}
+
+  # ============================================================
+  # Deploy MagicPlan Lambda
+  # ============================================================
+  magic_plan_lambda:
+    needs: [magic_plan_image, determine_stage]
+    uses: ./.github/workflows/_deploy_lambda.yml
+    with:
+      lambda_name: magic_plan
+      lambda_path: infrastructure/terraform/lambda/magic_plan
+      stage: ${{ needs.determine_stage.outputs.stage }}
+      ecr_repo: magic-plan-${{ needs.determine_stage.outputs.stage }}
+      image_digest: ${{ needs.magic_plan_image.outputs.image_digest }}
+      terraform_apply: ${{ needs.determine_stage.outputs.terraform_apply }}
+    secrets:
+      AWS_ACCESS_KEY_ID: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }}
+      AWS_REGION: ${{ secrets.DEV_AWS_REGION }}
+      TF_VAR_db_host: ${{ secrets.DEV_DB_HOST }}
+      TF_VAR_db_name: ${{ secrets.DEV_DB_NAME }}
+      TF_VAR_db_port: ${{ secrets.DEV_DB_PORT }}
+      TF_VAR_magicplan_customer_id: ${{ secrets.MAGICPLAN_CUSTOMER_ID }}
+      TF_VAR_magicplan_api_key: ${{ secrets.MAGICPLAN_API_KEY }}
+
   # ============================================================
   # Deploy Hubspot ETL Lambda
   # ============================================================
   hubspot_etl_lambda:
-    needs: [hubspot_etl_image, determine_stage, pashub_to_ara_lambda]
+    needs: [hubspot_etl_image, determine_stage, pashub_to_ara_lambda, magic_plan_lambda]
     uses: ./.github/workflows/_deploy_lambda.yml
     with:
       lambda_name: hubspot-etl-to-ara

backend/magic_plan/handler/Dockerfile

@@ -0,0 +1,26 @@
+FROM mcr.microsoft.com/playwright/python:v1.58.0-jammy
+
+# Install AWS Lambda RIE
+ADD https://github.com/aws/aws-lambda-runtime-interface-emulator/releases/latest/download/aws-lambda-rie /usr/local/bin/aws-lambda-rie
+RUN chmod +x /usr/local/bin/aws-lambda-rie
+
+# Set working directory (Lambda task root)
+WORKDIR /var/task
+
+COPY backend/magic_plan/handler/requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+
+COPY utils/ utils/
+COPY backend/ backend/
+COPY datatypes/ datatypes/
+
+# Local lambda entrypoint
+# ENTRYPOINT ["/usr/local/bin/aws-lambda-rie", "python", "-m", "awslambdaric"]
+
+# AWS lambda entrypoint
+ENTRYPOINT ["python", "-m", "awslambdaric"]
+
+# -----------------------------
+# Lambda handler
+# -----------------------------
+CMD ["backend.magic_plan.handler.handler"]

backend/magic_plan/handler/requirements.txt

@@ -0,0 +1,7 @@
+awslambdaric
+requests
+sqlalchemy==2.0.36
+sqlmodel
+psycopg2-binary==2.9.10
+pydantic-settings==2.6.0
+boto3==1.35.44

infrastructure/terraform/lambda/hubspot_deal_etl/main.tf

@@ -12,7 +12,16 @@ data "terraform_remote_state" "pashub_to_ara" {
   config = {
     bucket = "pashub-to-ara-terraform-state"
     key    = "env:/${var.stage}/terraform.tfstate"
-    region = "eu-west-2" 
+    region = "eu-west-2"
+  }
+}
+
+data "terraform_remote_state" "magic_plan" {
+  backend = "s3"
+  config = {
+    bucket = "magic-plan-hubspot-trigger-terraform-state"
+    key    = "env:/${var.stage}/terraform.tfstate"
+    region = "eu-west-2"
   }
 }
 
@@ -49,6 +58,7 @@ module "hubspot_deal_etl" {
     HUBSPOT_API_KEY = var.hubspot_api_key
 
     PASHUB_TO_ARA_SQS_URL = data.terraform_remote_state.pashub_to_ara.outputs.pashub_to_ara_queue_url
+    MAGICPLAN_SQS_URL     = data.terraform_remote_state.magic_plan.outputs.magic_plan_queue_url
   }
 }
 
@@ -76,4 +86,18 @@ module "hubspot_deal_etl_sqs_policy" {
 resource "aws_iam_role_policy_attachment" "hubspot_deal_etl_sqs_send" {
   role       = module.hubspot_deal_etl.role_name
   policy_arn = module.hubspot_deal_etl_sqs_policy.policy_arn
+}
+
+module "hubspot_deal_etl_magicplan_sqs_policy" {
+  source = "../../modules/general_iam_policy"
+
+  policy_name        = "hubspot-deal-etl-magicplan-sqs-send-${var.stage}"
+  policy_description = "Allow HubSpot ETL Lambda to send messages to MagicPlan queue"
+  actions            = ["sqs:SendMessage"]
+  resources          = [data.terraform_remote_state.magic_plan.outputs.magic_plan_queue_arn]
+}
+
+resource "aws_iam_role_policy_attachment" "hubspot_deal_etl_magicplan_sqs_send" {
+  role       = module.hubspot_deal_etl.role_name
+  policy_arn = module.hubspot_deal_etl_magicplan_sqs_policy.policy_arn
 }