Merge pull request #137 from Hestia-Homes/feautre/fix_month_end

month end automation complete

.db-env

@@ -1,10 +1,10 @@
-POSTGRES_USER=postgres
+OSTGRES_USER=postgres
 PGDATABASE=surveyDB
 POSTGRES_PASSWORD=makingwarmhomes
 POSTGRES_HOST=localhost
 POSTGRES_PORT=5432
 PGADMIN_DEFAULT_EMAIL=junte@domna.homes
 PGADMIN_DEFAULT_PASSWORD=makingwarmhomes
-DATABASE_URL=postgresql://postgres:makingwarmhomes@db:5432/postgres
-# Prod(dev-aws) Database Don't use!!!!
-#DATABASE_URL=postgresql://postgres:makingwarmhomes@terraform-20250331175522503500000002.cdgzupxvdyp0.eu-west-2.rds.amazonaws.com:5432/surveyDB
+# DATABASE_URL=postgresql://postgres:makingwarmhomes@db:5432/postgres
+# Prod(dev-aws) Database Don't use
+DATABASE_URL=postgresql://postgres:makingwarmhomes@terraform-20250331175522503500000002.cdgzupxvdyp0.eu-west-2.rds.amazonaws.com:5432/surveyDB

.devcontainer/Dockerfile

@@ -3,6 +3,12 @@ FROM library/python:3.12-bullseye
 ARG USER=vscode
 ARG DEBIAN_FRONTEND=noninteractive
 
+# DO NOT PUSH IMAGE TO ECR!!! as anyone with access to image can log on to our aws
+# Will log on as aws Jun-te account, change in the future to development account
+ENV AWS_ACCESS_KEY_ID=AKIAU5A36PPNK7RXX52V
+ENV AWS_SECRET_ACCESS_KEY=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
+ENV AWS_DEFAULT_REGION=eu-west-2
+
 # Install system dependencies in a single layer
 RUN apt update && apt install -y --no-install-recommends \
     sudo jq vim curl\
@@ -10,28 +16,19 @@ RUN apt update && apt install -y --no-install-recommends \
     && rm -rf /var/lib/apt/lists/*
 
 # Create the user and grant sudo privileges
-RUN useradd -m -s /usr/bin/bash ${USER} \
+RUN useradd -m -s /bin/bash ${USER} \
     && echo "${USER} ALL=(ALL) NOPASSWD: ALL" >/etc/sudoers.d/${USER} \
     && chmod 0440 /etc/sudoers.d/${USER}
 
 # Install Poetry
 RUN pip install --no-cache-dir poetry
 
-# Download and install nvm:
-# RUN curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.3/install.sh | bash
 
-# # in lieu of restarting the shell
-# RUN \. "$HOME/.nvm/nvm.sh"
-
-# # Download and install Node.js:
-# RUN nvm install 22
-
-# # Verify the Node.js version:
-# RUN node -v # Should print "v22.16.0".
-# RUN nvm current # Should print "v22.16.0".
-
-# # Verify npm version:
-# RUN npm -v # Should print "10.9.2".
+# Install Node.js 22 (from NodeSource)
+RUN curl -fsSL https://deb.nodesource.com/setup_22.x | bash - \
+    && apt install -y nodejs \
+    && node -v \
+    && npm -v
 
 
 # Install aws
@@ -53,4 +50,4 @@ RUN terraform -install-autocomplete
 
 
 # Set the working directory
-WORKDIR /workspaces/survey-extractor:q
+WORKDIR /workspaces/survey-extractor

.devcontainer/devcontainer.json

@@ -5,10 +5,16 @@
   "remoteUser": "vscode",
   "workspaceFolder": "/workspaces/survey-extractor",
   "postStartCommand": "bash .devcontainer/post-install.sh",
+
+  "features": {
+    // "ghcr.io/devcontainers/features/ssh-agent:1": {}
+  },
+
   "mounts": [
-    "source=${localEnv:HOME},target=/workspaces/home,type=bind",
-    "source=${localEnv:HOME}/.aws/,target=/home/vscode/.aws/,type=bind"
+    // Optional convenience mount
+    "source=${localEnv:HOME},target=/workspaces/home,type=bind"
   ],
+
   "customizations": {
     "vscode": {
       "settings": {
@@ -22,8 +28,12 @@
         "lindacong.vscode-book-reader",
         "4ops.terraform",
         "fabiospampinato.vscode-todo-plus",
-        "jgclark.vscode-todo-highlight"
+        "jgclark.vscode-todo-highlight",
+        "corentinartaud.pdfpreview",
+        "GrapeCity.gc-excelviewer",
+        "anthropic.claude-code"
       ]
     }
   }
 }
+

.devcontainer/docker-compose.yml

@@ -11,39 +11,40 @@ services:
       - ../.db-env
     volumes:
       - ..:/workspaces/survey-extractor
-    depends_on:
-      - db
+    # depends_on:
+    #   - db
     networks:
       - survey-net
 
-  db:
-    image: postgres:17.4
-    restart: unless-stopped
-    ports:
-      - 5432:5432
-    env_file:
-      - ../.db-env
-    volumes:
-      - postgres-data:/var/lib/postgresql/data
-    networks:
-      - survey-net
+  # db:
+  #   image: postgres:17.4
+  #   restart: unless-stopped
+  #   ports:
+  #     - 5432:5432
+  #   env_file:
+  #     - ../.db-env
+  #   volumes:
+  #     - postgres-data-two:/var/lib/postgresql/data
+  #   networks:
+  #     - survey-net
 
-  pgadmin:
-    image: dpage/pgadmin4
-    hostname: pgadmin
-    ports:
-      - 5555:80
-    env_file:
-      - ../.db-env
-    restart: unless-stopped
-    depends_on:
-      - db
-    networks:
-      - survey-net
+  # pgadmin:
+  #   image: dpage/pgadmin4
+  #   hostname: pgadmin
+  #   ports:
+  #     - 5555:80
+  #   env_file:
+  #     - ../.db-env
+  #   restart: unless-stopped
+  #   depends_on:
+  #     - db
+  #   networks:
+  #     - survey-net
 
 networks:
   survey-net:
     driver: bridge
 
 volumes:
-  postgres-data:
+  postgres-data-two:
+

.github/workflows/actions/lambda-deploy/action.yml

@@ -0,0 +1,86 @@
+name: "Build and Push Lambda Image to ECR"
+description: "Reusable action for building and pushing lambda Docker image to ECR"
+
+inputs:
+  ecr_name:
+    description: "Lambda name / ECR repo name"
+    required: true
+  dockerfile_path:
+    description: "Path to Dockerfile"
+    required: true
+  ecr_tf_dir:
+    description: "Path to ECR terraform directory"
+    required: true
+  lambda_tf_dir:
+    description: "Path to Lambda terraform directory"
+    required: true
+  aws-access-key-id:
+    description: "AWS access key"
+    required: true
+  aws-secret-access-key:
+    description: "AWS secret key"
+    required: true
+  aws-region:
+    description: "AWS region"
+    required: true
+  git-sha:
+    description: "Git commit SHA"
+    required: true
+  git-ref:
+    description: "Git ref name"
+    required: true
+
+runs:
+  using: "composite"
+  steps:
+    - uses: actions/checkout@v4
+
+    - name: Configure AWS credentials
+      uses: aws-actions/configure-aws-credentials@v4
+      with:
+        aws-access-key-id: ${{ inputs.aws-access-key-id }}
+        aws-secret-access-key: ${{ inputs.aws-secret-access-key }}
+        aws-region: ${{ inputs.aws-region }}
+
+    - name: Log in to Amazon ECR
+      id: login-ecr
+      uses: aws-actions/amazon-ecr-login@v2
+
+    - name: Deploy ECR
+      uses: ./.github/workflows/actions/terraform-deploy
+      with:
+        working_directory: ${{ inputs.ecr_tf_dir }}
+        aws-access-key-id: ${{ inputs.aws-access-key-id }}
+        aws-secret-access-key: ${{ inputs.aws-secret-access-key }}
+        aws-region: ${{ inputs.aws-region }}
+    - name: Set Docker image tag
+      id: set_tag
+      shell: bash
+      run: |
+        SHORT_SHA=$(echo "${{ inputs.git-sha }}" | cut -c1-7)
+        BRANCH=$(echo "${{ inputs.git-ref }}" | tr '/' '-')
+        TAG="${BRANCH}-${SHORT_SHA}"
+        echo "IMAGE_TAG=${TAG}" >> $GITHUB_ENV
+        echo "tag=$TAG" >> $GITHUB_OUTPUT
+
+    - name: Build and push Docker image
+      shell: bash
+      run: |
+        IMAGE_URI=${{ steps.login-ecr.outputs.registry }}/${{ inputs.ecr_name }}:${{ steps.set_tag.outputs.tag }}
+        echo "Building Docker image for ${{ inputs.ecr_name }}..."
+        docker build -t $IMAGE_URI -f ${{ inputs.dockerfile_path }} .
+        
+        echo "Pushing to ECR..."
+        docker push $IMAGE_URI
+
+    - name: Deploy Lambda
+      uses: ./.github/workflows/actions/terraform-deploy
+      with:
+        working_directory: ${{ inputs.lambda_tf_dir }}
+        aws-access-key-id: ${{ inputs.aws-access-key-id }}
+        aws-secret-access-key: ${{ inputs.aws-secret-access-key }}
+        aws-region: ${{ inputs.aws-region }}
+        lambda-image-tag: ${{ steps.set_tag.outputs.tag }}
+
+
+

.github/workflows/actions/terraform-deploy/action.yml

@@ -0,0 +1,55 @@
+name: "Terraform Plan Shared Config"
+description: "Plans shared Terraform config for Lambdas"
+
+inputs:
+  working_directory:
+    description: "Directory containing Terraform config"
+    required: true
+  aws-access-key-id:
+    description: "AWS access key"
+    required: true
+  aws-secret-access-key:
+    description: "AWS secret key"
+    required: true
+  aws-region:
+    description: "AWS region"
+    required: true
+  lambda-image-tag:
+    description: "Tag of the Lambda image (e.g., GitHub SHA)"
+    required: false
+
+runs:
+  using: "composite"
+  steps:
+    - uses: actions/checkout@v4
+
+    - name: Configure AWS credentials
+      uses: aws-actions/configure-aws-credentials@v4
+      with:
+        aws-access-key-id: ${{ inputs.aws-access-key-id }}
+        aws-secret-access-key: ${{ inputs.aws-secret-access-key }}
+        aws-region: ${{ inputs.aws-region }}
+
+    - name: Setup Terraform
+      uses: hashicorp/setup-terraform@v3
+
+    - name: Terraform Init
+      working-directory: ${{ inputs.working_directory }}
+      shell: bash
+      run: terraform init -reconfigure
+
+    - name: Terraform Plan
+      working-directory: ${{ inputs.working_directory }}
+      shell: bash
+      run: |
+        if [ -n "${{ inputs.lambda-image-tag }}" ]; then
+          terraform plan -out=tfplan -var="lambda_image_tag=${{ inputs.lambda-image-tag }}"
+        else
+          terraform plan -out=tfplan
+        fi
+
+    - name: Terraform Apply
+      working-directory: ${{ inputs.working_directory }}
+      shell: bash
+      run: terraform apply -auto-approve tfplan
+

.github/workflows/hubspot_deal_notes.yml

@@ -1,29 +0,0 @@
-name: Deal Notes From HubSpot Scraper
-on:
-  schedule:
-    - cron: '0 19 * * 0'
-  workflow_dispatch:
-
-jobs:
-  sharepoint-validator:
-    runs-on: ubuntu-22.04
-    steps:
-      - uses: actions/checkout@v3
-
-      - name: Set up Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: '3.12'
-
-      - name: Install dependencies
-        run: |
-          pip install poetry
-          poetry install --no-root
-
-      - name: run script
-        run: |
-          pwd
-          ls -la
-          poetry run python etl/dimitra_hubspot_notes_gather.py
-        env:
-          PYTHONPATH: ${{ github.workspace }}

.github/workflows/hubspot_sync.yml

@@ -0,0 +1,51 @@
+# name: Hubspot Sync
+
+# on:
+#   schedule:
+#     # Every 15 minutes, 07:00–18:59, Monday–Friday (UTC)
+#     - cron: '0 7-18/2 * * 1-5'
+
+#     # Once on Saturday at 09:00 UTC
+#     - cron: '0 9 * * 6'
+
+#     # Once on Sunday at 09:00 UTC
+#     - cron: '0 9 * * 0'
+#   workflow_dispatch:
+
+# jobs:
+#   hubspot-sync:
+#     runs-on: [self-hosted, mist]
+#     steps:
+#       - uses: actions/checkout@v4
+
+#       - name: Set up Python
+#         uses: actions/setup-python@v5
+#         with:
+#           python-version: '3.12'
+
+#       - name: Install dependencies
+#         run: |
+#           pip install poetry
+#           poetry install --no-root
+
+#       # - name: Run scripts
+#       #   env:
+#       #     PYTHONPATH: ${{ github.workspace }}
+#       #     DATABASE_URL: ${{ secrets.PROD_DATABASE_URL }}
+#       #   run: |
+#       #     pwd
+#       #     ls -la
+#       #     poetry run python etl/hubSpotClient/scripts/hubspot_gather_all_deals.py
+      
+
+#       - name: Run scripts
+#         env:
+#           PYTHONPATH: ${{ github.workspace }}
+#           DATABASE_URL: ${{ secrets.PROD_DATABASE_URL }}
+#         run: |
+#           pwd
+#           ls -la
+#           poetry run python etl/hubSpotClient/scripts/hubspot_update_script.py
+     
+      
+      

.github/workflows/hubspot_to_invoice.yml

@@ -1,37 +0,0 @@
-name: HubSpot Deals to DB loading and Invoice Calculator
-on:
-  # schedule:
-  #   - cron: '0 19 * * 0'
-  workflow_dispatch:
-
-jobs:
-  hubspot-deals-to-db:
-    runs-on: ubuntu-22.04
-    steps:
-      - uses: actions/checkout@v3
-
-      - name: Set up Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: '3.12'
-
-      - name: Install dependencies
-        run: |
-          pip install poetry
-          poetry install --no-root
-
-      - name: run script
-        run: |
-          pwd
-          ls -la
-          poetry run python etl/hubspot_to_invoice_rewrite.py
-        env:
-          PYTHONPATH: ${{ github.workspace }}
-          DATABASE_URL: postgresql://postgres:makingwarmhomes@terraform-20250331175522503500000002.cdgzupxvdyp0.eu-west-2.rds.amazonaws.com:5432/surveyDB
-          SOUTH_COAST_INSULATION_SERVICE_SHAREPOINT_ID: ${{ secrets.SOUTH_COAST_INSULATION_SERVICE_SHAREPOINT_ID }}
-          JJC_SERVICE_SHAREPOINT_ID: ${{ secrets.JJC_SERVICE_SHAREPOINT_ID }}
-          BAXTER_KELLY_SERVICE_SHAREPOINT_ID: ${{ secrets.BAXTER_KELLY_SERVICE_SHAREPOINT_ID }}
-          SGEC_SERVICE_SHAREPOINT_ID: ${{ secrets.SGEC_SERVICE_SHAREPOINT_ID }}
-          SHAREPOINT_CLIENT_ID: ${{ secrets.SHAREPOINT_CLIENT_ID }}
-          SHAREPOINT_CLIENT_SECRET: ${{ secrets.SHAREPOINT_CLIENT_SECRET }}
-          SHAREPOINT_TENANT_ID: ${{ secrets.SHAREPOINT_TENANT_ID }}

.github/workflows/lambda_main.yml

@@ -0,0 +1,92 @@
+name: Lambda Main Workflow
+
+on:
+  push:
+    branches: [main, feautre/walthamforest_etl]
+
+env:
+  AWS_REGION: eu-west-2
+
+jobs:
+  shared-lambda-terraform:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v4
+      - name: Deploy shared Lambda Config Terraform
+        uses: ./.github/workflows/actions/terraform-deploy
+        with:
+          working_directory: ./deployment/lambda/lambda_shared
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ secrets.AWS_REGION }}
+
+  lambda-ecr-example:
+    runs-on: ubuntu-latest
+    needs: shared-lambda-terraform 
+    permissions:
+      id-token: write
+      contents: read
+
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v4
+      - name: Build and deploy Lambda example
+        uses: ./.github/workflows/actions/lambda-deploy
+        with:
+          ecr_name: lambda_example
+          dockerfile_path: ./deployment/lambda/lambda_example/docker/Dockerfile
+          ecr_tf_dir: ./deployment/lambda/lambda_example/docker/
+          lambda_tf_dir: ./deployment/lambda/lambda_example/
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ secrets.AWS_REGION }}
+          git-sha: ${{ github.sha }}
+          git-ref: ${{ github.ref_name }}
+
+  extractor-and-loader:
+    runs-on: ubuntu-latest
+    needs: shared-lambda-terraform 
+    permissions:
+      id-token: write
+      contents: read
+
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v4
+      - name: Build and deploy Extractor & Loader Lambda
+        uses: ./.github/workflows/actions/lambda-deploy
+        with:
+          ecr_name: extractor_and_loader
+          dockerfile_path: ./deployment/lambda/extractor_and_loader/docker/Dockerfile
+          ecr_tf_dir: ./deployment/lambda/extractor_and_loader/docker/
+          lambda_tf_dir: ./deployment/lambda/extractor_and_loader/
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ secrets.AWS_REGION }}
+          git-sha: ${{ github.sha }}
+          git-ref: ${{ github.ref_name }}
+
+
+  walthamforest-etl:
+    runs-on: ubuntu-latest
+    needs: shared-lambda-terraform 
+    permissions:
+      id-token: write
+      contents: read
+
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v4
+      - name: Build and deploy WalthamForest ETL
+        uses: ./.github/workflows/actions/lambda-deploy
+        with:
+          ecr_name: walthamforest_etl_adhoc_ecr
+          dockerfile_path: ./deployment/lambda/walthamforest_etl/docker/Dockerfile
+          ecr_tf_dir: ./deployment/lambda/walthamforest_etl/docker/
+          lambda_tf_dir: ./deployment/lambda/walthamforest_etl/
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ secrets.AWS_REGION }}
+          git-sha: ${{ github.sha }}
+          git-ref: ${{ github.ref_name }}

.github/workflows/months_end.yml

@@ -0,0 +1,39 @@
+name: Months End
+
+on:
+  schedule:
+    - cron: '0 7 23-31 * *'  # Every day from the 23rd to end of month at 07:00 UTC
+  workflow_dispatch:
+
+jobs:
+  surveyed-needs-sign-off:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+
+      - name: Install dependencies
+        run: |
+          pip install poetry
+          poetry install --no-root
+
+      - name: Run scripts
+        env:
+          PYTHONPATH: ${{ github.workspace }}
+        run: |
+          pwd
+          ls -la
+          poetry run python etl/month_end_automation_wave_2_layout.py # Done
+          poetry run python etl/month_end_automation_wave_2_no_3.py # Done
+          poetry run python etl/month_end_automation_wave_2_no_4.py # Done
+          poetry run python etl/month_end_automation_wave_2_no_6.py # Check with Matt if this can be deleted
+          poetry run python etl/month_end_automation_wave_2_no_7.py # Done
+          poetry run python etl/month_end_automation_wave_2_no_8.py # Done
+          poetry run python etl/month_end_automation_wave_2_no_11.py # Done
+          poetry run python etl/month_end_automation_wave_2_no_15.py # Done
+          poetry run python etl/month_end_automation_wave_accent_housing.py # Done
+          poetry run python etl/month_end_automation_wave_3_layout.py # Done

.github/workflows/pytest.yml

@@ -1,42 +0,0 @@
-name: Run Pytest
-
-on:
-  push:
-    branches:
-      - '**'  # Run on all branches
-  pull_request:
-    branches:
-      - main
-
-jobs:
-  etl-unit-tests:
-    runs-on: ubuntu-22.04
-
-    steps:
-      - name: Checkout Repository
-        uses: actions/checkout@v4
-
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: '3.12'
-
-      - name: Set up Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: '3.12'
-
-      - name: Install dependencies
-        run: |
-          pip install poetry
-          poetry install --no-root
-
-      - name: Run Tests
-        run: |
-          poetry run pytest -W ignore::DeprecationWarning
-        env:
-          PYTHONPATH: ${{ github.workspace }}
-
-
-
-    continue-on-error: ${{ github.event_name == 'push' && github.ref != 'refs/heads/main' }}

.github/workflows/scis_invoice_calculator.yml

@@ -1,39 +0,0 @@
-name: SCIS Invoice Calculator
-on:
-  schedule:
-    - cron: '0 6 * * *'
-  workflow_dispatch:
-
-jobs:
-  scis_invoice_calculator:
-    runs-on: ubuntu-22.04
-    steps:
-      - uses: actions/checkout@v3
-
-      - name: Set up Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: '3.12'
-
-      - name: Install dependencies
-        run: |
-          pip install poetry
-          poetry install --no-root
-
-      - name: run script
-        run: |
-          bash scis_invoice.sh
-        env:
-          PYTHONPATH: ${{ github.workspace }}
-          SOUTH_COAST_INSULATION_SERVICE_SHAREPOINT_ID: ${{ secrets.SOUTH_COAST_INSULATION_SERVICE_SHAREPOINT_ID }}
-          JJC_SERVICE_SHAREPOINT_ID: ${{ secrets.JJC_SERVICE_SHAREPOINT_ID }}
-          BAXTER_KELLY_SERVICE_SHAREPOINT_ID: ${{ secrets.BAXTER_KELLY_SERVICE_SHAREPOINT_ID }}
-          SGEC_SERVICE_SHAREPOINT_ID: ${{ secrets.SGEC_SERVICE_SHAREPOINT_ID }}
-          SHAREPOINT_CLIENT_ID: ${{ secrets.SHAREPOINT_CLIENT_ID }}
-          SHAREPOINT_CLIENT_SECRET: ${{ secrets.SHAREPOINT_CLIENT_SECRET }}
-          SHAREPOINT_TENANT_ID: ${{ secrets.SHAREPOINT_TENANT_ID }}
-      - name: Upload Excel file
-        uses: actions/upload-artifact@v4
-        with:
-          name: my-excel-file
-          path: survey_data.xlsx

.github/workflows/sharepoint_validator.yml

@@ -1,36 +0,0 @@
-name: SharePoint Validator
-on:
-  schedule:
-    - cron: '0 6 * * *'
-  workflow_dispatch:
-
-jobs:
-  sharepoint-validator:
-    runs-on: ubuntu-22.04
-    steps:
-      - uses: actions/checkout@v3
-
-      - name: Set up Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: '3.12'
-
-      - name: Install dependencies
-        run: |
-          pip install poetry
-          poetry install --no-root
-
-      - name: run script
-        run: |
-          pwd
-          ls -la
-          bash run_daily_script.sh
-        env:
-          PYTHONPATH: ${{ github.workspace }}
-          SOUTH_COAST_INSULATION_SERVICE_SHAREPOINT_ID: ${{ secrets.SOUTH_COAST_INSULATION_SERVICE_SHAREPOINT_ID }}
-          JJC_SERVICE_SHAREPOINT_ID: ${{ secrets.JJC_SERVICE_SHAREPOINT_ID }}
-          BAXTER_KELLY_SERVICE_SHAREPOINT_ID: ${{ secrets.BAXTER_KELLY_SERVICE_SHAREPOINT_ID }}
-          SGEC_SERVICE_SHAREPOINT_ID: ${{ secrets.SGEC_SERVICE_SHAREPOINT_ID }}
-          SHAREPOINT_CLIENT_ID: ${{ secrets.SHAREPOINT_CLIENT_ID }}
-          SHAREPOINT_CLIENT_SECRET: ${{ secrets.SHAREPOINT_CLIENT_SECRET }}
-          SHAREPOINT_TENANT_ID: ${{ secrets.SHAREPOINT_TENANT_ID }}

.gitignore

@@ -4,4 +4,6 @@ __pycache__/
 data/
 *ipynb
 etl/survery_data.csv
-foo.env.py
+foo.env.py
+*.xlsx
+*.csv

.vscode/settings.json

@@ -3,6 +3,7 @@
     "python.REPL.sendToNativeREPL": true,
     "notebook.output.scrolling": true,
     "terminal.integrated.defaultProfile.linux": "bash",
+    "editor.rulers": [67],
     "terminal.integrated.profiles.linux": {
         "bash": {
             "path": "/bin/bash"
@@ -14,5 +15,27 @@
     //     "%load_ext autoreload", "%autoreload 2"
     // ]
 
+    "vim.enableNeovim": false,
+
+    // Allow VSCode native keybindings to override Vim when needed
+    "vim.handleKeys": {
+        "<C-p>": false,
+        "<C-P>": false,
+        "<C-S-p>": false,
+        "<C-c>": false,
+        "<C-v>": false,
+        "<C-S-v>": false,
+        "<C-S-e>": false,
+        "<C-b>": false,
+        "<C-j>": false,
+        "<C-S-c>": false
+    },
+
+    // Terminal copy/paste via Ctrl+Shift+C / Ctrl+Shift+V
+    "terminal.integrated.copyOnSelection": false,
+    "terminal.integrated.commandsToSkipShell": [
+        "workbench.action.terminal.copySelection",
+        "workbench.action.terminal.paste"
+    ],
 
 }

alembic/env.py

@@ -5,8 +5,10 @@ from sqlalchemy import pool
 from alembic import context
 from sqlmodel import SQLModel
 
-from etl.load.topLevel import *
-from etl.load.preSiteNoteTypes import *
+from etl.models.topLevel import *
+from etl.models.preSiteNoteTypes import *
+from etl.models.conditionReport import *
+from etl.fileReader.reportType import ReportType
 
 import os
 
@@ -32,6 +34,8 @@ def run_migrations_offline() -> None:
         target_metadata=target_metadata,
         literal_binds=True,
         dialect_opts={"paramstyle": "named"},
+        compare_type=True,
+        compare_server_default=True,
     )
     with context.begin_transaction():
         context.run_migrations()
@@ -44,6 +48,8 @@ def run_migrations_online() -> None:
         context.configure(
             connection=connection,
             target_metadata=target_metadata,
+            compare_type=True,
+            compare_server_default=True,
         )
         with context.begin_transaction():
             context.run_migrations()

alembic/things_alembic_doesn't_automate_for.txt

@@ -0,0 +1,19 @@
+Source: https://alembic.sqlalchemy.org/en/latest/autogenerate.html
+
+Autogenerate can not detect:
+
+    Changes of table name. These will come out as an add/drop of two different tables, and should be hand-edited into a name change instead.
+
+    Changes of column name. Like table name changes, these are detected as a column add/drop pair, which is not at all the same as a name change.
+
+    Anonymously named constraints. Give your constraints a name, e.g. UniqueConstraint('col1', 'col2', name="my_name"). See the section The Importance of Naming Constraints for background on how to configure automatic naming schemes for constraints.
+
+    Special SQLAlchemy types such as Enum when generated on a backend which doesn’t support ENUM directly - this because the representation of such a type in the non-supporting database, i.e. a CHAR+ CHECK constraint, could be any kind of CHAR+CHECK. For SQLAlchemy to determine that this is actually an ENUM would only be a guess, something that’s generally a bad idea. To implement your own “guessing” function here, use the sqlalchemy.events.DDLEvents.column_reflect() event to detect when a CHAR (or whatever the target type is) is reflected, and change it to an ENUM (or whatever type is desired) if it is known that that’s the intent of the type. The sqlalchemy.events.DDLEvents.after_parent_attach() can be used within the autogenerate process to intercept and un-attach unwanted CHECK constraints.
+
+
+
+Autogenerate can’t currently, but will eventually detect:
+
+    Some free-standing constraint additions and removals may not be supported, including PRIMARY KEY, EXCLUDE, CHECK; these are not necessarily implemented within the autogenerate detection system and also may not be supported by the supporting SQLAlchemy dialect.
+
+    Sequence additions, removals - not yet implemented.

alembic/versions/1f1d3d560ccb_db_defaults_enum_binding_for_uploaded_.py

@@ -0,0 +1,46 @@
+"""db defaults + enum binding for uploaded_files
+
+Revision ID: 1f1d3d560ccb
+Revises: 270ba252bc11
+Create Date: 2025-08-14 17:11:41.866908
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision: str = '1f1d3d560ccb'
+down_revision: Union[str, None] = '270ba252bc11'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    """Upgrade schema."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.alter_column('uploaded_files', 's3_file_upload_timestamp',
+               existing_type=postgresql.TIMESTAMP(),
+               type_=sa.DateTime(timezone=True),
+               existing_nullable=False)
+    op.alter_column('uploaded_files', 's3_json_upload_timestamp',
+               existing_type=postgresql.TIMESTAMP(),
+               type_=sa.DateTime(timezone=True),
+               existing_nullable=True)
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    """Downgrade schema."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.alter_column('uploaded_files', 's3_json_upload_timestamp',
+               existing_type=sa.DateTime(timezone=True),
+               type_=postgresql.TIMESTAMP(),
+               existing_nullable=True)
+    op.alter_column('uploaded_files', 's3_file_upload_timestamp',
+               existing_type=sa.DateTime(timezone=True),
+               type_=postgresql.TIMESTAMP(),
+               existing_nullable=False)
+    # ### end Alembic commands ###

alembic/versions/20c418a7d5ec_add_company_id_in_hubspot_data_and_.py

@@ -0,0 +1,68 @@
+"""add company id in hubspot data and rename table
+
+Revision ID: 20c418a7d5ec
+Revises: e72e15f7e0c3
+Create Date: 2025-10-27 16:20:11.362657
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision: str = '20c418a7d5ec'
+down_revision: Union[str, None] = 'e72e15f7e0c3'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+import sqlmodel
+
+def upgrade() -> None:
+    """Upgrade schema."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('hubspot_deal_data',
+    sa.Column('id', sa.Uuid(), nullable=False),
+    sa.Column('deal_id', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
+    sa.Column('dealname', sqlmodel.sql.sqltypes.AutoString(), nullable=True),
+    sa.Column('dealstage', sqlmodel.sql.sqltypes.AutoString(), nullable=True),
+    sa.Column('company_id', sqlmodel.sql.sqltypes.AutoString(), nullable=True),
+    sa.Column('landlord_property_id', sqlmodel.sql.sqltypes.AutoString(), nullable=True),
+    sa.Column('uprn', sqlmodel.sql.sqltypes.AutoString(), nullable=True),
+    sa.Column('outcome', sqlmodel.sql.sqltypes.AutoString(), nullable=True),
+    sa.Column('outcome_notes', sqlmodel.sql.sqltypes.AutoString(), nullable=True),
+    sa.Column(
+        'created_at',
+        sa.DateTime(timezone=True),
+        server_default=sa.text('(CURRENT_TIMESTAMP AT TIME ZONE \'UTC\')'),
+        nullable=False,
+    ),
+    sa.Column('updated_at', sa.DateTime(timezone=True), nullable=True),
+    sa.PrimaryKeyConstraint('id')
+    )
+    op.create_index(op.f('ix_hubspot_deal_data_deal_id'), 'hubspot_deal_data', ['deal_id'], unique=False)
+    op.drop_index('ix_hubspot_data_deal_id', table_name='hubspot_data')
+    op.drop_table('hubspot_data')
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    """Downgrade schema."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('hubspot_data',
+    sa.Column('id', sa.UUID(), autoincrement=False, nullable=False),
+    sa.Column('deal_id', sa.VARCHAR(), autoincrement=False, nullable=False),
+    sa.Column('dealname', sa.VARCHAR(), autoincrement=False, nullable=True),
+    sa.Column('dealstage', sa.VARCHAR(), autoincrement=False, nullable=True),
+    sa.Column('landlord_property_id', sa.VARCHAR(), autoincrement=False, nullable=True),
+    sa.Column('uprn', sa.VARCHAR(), autoincrement=False, nullable=True),
+    sa.Column('outcome', sa.VARCHAR(), autoincrement=False, nullable=True),
+    sa.Column('outcome_notes', sa.VARCHAR(), autoincrement=False, nullable=True),
+    sa.Column('created_at', postgresql.TIMESTAMP(timezone=True), server_default=sa.text("(CURRENT_TIMESTAMP AT TIME ZONE 'UTC'::text)"), autoincrement=False, nullable=False),
+    sa.Column('updated_at', postgresql.TIMESTAMP(timezone=True), autoincrement=False, nullable=True),
+    sa.PrimaryKeyConstraint('id', name='hubspot_data_pkey')
+    )
+    op.create_index('ix_hubspot_data_deal_id', 'hubspot_data', ['deal_id'], unique=False)
+    op.drop_index(op.f('ix_hubspot_deal_data_deal_id'), table_name='hubspot_deal_data')
+    op.drop_table('hubspot_deal_data')
+    # ### end Alembic commands ###

alembic/versions/23a4e2cc5467_added_project_code.py

@@ -0,0 +1,32 @@
+"""added project code
+
+Revision ID: 23a4e2cc5467
+Revises: 20c418a7d5ec
+Create Date: 2025-10-27 16:23:16.984274
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+import sqlmodel
+
+# revision identifiers, used by Alembic.
+revision: str = '23a4e2cc5467'
+down_revision: Union[str, None] = '20c418a7d5ec'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    """Upgrade schema."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('hubspot_deal_data', sa.Column('project_code', sqlmodel.sql.sqltypes.AutoString(), nullable=True))
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    """Downgrade schema."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column('hubspot_deal_data', 'project_code')
+    # ### end Alembic commands ###

alembic/versions/2409147995c5_added_hubspot_table.py

@@ -0,0 +1,48 @@
+"""added hubspot table
+
+Revision ID: 2409147995c5
+Revises: 4c67501b7451
+Create Date: 2025-10-27 15:05:01.552689
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+import sqlmodel
+
+# revision identifiers, used by Alembic.
+revision: str = '2409147995c5'
+down_revision: Union[str, None] = '4c67501b7451'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+def upgrade() -> None:
+    op.create_table(
+        'hubspot_data',
+        sa.Column('id', sa.Uuid(), nullable=False),
+        sa.Column('deal_id', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
+        sa.Column('dealname', sqlmodel.sql.sqltypes.AutoString(), nullable=True),
+        sa.Column('dealstage', sqlmodel.sql.sqltypes.AutoString(), nullable=True),
+        sa.Column('landlord_property_id', sqlmodel.sql.sqltypes.AutoString(), nullable=True),
+        sa.Column('uprn', sqlmodel.sql.sqltypes.AutoString(), nullable=True),
+        sa.Column('outcome', sqlmodel.sql.sqltypes.AutoString(), nullable=True),
+        sa.Column('outcome_notes', sqlmodel.sql.sqltypes.AutoString(), nullable=True),
+        sa.Column('raw_data', postgresql.JSON(astext_type=sa.Text()), nullable=True),
+        sa.Column(
+            'created_at',
+            sa.DateTime(timezone=True),
+            server_default=sa.text('(CURRENT_TIMESTAMP AT TIME ZONE \'UTC\')'),
+            nullable=False,
+        ),
+        sa.Column('updated_at', sa.DateTime(timezone=True), nullable=True),
+        sa.PrimaryKeyConstraint('id')
+    )
+
+def downgrade() -> None:
+    """Downgrade schema."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_index(op.f('ix_hubspot_data_deal_id'), table_name='hubspot_data')
+    op.drop_table('hubspot_data')
+    # ### end Alembic commands ###

alembic/versions/253a1047c623_auto_generate_id.py

@@ -0,0 +1,46 @@
+"""auto generate id
+
+Revision ID: 253a1047c623
+Revises: e8507a27795a
+Create Date: 2025-08-14 17:25:54.010315
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision: str = '253a1047c623'
+down_revision: Union[str, None] = 'e8507a27795a'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    """Upgrade schema."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.alter_column('uploaded_files', 'id',
+               existing_type=sa.UUID(),
+               server_default=sa.text('gen_random_uuid()'),
+               existing_nullable=False)
+    op.alter_column('uploaded_files', 's3_file_upload_timestamp',
+               existing_type=postgresql.TIMESTAMP(timezone=True),
+               server_default=sa.text("NOW() AT TIME ZONE 'utc'"),
+               existing_nullable=False)
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    """Downgrade schema."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.alter_column('uploaded_files', 's3_file_upload_timestamp',
+               existing_type=postgresql.TIMESTAMP(timezone=True),
+               server_default=None,
+               existing_nullable=False)
+    op.alter_column('uploaded_files', 'id',
+               existing_type=sa.UUID(),
+               server_default=None,
+               existing_nullable=False)
+    # ### end Alembic commands ###

alembic/versions/270ba252bc11_update_enum.py

@@ -0,0 +1,30 @@
+"""update enum
+
+Revision ID: 270ba252bc11
+Revises: a6e4562797e4
+Create Date: 2025-08-14 16:52:17.473370
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision: str = '270ba252bc11'
+down_revision: Union[str, None] = 'a6e4562797e4'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.execute(
+        "ALTER TYPE reporttype ADD VALUE IF NOT EXISTS 'osmosis_condition_pas_2035_report'"
+    )
+    
+def downgrade() -> None:
+    """Downgrade schema."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    pass
+    # ### end Alembic commands ###

alembic/versions/29113d69989e_enum_things.py

@@ -0,0 +1,34 @@
+"""enum things
+
+Revision ID: 29113d69989e
+Revises: 2cf02c9f71f8
+Create Date: 2025-08-19 11:40:52.712131
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision: str = '29113d69989e'
+down_revision: Union[str, None] = '2cf02c9f71f8'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade():
+    op.execute("ALTER TYPE reporttype ADD VALUE IF NOT EXISTS 'osmosis_condition_pas_2035_report'")
+    op.execute("ALTER TYPE reporttype ADD VALUE IF NOT EXISTS 'warm_homes_condition_pas_2035_report'")
+    op.execute("ALTER TYPE reporttype ADD VALUE IF NOT EXISTS 'energy_performance_report_with_data'")
+    op.execute("ALTER TYPE reporttype ADD VALUE IF NOT EXISTS 'energy_performance_report_summary_information'")
+    op.execute("ALTER TYPE reporttype ADD VALUE IF NOT EXISTS 'lodgement_xml_needed_for_lodgement_to_like_trademark'")
+    op.execute("ALTER TYPE reporttype ADD VALUE IF NOT EXISTS 'reduce_xml_needed_to_generate_full_sap_xml'")
+    op.execute("ALTER TYPE reporttype ADD VALUE IF NOT EXISTS 'full_xml_needed_for_co_ordination'")
+
+def downgrade() -> None:
+    """Downgrade schema."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    pass
+    # ### end Alembic commands ###

alembic/versions/2cf02c9f71f8_add_missing_report_type.py

@@ -0,0 +1,41 @@
+"""add missing report type
+
+Revision ID: 2cf02c9f71f8
+Revises: 253a1047c623
+Create Date: 2025-08-19 11:36:16.006276
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+from sqlalchemy.dialects import postgresql as psql
+
+# revision identifiers, used by Alembic.
+revision: str = '2cf02c9f71f8'
+down_revision: Union[str, None] = '253a1047c623'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+
+def upgrade() -> None:
+    op.alter_column(
+        "uploaded_files",
+        "s3_json_uri",
+        type_=psql.JSON(),           # or psql.JSONB()
+        postgresql_using="s3_json_uri::json",  # or ::jsonb
+        existing_type=sa.VARCHAR(),
+        existing_nullable=True,
+    )
+
+
+def downgrade() -> None:
+    """Downgrade schema."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.alter_column('uploaded_files', 's3_json_uri',
+               existing_type=postgresql.JSON(astext_type=sa.Text()),
+               type_=sa.VARCHAR(),
+               existing_nullable=True)
+    # ### end Alembic commands ###

alembic/versions/427e65da69c1_initial_table.py

@@ -1,354 +0,0 @@
-"""Initial table
-
-Revision ID: 427e65da69c1
-Revises: 
-Create Date: 2025-05-14 15:36:08.611971
-
-"""
-from typing import Sequence, Union
-
-from alembic import op
-import sqlalchemy as sa
-import sqlmodel
-
-
-# revision identifiers, used by Alembic.
-revision: str = '427e65da69c1'
-down_revision: Union[str, None] = None
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
-
-
-def upgrade() -> None:
-    """Upgrade schema."""
-    # ### commands auto generated by Alembic - please adjust! ###
-    op.create_table('buildings',
-    sa.Column('id', sa.Uuid(), nullable=False),
-    sa.Column('address', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
-    sa.Column('postcode', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
-    sa.Column('UPRN', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
-    sa.Column('landlord_id', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
-    sa.Column('domna_id', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
-    sa.PrimaryKeyConstraint('id')
-    )
-    op.create_table('companyinfo',
-    sa.Column('id', sa.Uuid(), nullable=False),
-    sa.Column('address', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
-    sa.Column('trading_name', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
-    sa.Column('post_code', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
-    sa.Column('fax_number', sqlmodel.sql.sqltypes.AutoString(), nullable=True),
-    sa.Column('related_party_disclosure', sqlmodel.sql.sqltypes.AutoString(), nullable=True),
-    sa.PrimaryKeyConstraint('id')
-    )
-    op.create_table('door',
-    sa.Column('id', sa.Uuid(), nullable=False),
-    sa.Column('no_of_doors', sa.Integer(), nullable=False),
-    sa.Column('no_of_insulated_doors', sa.Integer(), nullable=False),
-    sa.Column('u_value_w_m2_k', sqlmodel.sql.sqltypes.AutoString(), nullable=True),
-    sa.PrimaryKeyConstraint('id')
-    )
-    op.create_table('floors',
-    sa.Column('id', sa.Uuid(), nullable=False),
-    sa.Column('floor_type', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
-    sa.Column('ground_floor_construction', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
-    sa.Column('ground_floor_insulation_type', sqlmodel.sql.sqltypes.AutoString(), nullable=True),
-    sa.Column('floor_insulation_thickness_mm', sa.Float(), nullable=True),
-    sa.Column('u_value_known', sa.Boolean(), nullable=False),
-    sa.PrimaryKeyConstraint('id')
-    )
-    op.create_table('fluegasheatrecoverysystem',
-    sa.Column('id', sa.Uuid(), nullable=False),
-    sa.Column('fghrs_present', sa.Boolean(), nullable=False),
-    sa.PrimaryKeyConstraint('id')
-    )
-    op.create_table('heatingsystemcontrols',
-    sa.Column('id', sa.Uuid(), nullable=False),
-    sa.Column('control_type', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
-    sa.Column('flue_type', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
-    sa.Column('fan_assisted_flue', sa.Boolean(), nullable=False),
-    sa.Column('heat_emitter_type', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
-    sa.Column('electricity_meter_type', sqlmodel.sql.sqltypes.AutoString(), nullable=True),
-    sa.Column('mains_gas_available', sa.Boolean(), nullable=True),
-    sa.PrimaryKeyConstraint('id')
-    )
-    op.create_table('heatingtype',
-    sa.Column('id', sa.Uuid(), nullable=False),
-    sa.Column('heating_type', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
-    sa.Column('fuel_type', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
-    sa.PrimaryKeyConstraint('id')
-    )
-    op.create_table('hotwatercylinder',
-    sa.Column('id', sa.Uuid(), nullable=False),
-    sa.Column('volume', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
-    sa.Column('insulation_type', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
-    sa.Column('insulation_thickness', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
-    sa.Column('thermostat', sa.Boolean(), nullable=False),
-    sa.PrimaryKeyConstraint('id')
-    )
-    op.create_table('insulation',
-    sa.Column('id', sa.Uuid(), nullable=False),
-    sa.Column('type', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
-    sa.PrimaryKeyConstraint('id')
-    )
-    op.create_table('lighting',
-    sa.Column('id', sa.Uuid(), nullable=False),
-    sa.Column('total_no_of_light_fittings', sa.Integer(), nullable=False),
-    sa.Column('total_no_of_lel_fittings', sa.Integer(), nullable=False),
-    sa.PrimaryKeyConstraint('id')
-    )
-    op.create_table('otherdetails',
-    sa.Column('id', sa.Uuid(), nullable=False),
-    sa.Column('electricity_meter_type', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
-    sa.Column('main_gas_avalible', sa.Boolean(), nullable=False),
-    sa.PrimaryKeyConstraint('id')
-    )
-    op.create_table('photovoltaicpanel',
-    sa.Column('id', sa.Uuid(), nullable=False),
-    sa.Column('pvs_are_connected_to_dwelling_electricity_meter', sa.Boolean(), nullable=False),
-    sa.Column('percentage_of_external_roof_area_with_pvs', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
-    sa.PrimaryKeyConstraint('id')
-    )
-    op.create_table('presitenotessummaryinfo',
-    sa.Column('id', sa.Uuid(), nullable=False),
-    sa.Column('reference_number', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
-    sa.Column('epc_language', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
-    sa.Column('uprn', sqlmodel.sql.sqltypes.AutoString(), nullable=True),
-    sa.Column('postcode', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
-    sa.Column('region', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
-    sa.Column('address', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
-    sa.Column('town', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
-    sa.Column('county', sqlmodel.sql.sqltypes.AutoString(), nullable=True),
-    sa.Column('property_tenure', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
-    sa.Column('transaction_type', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
-    sa.Column('inspection_date', sa.DateTime(), nullable=False),
-    sa.Column('current_sap', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
-    sa.Column('potential_sap', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
-    sa.Column('current_ei', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
-    sa.Column('potential_ei', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
-    sa.Column('current_annual_emissions', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
-    sa.Column('current_annual_emission_including_0925_multiplayer', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
-    sa.Column('current_annual_energy_costs', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
-    sa.PrimaryKeyConstraint('id')
-    )
-    op.create_table('roofs',
-    sa.Column('id', sa.Uuid(), nullable=False),
-    sa.Column('construction', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
-    sa.Column('insulation_type', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
-    sa.Column('insulation_thickness', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
-    sa.Column('u_value_known', sa.Boolean(), nullable=False),
-    sa.PrimaryKeyConstraint('id')
-    )
-    op.create_table('showerandbaths',
-    sa.Column('id', sa.Uuid(), nullable=False),
-    sa.Column('no_of_rooms_with_baths_and_or_shower', sa.Integer(), nullable=False),
-    sa.Column('no_of_rooms_with_mixer_shower_and_no_baths', sa.Integer(), nullable=False),
-    sa.Column('no_of_rooms_with_mixer_shower_and_baths', sa.Integer(), nullable=False),
-    sa.PrimaryKeyConstraint('id')
-    )
-    op.create_table('solarwaterheating',
-    sa.Column('id', sa.Uuid(), nullable=False),
-    sa.Column('solar_water_heating_details_known', sa.Boolean(), nullable=False),
-    sa.PrimaryKeyConstraint('id')
-    )
-    op.create_table('ventilationandcooling',
-    sa.Column('id', sa.Uuid(), nullable=False),
-    sa.Column('no_of_open_fireplaces', sa.Integer(), nullable=False),
-    sa.Column('ventilation_type', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
-    sa.Column('space_cooling_system_present', sa.Boolean(), nullable=False),
-    sa.PrimaryKeyConstraint('id')
-    )
-    op.create_table('walls',
-    sa.Column('id', sa.Uuid(), nullable=False),
-    sa.Column('construction', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
-    sa.Column('insulation', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
-    sa.Column('insulation_thickness_mm', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
-    sa.Column('wall_thickness_measured', sa.Boolean(), nullable=False),
-    sa.Column('wall_thickness_mm', sa.Integer(), nullable=True),
-    sa.Column('u_value_known', sa.Boolean(), nullable=False),
-    sa.Column('u_value_w_m2_k', sa.Float(), nullable=True),
-    sa.Column('dry_lining', sa.Boolean(), nullable=False),
-    sa.Column('alternative_wall_present', sa.Boolean(), nullable=False),
-    sa.PrimaryKeyConstraint('id')
-    )
-    op.create_table('waterheating',
-    sa.Column('id', sa.Uuid(), nullable=False),
-    sa.Column('heating_type', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
-    sa.Column('fuel_type', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
-    sa.PrimaryKeyConstraint('id')
-    )
-    op.create_table('windturbine',
-    sa.Column('id', sa.Uuid(), nullable=False),
-    sa.Column('wind_turbine', sa.Boolean(), nullable=False),
-    sa.PrimaryKeyConstraint('id')
-    )
-    op.create_table('assessorinfo',
-    sa.Column('id', sa.Uuid(), nullable=False),
-    sa.Column('accreditation_number', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
-    sa.Column('name', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
-    sa.Column('phone_number', sqlmodel.sql.sqltypes.AutoString(), nullable=True),
-    sa.Column('email_address', sqlmodel.sql.sqltypes.AutoString(), nullable=True),
-    sa.Column('company_id', sa.Uuid(), nullable=True),
-    sa.ForeignKeyConstraint(['company_id'], ['companyinfo.id'], ),
-    sa.PrimaryKeyConstraint('id')
-    )
-    op.create_table('heating',
-    sa.Column('id', sa.Uuid(), nullable=False),
-    sa.Column('type', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
-    sa.Column('heating_source', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
-    sa.Column('efficiency_source', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
-    sa.Column('heating_fuel', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
-    sa.Column('brand_name', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
-    sa.Column('model_name', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
-    sa.Column('model_qualifer', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
-    sa.Column('sap_2009_table', sqlmodel.sql.sqltypes.AutoString(), nullable=True),
-    sa.Column('percentage_of_heated_floor_area_served', sqlmodel.sql.sqltypes.AutoString(), nullable=True),
-    sa.Column('controls_id', sa.Uuid(), nullable=True),
-    sa.ForeignKeyConstraint(['controls_id'], ['heatingsystemcontrols.id'], ),
-    sa.PrimaryKeyConstraint('id')
-    )
-    op.create_table('propertydetail',
-    sa.Column('id', sa.Uuid(), nullable=False),
-    sa.Column('age_band', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
-    sa.Column('wall_id', sa.Uuid(), nullable=True),
-    sa.Column('roof_id', sa.Uuid(), nullable=True),
-    sa.Column('floor_id', sa.Uuid(), nullable=True),
-    sa.ForeignKeyConstraint(['floor_id'], ['floors.id'], ),
-    sa.ForeignKeyConstraint(['roof_id'], ['roofs.id'], ),
-    sa.ForeignKeyConstraint(['wall_id'], ['walls.id'], ),
-    sa.PrimaryKeyConstraint('id')
-    )
-    op.create_table('dimension',
-    sa.Column('id', sa.Uuid(), nullable=False),
-    sa.Column('floor_area_m2', sa.Float(), nullable=False),
-    sa.Column('room_height_m', sa.Float(), nullable=False),
-    sa.Column('loss_perimeter_m', sa.Float(), nullable=False),
-    sa.Column('party_wall_length_m', sa.Float(), nullable=False),
-    sa.Column('property_detail_id', sa.Uuid(), nullable=True),
-    sa.ForeignKeyConstraint(['property_detail_id'], ['propertydetail.id'], ),
-    sa.PrimaryKeyConstraint('id')
-    )
-    op.create_table('documents',
-    sa.Column('id', sa.Uuid(), nullable=False),
-    sa.Column('assessor_id', sa.Uuid(), nullable=False),
-    sa.Column('created_at', sa.DateTime(), nullable=False),
-    sa.Column('document_type', sa.Enum('QUIDOS_PRESITE_NOTE', 'CHARTED_SURVEYOR_REPORT', 'ENERGY_PERFORMANCE_REPORT', 'U_VALUE_CALCULATOR_REPORT', 'OVERWRITING_U_VALUE_DECLARATION_FORM', name='reporttype'), nullable=False),
-    sa.Column('building_id', sa.Uuid(), nullable=False),
-    sa.Column('target_table', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
-    sa.Column('target_id', sa.Uuid(), nullable=False),
-    sa.ForeignKeyConstraint(['assessor_id'], ['assessorinfo.id'], ),
-    sa.ForeignKeyConstraint(['building_id'], ['buildings.id'], ),
-    sa.PrimaryKeyConstraint('id')
-    )
-    op.create_table('propertydescription',
-    sa.Column('id', sa.Uuid(), nullable=False),
-    sa.Column('built_form', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
-    sa.Column('detachment_or_position', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
-    sa.Column('no_of_main_property', sa.Integer(), nullable=False),
-    sa.Column('no_of_extension_1', sa.Integer(), nullable=True),
-    sa.Column('no_of_extension_2', sa.Integer(), nullable=True),
-    sa.Column('no_of_extension_3', sa.Integer(), nullable=True),
-    sa.Column('no_of_extension_4', sa.Integer(), nullable=True),
-    sa.Column('no_of_habitable_rooms', sa.Integer(), nullable=False),
-    sa.Column('no_of_heated_rooms', sa.Integer(), nullable=False),
-    sa.Column('heated_basement', sa.Boolean(), nullable=False),
-    sa.Column('conservatory_type', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
-    sa.Column('percentage_of_draught_proofed', sa.Integer(), nullable=False),
-    sa.Column('terrain_type', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
-    sa.Column('conservatory', sa.Boolean(), nullable=False),
-    sa.Column('main_property_id', sa.Uuid(), nullable=False),
-    sa.Column('ex1_property_id', sa.Uuid(), nullable=True),
-    sa.Column('ex2_property_id', sa.Uuid(), nullable=True),
-    sa.Column('ex3_property_id', sa.Uuid(), nullable=True),
-    sa.Column('ex4_property_id', sa.Uuid(), nullable=True),
-    sa.Column('door_id', sa.Uuid(), nullable=True),
-    sa.Column('ventilation_and_cooling_id', sa.Uuid(), nullable=True),
-    sa.Column('lighting_id', sa.Uuid(), nullable=True),
-    sa.Column('water_heating_id', sa.Uuid(), nullable=True),
-    sa.Column('hot_water_cylinder_id', sa.Uuid(), nullable=True),
-    sa.Column('solar_water_heating_id', sa.Uuid(), nullable=True),
-    sa.Column('shower_and_baths_id', sa.Uuid(), nullable=True),
-    sa.Column('flue_gas_heat_recovery_system_id', sa.Uuid(), nullable=True),
-    sa.Column('photovoltaic_panel_id', sa.Uuid(), nullable=True),
-    sa.Column('wind_turbine_id', sa.Uuid(), nullable=True),
-    sa.Column('other_details_id', sa.Uuid(), nullable=True),
-    sa.Column('main_heating_id', sa.Uuid(), nullable=True),
-    sa.Column('main_heating2_id', sa.Uuid(), nullable=True),
-    sa.Column('secondary_heating_type_id', sa.Uuid(), nullable=True),
-    sa.ForeignKeyConstraint(['door_id'], ['door.id'], ),
-    sa.ForeignKeyConstraint(['ex1_property_id'], ['propertydetail.id'], ),
-    sa.ForeignKeyConstraint(['ex2_property_id'], ['propertydetail.id'], ),
-    sa.ForeignKeyConstraint(['ex3_property_id'], ['propertydetail.id'], ),
-    sa.ForeignKeyConstraint(['ex4_property_id'], ['propertydetail.id'], ),
-    sa.ForeignKeyConstraint(['flue_gas_heat_recovery_system_id'], ['fluegasheatrecoverysystem.id'], ),
-    sa.ForeignKeyConstraint(['hot_water_cylinder_id'], ['hotwatercylinder.id'], ),
-    sa.ForeignKeyConstraint(['lighting_id'], ['lighting.id'], ),
-    sa.ForeignKeyConstraint(['main_heating2_id'], ['heating.id'], ),
-    sa.ForeignKeyConstraint(['main_heating_id'], ['heating.id'], ),
-    sa.ForeignKeyConstraint(['main_property_id'], ['propertydetail.id'], ),
-    sa.ForeignKeyConstraint(['other_details_id'], ['otherdetails.id'], ),
-    sa.ForeignKeyConstraint(['photovoltaic_panel_id'], ['photovoltaicpanel.id'], ),
-    sa.ForeignKeyConstraint(['secondary_heating_type_id'], ['heatingtype.id'], ),
-    sa.ForeignKeyConstraint(['shower_and_baths_id'], ['showerandbaths.id'], ),
-    sa.ForeignKeyConstraint(['solar_water_heating_id'], ['solarwaterheating.id'], ),
-    sa.ForeignKeyConstraint(['ventilation_and_cooling_id'], ['ventilationandcooling.id'], ),
-    sa.ForeignKeyConstraint(['water_heating_id'], ['waterheating.id'], ),
-    sa.ForeignKeyConstraint(['wind_turbine_id'], ['windturbine.id'], ),
-    sa.PrimaryKeyConstraint('id')
-    )
-    op.create_table('windows',
-    sa.Column('id', sa.Uuid(), nullable=False),
-    sa.Column('glazing_type', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
-    sa.Column('area_m2', sa.Float(), nullable=False),
-    sa.Column('roof_window', sa.Boolean(), nullable=False),
-    sa.Column('orientation', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
-    sa.Column('u_value_w_m2_k', sa.Integer(), nullable=False),
-    sa.Column('g_value', sa.Integer(), nullable=False),
-    sa.Column('property_detail_id', sa.Uuid(), nullable=True),
-    sa.ForeignKeyConstraint(['property_detail_id'], ['propertydetail.id'], ),
-    sa.PrimaryKeyConstraint('id')
-    )
-    op.create_table('presitenote',
-    sa.Column('id', sa.Uuid(), nullable=False),
-    sa.Column('summary_info_id', sa.Uuid(), nullable=False),
-    sa.Column('assessor_id', sa.Uuid(), nullable=False),
-    sa.Column('pre_site_note_description_id', sa.Uuid(), nullable=True),
-    sa.ForeignKeyConstraint(['assessor_id'], ['assessorinfo.id'], ),
-    sa.ForeignKeyConstraint(['pre_site_note_description_id'], ['propertydescription.id'], ),
-    sa.ForeignKeyConstraint(['summary_info_id'], ['presitenotessummaryinfo.id'], ),
-    sa.PrimaryKeyConstraint('id')
-    )
-    # ### end Alembic commands ###
-
-
-def downgrade() -> None:
-    """Downgrade schema."""
-    # ### commands auto generated by Alembic - please adjust! ###
-    op.drop_table('presitenote')
-    op.drop_table('windows')
-    op.drop_table('propertydescription')
-    op.drop_table('documents')
-    op.drop_table('dimension')
-    op.drop_table('propertydetail')
-    op.drop_table('heating')
-    op.drop_table('assessorinfo')
-    op.drop_table('windturbine')
-    op.drop_table('waterheating')
-    op.drop_table('walls')
-    op.drop_table('ventilationandcooling')
-    op.drop_table('solarwaterheating')
-    op.drop_table('showerandbaths')
-    op.drop_table('roofs')
-    op.drop_table('presitenotessummaryinfo')
-    op.drop_table('photovoltaicpanel')
-    op.drop_table('otherdetails')
-    op.drop_table('lighting')
-    op.drop_table('insulation')
-    op.drop_table('hotwatercylinder')
-    op.drop_table('heatingtype')
-    op.drop_table('heatingsystemcontrols')
-    op.drop_table('fluegasheatrecoverysystem')
-    op.drop_table('floors')
-    op.drop_table('door')
-    op.drop_table('companyinfo')
-    op.drop_table('buildings')
-    # ### end Alembic commands ###

alembic/versions/4c67501b7451_added_more_enums.py

@@ -0,0 +1,60 @@
+"""added more enums
+
+Revision ID: 4c67501b7451
+Revises: ac8dba8cef50
+Create Date: 2025-09-23 10:22:20.648664
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision: str = "4c67501b7451"
+down_revision: Union[str, None] = "ac8dba8cef50"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+ENUM_NAME = "reporttype"
+
+# Values that were already present BEFORE this migration
+OLD_VALUES = (
+    "QUIDOS_PRESITE_NOTE",
+    "CHARTED_SURVEYOR_REPORT",
+    "ENERGY_PERFORMANCE_REPORT",
+    "U_VALUE_CALCULATOR_REPORT",
+    "OVERWRITING_U_VALUE_DECLARATION_FORM",
+    "OSMOSIS_CONDITION_PAS_2035_REPORT",
+    "DOMNA_CONDITION_PAS_2035_REPORT",
+)
+
+# Values we are ADDING in this migration
+NEW_VALUES = (
+    "DECENT_HOMES_RAW_DATA",
+    "DECENT_HOMES_SUMMARY",
+    "DECENT_HOMES_PROPERTY_META",
+)
+
+def upgrade() -> None:
+    for v in NEW_VALUES:
+        op.execute(f"ALTER TYPE {ENUM_NAME} ADD VALUE IF NOT EXISTS '{v}'")
+
+
+def downgrade() -> None:
+    # 1) Create a replacement type with ONLY the old values
+    old_vals = ", ".join(f"'{v}'" for v in OLD_VALUES)
+    op.execute(f"CREATE TYPE {ENUM_NAME}_old AS ENUM ({old_vals})")
+
+    # 2) Move columns to the temporary type
+    op.execute(
+        f"ALTER TABLE documents ALTER COLUMN document_type TYPE {ENUM_NAME}_old "
+        f"USING document_type::text::{ENUM_NAME}_old"
+    )
+    op.execute(
+        f"ALTER TABLE uploaded_files ALTER COLUMN doc_type TYPE {ENUM_NAME}_old "
+        f"USING doc_type::text::{ENUM_NAME}_old"
+    )
+
+    # 3) Drop original type and rename the temp back
+    op.execute(f"DROP TYPE {ENUM_NAME}")
+    op.execute(f"ALTER TYPE {ENUM_NAME}_old RENAME TO {ENUM_NAME}")

alembic/versions/57c0dc06cd25_add_company_info.py

@@ -0,0 +1,46 @@
+"""add company info
+
+Revision ID: 57c0dc06cd25
+Revises: 23a4e2cc5467
+Create Date: 2025-10-27 20:25:27.686455
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+import sqlmodel
+
+# revision identifiers, used by Alembic.
+revision: str = '57c0dc06cd25'
+down_revision: Union[str, None] = '23a4e2cc5467'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+def upgrade() -> None:
+    """Upgrade schema."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('hubspot_company_data',
+    sa.Column('id', sa.Uuid(), nullable=False),
+    sa.Column('company_id', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
+    sa.Column('company_name', sqlmodel.sql.sqltypes.AutoString(), nullable=True),
+    sa.Column('group_id', sqlmodel.sql.sqltypes.AutoString(), nullable=True),
+    sa.Column(
+        'created_at',
+        sa.DateTime(timezone=True),
+        server_default=sa.text('(CURRENT_TIMESTAMP AT TIME ZONE \'UTC\')'),
+        nullable=False,
+    ),
+    sa.Column('updated_at', sa.DateTime(timezone=True), nullable=True),
+    sa.PrimaryKeyConstraint('id')
+    )
+    op.create_index(op.f('ix_hubspot_company_data_company_id'), 'hubspot_company_data', ['company_id'], unique=False)
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    """Downgrade schema."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_index(op.f('ix_hubspot_company_data_company_id'), table_name='hubspot_company_data')
+    op.drop_table('hubspot_company_data')
+    # ### end Alembic commands ###

alembic/versions/66d2c9c325d6_auto_update.py

@@ -0,0 +1,57 @@
+"""auto update
+
+Revision ID: 66d2c9c325d6
+Revises: 57c0dc06cd25
+Create Date: 2025-10-28 11:58:28.356864
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision: str = '66d2c9c325d6'
+down_revision: Union[str, None] = '57c0dc06cd25'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    """Upgrade schema safely."""
+    # 1️⃣ Fill existing NULLs with current UTC time
+    op.execute("UPDATE hubspot_company_data SET updated_at = NOW() AT TIME ZONE 'utc' WHERE updated_at IS NULL;")
+    op.execute("UPDATE hubspot_deal_data SET updated_at = NOW() AT TIME ZONE 'utc' WHERE updated_at IS NULL;")
+
+    # 2️⃣ Now alter the column defaults and nullability
+    op.alter_column(
+        'hubspot_company_data',
+        'updated_at',
+        existing_type=sa.TIMESTAMP(timezone=True),
+        server_default=sa.text("NOW() AT TIME ZONE 'utc'"),
+        nullable=False,
+    )
+
+    op.alter_column(
+        'hubspot_deal_data',
+        'updated_at',
+        existing_type=sa.TIMESTAMP(timezone=True),
+        server_default=sa.text("NOW() AT TIME ZONE 'utc'"),
+        nullable=False,
+    )
+
+
+
+def downgrade() -> None:
+    """Downgrade schema."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.alter_column('hubspot_deal_data', 'updated_at',
+               existing_type=postgresql.TIMESTAMP(timezone=True),
+               server_default=None,
+               nullable=True)
+    op.alter_column('hubspot_company_data', 'updated_at',
+               existing_type=postgresql.TIMESTAMP(timezone=True),
+               server_default=None,
+               nullable=True)
+    # ### end Alembic commands ###

alembic/versions/a6e4562797e4_add_new_uploaded_file_table.py

@@ -0,0 +1,46 @@
+"""add new uploaded file table 
+
+Revision ID: a6e4562797e4
+Revises: 
+Create Date: 2025-08-14 14:44:40.992608
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+import sqlmodel
+
+# revision identifiers, used by Alembic.
+revision: str = 'a6e4562797e4'
+down_revision: Union[str, None] = None
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    """Upgrade schema."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('uploaded_files',
+    sa.Column('id', sa.Uuid(), nullable=False),
+    sa.Column('s3_json_uri', sqlmodel.sql.sqltypes.AutoString(), nullable=True),
+    sa.Column('s3_file_uri', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
+    sa.Column('doc_type', sa.Enum('QUIDOS_PRESITE_NOTE', 'CHARTED_SURVEYOR_REPORT', 'U_VALUE_CALCULATOR_REPORT', 'OVERWRITING_U_VALUE_DECLARATION_FORM', 'ECO_CONDITION_REPORT', 'WARM_HOMES_CONDITION_REPORT', 'ENERGY_PERFORMANCE_REPORT_WITH_DATA', 'ENERGY_PERFORMANCE_REPORT_SUMMARY_INFORMATION', 'LIG_XML', 'RDSAP_XML', 'FULLSAP_XML', name='reporttype'), nullable=False),
+    sa.Column('s3_file_upload_timestamp', sa.DateTime(), nullable=False),
+    sa.Column('s3_json_upload_timestamp', sa.DateTime(), nullable=True),
+    sa.Column('uprn', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
+    sa.PrimaryKeyConstraint('id')
+    )
+    op.create_index(op.f('ix_uploaded_files_s3_file_uri'), 'uploaded_files', ['s3_file_uri'], unique=False)
+    op.create_index(op.f('ix_uploaded_files_uprn'), 'uploaded_files', ['uprn'], unique=False)
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    """Downgrade schema."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_index(op.f('ix_uploaded_files_uprn'), table_name='uploaded_files')
+    op.drop_index(op.f('ix_uploaded_files_s3_file_uri'), table_name='uploaded_files')
+    op.drop_table('uploaded_files')
+    # ### end Alembic commands ###

alembic/versions/a8cc4a5fccb6_json_uri_is_a_string.py

@@ -0,0 +1,38 @@
+"""json_uri is a string
+
+Revision ID: a8cc4a5fccb6
+Revises: 29113d69989e
+Create Date: 2025-08-19 12:35:59.456912
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision: str = 'a8cc4a5fccb6'
+down_revision: Union[str, None] = '29113d69989e'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    """Upgrade schema."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.alter_column('uploaded_files', 's3_json_uri',
+               existing_type=postgresql.JSON(astext_type=sa.Text()),
+               type_=sa.Text(),
+               existing_nullable=True)
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    """Downgrade schema."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.alter_column('uploaded_files', 's3_json_uri',
+               existing_type=sa.Text(),
+               type_=postgresql.JSON(astext_type=sa.Text()),
+               existing_nullable=True)
+    # ### end Alembic commands ###

alembic/versions/ac8dba8cef50_added_more_report_type.py

@@ -0,0 +1,38 @@
+"""added more report type
+
+Revision ID: ac8dba8cef50
+Revises: a8cc4a5fccb6
+Create Date: 2025-09-23 10:14:54.461633
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision: str = 'ac8dba8cef50'
+down_revision: Union[str, None] = 'a8cc4a5fccb6'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    """Upgrade schema."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.alter_column('uploaded_files', 'id',
+               existing_type=sa.UUID(),
+               server_default=None,
+               existing_nullable=False)
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    """Downgrade schema."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.alter_column('uploaded_files', 'id',
+               existing_type=sa.UUID(),
+               server_default=sa.text('gen_random_uuid()'),
+               existing_nullable=False)
+    # ### end Alembic commands ###

alembic/versions/c8af22cece92_s3_add.py

@@ -0,0 +1,33 @@
+"""s3 add 
+
+Revision ID: c8af22cece92
+Revises: ed6aaa298de4
+Create Date: 2025-11-07 15:00:32.917157
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+import sqlmodel
+
+
+# revision identifiers, used by Alembic.
+revision: str = 'c8af22cece92'
+down_revision: Union[str, None] = 'ed6aaa298de4'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    """Upgrade schema."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('hubspot_deal_data', sa.Column('major_condition_issue_evidence_s3_url', sqlmodel.sql.sqltypes.AutoString(), nullable=True))
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    """Downgrade schema."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column('hubspot_deal_data', 'major_condition_issue_evidence_s3_url')
+    # ### end Alembic commands ###

alembic/versions/e72e15f7e0c3_delete_raw_data.py

@@ -0,0 +1,34 @@
+"""delete raw_data
+
+Revision ID: e72e15f7e0c3
+Revises: 2409147995c5
+Create Date: 2025-10-27 15:31:20.870827
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision: str = 'e72e15f7e0c3'
+down_revision: Union[str, None] = '2409147995c5'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    """Upgrade schema."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_index(op.f('ix_hubspot_data_deal_id'), 'hubspot_data', ['deal_id'], unique=False)
+    op.drop_column('hubspot_data', 'raw_data')
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    """Downgrade schema."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('hubspot_data', sa.Column('raw_data', postgresql.JSON(astext_type=sa.Text()), autoincrement=False, nullable=True))
+    op.drop_index(op.f('ix_hubspot_data_deal_id'), table_name='hubspot_data')
+    # ### end Alembic commands ###

alembic/versions/e8507a27795a_auto_generate_id.py

@@ -0,0 +1,32 @@
+"""auto generate id
+
+Revision ID: e8507a27795a
+Revises: 1f1d3d560ccb
+Create Date: 2025-08-14 17:20:03.632337
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision: str = 'e8507a27795a'
+down_revision: Union[str, None] = '1f1d3d560ccb'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    """Upgrade schema."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    pass
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    """Downgrade schema."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    pass
+    # ### end Alembic commands ###

alembic/versions/eccdd5f607c1_add_coorodiantion_and_design_status.py

@@ -0,0 +1,35 @@
+"""add coorodiantion and design status
+
+Revision ID: eccdd5f607c1
+Revises: c8af22cece92
+Create Date: 2026-02-19 17:07:14.085232
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+import sqlmodel
+
+
+# revision identifiers, used by Alembic.
+revision: str = 'eccdd5f607c1'
+down_revision: Union[str, None] = 'c8af22cece92'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    """Upgrade schema."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('hubspot_deal_data', sa.Column('coordination_status', sqlmodel.sql.sqltypes.AutoString(), nullable=True))
+    op.add_column('hubspot_deal_data', sa.Column('design_status', sqlmodel.sql.sqltypes.AutoString(), nullable=True))
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    """Downgrade schema."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column('hubspot_deal_data', 'design_status')
+    op.drop_column('hubspot_deal_data', 'coordination_status')
+    # ### end Alembic commands ###

alembic/versions/ed6aaa298de4_added_major_condition_issue_things.py

@@ -0,0 +1,34 @@
+"""added major condition issue things
+
+Revision ID: ed6aaa298de4
+Revises: 66d2c9c325d6
+Create Date: 2025-11-05 15:03:11.447367
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+import sqlmodel
+
+# revision identifiers, used by Alembic.
+revision: str = 'ed6aaa298de4'
+down_revision: Union[str, None] = '66d2c9c325d6'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    """Upgrade schema."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('hubspot_deal_data', sa.Column('major_condition_issue_description', sqlmodel.sql.sqltypes.AutoString(), nullable=True))
+    op.add_column('hubspot_deal_data', sa.Column('major_condition_issue_photos', sqlmodel.sql.sqltypes.AutoString(), nullable=True))
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    """Downgrade schema."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column('hubspot_deal_data', 'major_condition_issue_photos')
+    op.drop_column('hubspot_deal_data', 'major_condition_issue_description')
+    # ### end Alembic commands ###

deployment/database/provider.tf

@@ -0,0 +1,15 @@
+terraform {
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = "~> 6.3.0"
+    }
+  }
+  backend "s3" {
+      = "survey-extractor-tf-state"
+    region  = "eu-west-2"
+    key     = "env:/dev/terraform.tfstate"
+  }
+
+  required_version = ">= 1.2.0"
+}

deployment/database/variables.tf

@@ -14,4 +14,4 @@ variable allocated_storage {
   description = "The allocated storage in gigabytes"
   type        = number
   default     = 20
-}
+}

deployment/lambda/extractor_and_loader/docker/.dockerignore

@@ -0,0 +1,21 @@
+# Ignore junk and large files
+*.pdf
+*.csv
+*.xml
+*.parquet
+*.ipynb
+*.mp4
+*.mov
+*.jpg
+*.png
+*.zip
+*.tar.gz
+__pycache__/
+*.pyc
+*.pyo
+*.pyd
+build/
+dist/
+.etl_cache/
+tests/
+docs/

deployment/lambda/extractor_and_loader/docker/Dockerfile

@@ -0,0 +1,25 @@
+FROM public.ecr.aws/lambda/python:3.12
+
+# Install Poetry (you could pin a version if you like)
+RUN curl -sSL https://install.python-poetry.org | python3 -
+
+# Add Poetry to PATH
+ENV PATH="/root/.local/bin:$PATH"
+
+# Set working directory
+WORKDIR /var/task
+
+# Copy Poetry files first to leverage Docker layer caching
+COPY pyproject.toml poetry.lock README.md ./
+COPY etl/ etl/
+
+
+# Install dependencies into /var/task
+RUN poetry config virtualenvs.create false \
+  && poetry install --only main --no-interaction --no-ansi
+
+# Copy app code
+COPY deployment/lambda/extractor_and_loader/docker/app.py ./
+
+# Set Lambda handler
+CMD ["app.handler"]

deployment/lambda/extractor_and_loader/docker/app.py

@@ -0,0 +1,244 @@
+import os
+import tempfile
+import requests
+import boto3
+from urllib.parse import urlparse
+from etl.fileReader.pdfReaderToText import pdfReaderToText
+from etl.fileReader.sitenotes import (
+    SiteNotesExtractor,
+    WarmHomesConditionReport
+)
+from uuid import UUID
+import json
+from typing import Any
+from etl.db.db import get_db_session, init_db
+from typing import Union
+import uuid
+from datetime import datetime, timezone
+from sqlmodel import select
+from sqlalchemy import update
+from etl.models.topLevel import uploaded_files
+
+def update_uploaded_file_json_uri_by_query(
+    db_session,
+    file_id: Union[str, uuid.UUID],
+    json_uri: str,
+):
+    """
+    Query uploaded_files by id, update s3_json_uri and s3_json_upload_timestamp,
+    commit, refresh, and return the ORM object. Raises ValueError if not found.
+    """
+    try:
+        file_id_norm = uuid.UUID(str(file_id))
+    except (ValueError, AttributeError, TypeError):
+        file_id_norm = file_id  # leave as-is if not a UUID
+
+    obj = (
+        db_session
+        .query(uploaded_files)
+        .filter(uploaded_files.id == file_id_norm)
+        .first()
+    )
+    obj.s3_json_uri = json_uri
+    obj.s3_json_upload_timestamp = datetime.now(timezone.utc)
+
+    db_session.add(obj)
+    db_session.commit()
+    db_session.refresh(obj)
+    return obj
+
+def serialize_model(model: Any):
+    """Recursively convert Pydantic models/lists into plain dicts."""
+    if hasattr(model, "dict"):
+        return {k: serialize_model(v) for k, v in model.dict().items()}
+    elif isinstance(model, list):
+        return [serialize_model(item) for item in model]
+    else:
+        return model
+
+def make_final_json(rooms_obj, heating_system_obj, occupant, access_and_elevations, bepoke_info):
+    # Convert to dict recursively
+    rooms_data = serialize_model(rooms_obj)
+    heating_data = serialize_model(heating_system_obj)
+    occupant_data = serialize_model(occupant)
+    access_and_elevations_data = serialize_model(access_and_elevations)
+
+    # Combine into one big JSON-ready dict
+    final_data = {
+        "rooms": rooms_data,
+        "heating_system": heating_data,
+        "occupant_info": occupant_data,
+        "access_and_elevations": access_and_elevations_data,
+        "bespoke_data": bepoke_info
+    }
+
+    # Convert to pretty JSON string
+    return final_data
+
+def parse_s3_uri(uri: str):
+    """
+    Parse an S3 URI or HTTPS S3 URL into bucket and key.
+    Supports formats:
+      - s3://bucket-name/path/to/file
+      - https://bucket-name.s3.region.amazonaws.com/path/to/file
+    """
+    parsed = urlparse(uri)
+
+    if parsed.scheme == "s3":
+        # s3://bucket/key
+        bucket = parsed.netloc
+        key = parsed.path.lstrip("/")
+    elif parsed.scheme in ("http", "https"):
+        # https://bucket-name.s3.region.amazonaws.com/key
+        host_parts = parsed.netloc.split(".")
+        if len(host_parts) >= 3 and host_parts[1] == "s3":
+            bucket = host_parts[0]
+        else:
+            raise ValueError("Not a valid S3 HTTPS URL format")
+        key = parsed.path.lstrip("/")
+    else:
+        raise ValueError("Unsupported URI scheme")
+
+    return bucket, key
+
+
+def download_private_s3_file(uri) -> str:
+    bucket_name, key = parse_s3_uri(uri)
+    """
+    Download a private S3 file using hardcoded AWS credentials.
+    Saves it to /tmp and returns the local file path.
+    """
+
+    # Hardcoded AWS credentials (quick testing only)
+    aws_access_key = "AKIAU5A36PPNJMZZ3KRW"
+    aws_secret_key = "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"
+    aws_region = "eu-west-2"
+
+    # Where to store the file locally
+    tmp_dir = tempfile.gettempdir()
+    filename = os.path.basename(key)
+    file_path = os.path.join(tmp_dir, filename)
+
+    # Create S3 client with hardcoded creds
+    s3 = boto3.client(
+        "s3",
+        aws_access_key_id=aws_access_key,
+        aws_secret_access_key=aws_secret_key,
+        region_name=aws_region
+    )
+
+    # Download file
+    s3.download_file(bucket_name, key, file_path)
+
+    return file_path
+
+def upload_json_to_s3(json_obj, dest_uri: str) -> str:
+    """
+    Upload a JSON-serializable object to S3 at the given s3:// or https S3 URL.
+    Returns the public-style HTTPS S3 URL (still private if bucket is private).
+    """
+    bucket, pdf_key = parse_s3_uri(dest_uri)
+    base_folder = os.path.dirname(pdf_key)  # e.g. ".../report"
+
+    # Build jsonified folder + timestamp filename
+    timestamp = datetime.utcnow().strftime("%Y%m%d_%H%M%S")
+    json_key = f"{base_folder}/jsonified/{timestamp}.json"
+
+    # Same region/creds you used for download
+    aws_access_key = "AKIAU5A36PPNJMZZ3KRW"
+    aws_secret_key = "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"
+    aws_region = "eu-west-2"
+
+    s3 = boto3.client(
+        "s3",
+        aws_access_key_id=aws_access_key,
+        aws_secret_access_key=aws_secret_key,
+        region_name=aws_region
+    )
+
+    body = json.dumps(json_obj, ensure_ascii=False, indent=2).encode("utf-8")
+
+    s3.put_object(
+        Bucket=bucket,
+        Key=json_key,
+        Body=body,
+        ContentType="application/json"
+        # Optional hardening:
+        # , ServerSideEncryption="AES256"
+    )
+
+    # Return an HTTPS-style S3 URL (matches your input style)
+    return f"https://{bucket}.s3.{aws_region}.amazonaws.com/{json_key}"
+
+def get_file_uri(id):
+    with get_db_session() as session:
+        obj = (
+            session
+            .query(uploaded_files)
+            .filter(uploaded_files.id == id)
+            .first()
+        )
+        if obj is None:
+            raise RuntimeError(f"Failed to find uploaded_files record with id {id}")
+        
+        return obj.s3_file_uri
+
+
+def handler(event, context):
+    try:
+        print("trying to connect to db")
+        init_db()
+        print("connected to db")
+        for r in event.get("Records", []):
+            body = json.loads(r["body"])
+            id_ = body.get("id") 
+            if not id_:  # covers None or empty string
+                raise ValueError(f"❌ Missing 'id' in SQS body: {body}")
+
+            print(f"Retrieving file uri with id {id_}") 
+            file_uri = get_file_uri(id_)
+            print(f"Retrieved file uri with  {file_uri}") 
+
+            print("Downloading file locally for extraction...")
+            local_path = download_private_s3_file(file_uri)
+
+            # Local development of file, please comment out for prod
+            # local_path = os.path.join(os.path.join(os.getcwd(), "../../../../../", "home/Downloads/works/67-Aylestone-Road-1 1.pdf"))
+            # local_path = os.path.join(os.path.join(os.getcwd(), "../../../../../", "home/Downloads/works/2-Wilford-Crescent-West.pdf"))
+            # local_path = os.path.join(os.path.join(os.getcwd(), "../../../../../", "home/Downloads/works/3-Carlinghow-court.pdf"))
+            # local_path = os.path.join(os.path.join(os.getcwd(), "../../../../../", "home/Downloads/works/26-Marden-Road.pdf"))
+            # local_path = os.path.join(os.path.join(os.getcwd(), "../../../../../", "home/Downloads/works/6E-plantagenet-street.pdf"))
+
+
+
+            print("Extracting file...")
+            reader = pdfReaderToText(local_path)
+            # obj2 = WarmHomesConditionReport(reader.text_list, debug=True)
+            obj = WarmHomesConditionReport(reader.text_list)
+            json_ = make_final_json(
+                obj.master_obj[0],
+                obj.master_obj[1],
+                obj.master_obj[2],
+                obj.master_obj[3],
+                {}
+            )
+            print("Extracted completed, made json")
+            
+            print("uploading json to s3 bucket...")
+            json_uri = upload_json_to_s3(json_, file_uri)
+
+            print("Updating Database with json_uri")
+            with get_db_session() as session:
+                update_uploaded_file_json_uri_by_query(
+                    session, 
+                    id_,
+                    json_uri,
+                )
+            print("job completed successfully")
+
+    except Exception as e:
+        print(f"❌ Error: {e}")
+        return {
+            "statusCode": 500,
+            "body": str(e)
+        }

deployment/lambda/extractor_and_loader/docker/ecr.tf

@@ -0,0 +1,62 @@
+# ECR repo
+resource "aws_ecr_repository" "extractor_and_loader" {
+  name = "extractor_and_loader"
+}
+
+# ECR policy to allow Lambda access
+resource "aws_ecr_repository_policy" "extractor_loader_ecr_access" {
+  repository = aws_ecr_repository.extractor_and_loader.name
+
+  policy = jsonencode({
+    Version = "2008-10-17",
+    Statement = [{
+      Sid       = "AllowLambdaPull",
+      Effect    = "Allow",
+      Principal = {
+        Service = "lambda.amazonaws.com"
+      },
+      Action = [
+        "ecr:GetDownloadUrlForLayer",
+        "ecr:BatchGetImage",
+        "ecr:BatchCheckLayerAvailability"
+      ]
+    }]
+  })
+}
+
+
+# ECR lifecycle policy to delete tagged images older than 14 days
+resource "aws_ecr_lifecycle_policy" "extractor_loader_lifecycle" {
+  repository = aws_ecr_repository.extractor_and_loader.name
+
+  policy = jsonencode({
+    "rules": [
+        {
+            "rulePriority": 2,
+            "description": "Expire images older than 14 days",
+            "selection": {
+                "tagStatus": "untagged",
+                "countType": "sinceImagePushed",
+                "countUnit": "days",
+                "countNumber": 1
+            },
+            "action": {
+                "type": "expire"
+            }
+        },
+        {
+            "rulePriority": 1,
+            "description": "Keep last 5 images",
+            "selection": {
+                "tagStatus": "tagged",
+                "tagPrefixList": ["feature"],
+                "countType": "imageCountMoreThan",
+                "countNumber": 5
+            },
+            "action": {
+                "type": "expire"
+            }
+        }
+    ]
+  })
+}

deployment/lambda/extractor_and_loader/docker/provider.tf

@@ -2,14 +2,13 @@ terraform {
   required_providers {
     aws = {
       source  = "hashicorp/aws"
-      version = "~> 4.16"
+      version = "~> 6.3.0"
     }
   }
   backend "s3" {
     bucket  = "survey-extractor-tf-state"
     region  = "eu-west-2"
-    profile = "domna.dev" # /home/vscode/aws/credentials
-    key     = "terraform.tfstate"
+    key     = "env:/dev/lambda/ecr/extractor_and_loader.tfstate"
   }
 
   required_version = ">= 1.2.0"

deployment/lambda/extractor_and_loader/extractor_and_loader_lambda.tf

@@ -0,0 +1,85 @@
+# Reference existing IAM role
+data "aws_iam_role" "lambda_exec_role" {
+  name = "lambda-exec-role"
+}
+
+# Reference existing ECR repository
+data "aws_ecr_repository" "extractor_and_loader" {
+  name = "extractor_and_loader"
+  
+}
+
+# SQS queue for extractor_and_loader
+resource "aws_sqs_queue" "extractor_and_loader_queue" {
+  name = "extractor-loader-queue"
+  visibility_timeout_seconds  = 1800  # 30 minutes (>= 300s and ~6x Lambda timeout)
+}
+
+
+# Custom IAM policy specific to lambda_example
+resource "aws_iam_policy" "extractor_loader_policy" {
+  name = "extractor_loader_policy"
+
+  policy = jsonencode({
+    Version = "2012-10-17",
+    Statement = [
+      {
+        Effect = "Allow",
+        Action = [
+          "sqs:ReceiveMessage",
+          "sqs:DeleteMessage",
+          "sqs:GetQueueAttributes",
+          "sqs:GetQueueUrl",
+          "sqs:ChangeMessageVisibility"
+        ],
+        Resource = aws_sqs_queue.extractor_and_loader_queue.arn
+      },
+      {
+        Effect = "Allow",
+        Action = [
+          "ecr:GetDownloadUrlForLayer",
+          "ecr:BatchGetImage",
+          "ecr:BatchCheckLayerAvailability"
+        ],
+        Resource = data.aws_ecr_repository.extractor_and_loader.arn
+      },
+      {
+        Effect = "Allow",
+        Action = ["ecr:GetAuthorizationToken"],
+        Resource = "*"
+      }
+    ]
+  })
+}
+
+resource "aws_iam_role_policy_attachment" "extractor_loader_policy_attach" {
+  role       = data.aws_iam_role.lambda_exec_role.name
+  policy_arn = aws_iam_policy.extractor_loader_policy.arn
+}
+
+# Lambda function
+resource "aws_lambda_function" "extractor_and_loader" {
+  function_name = "extractor-and-loader-lambda"
+  role          = data.aws_iam_role.lambda_exec_role.arn
+  package_type  = "Image"
+  image_uri     = "${data.aws_ecr_repository.extractor_and_loader.repository_url}:${var.lambda_image_tag}"
+  # Increase timeout (max 900 sec / 15 min)
+  timeout       = 300   # e.g. 5 minutes
+
+  # Increase memory (default 128 MB)
+  memory_size   = 2048  # try 1024 or 2048 MB to start
+
+  environment {
+    variables = {
+      DATABASE_URL = "postgresql://postgres:makingwarmhomes@terraform-20250331175522503500000002.cdgzupxvdyp0.eu-west-2.rds.amazonaws.com:5432/surveyDB"
+    }
+  }
+}
+
+# SQS trigger
+resource "aws_lambda_event_source_mapping" "extractor_and_loader_trigger" {
+  event_source_arn = aws_sqs_queue.extractor_and_loader_queue.arn
+  function_name    = aws_lambda_function.extractor_and_loader.arn
+  batch_size       = 1
+}
+

deployment/lambda/extractor_and_loader/provider.tf

@@ -0,0 +1,15 @@
+terraform {
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = "~> 6.3.0"
+    }
+  }
+  backend "s3" {
+    bucket  = "survey-extractor-tf-state"
+    region  = "eu-west-2"
+    key     = "env:/dev/lambda/eachlambda/extractor_and_loader_lambda.tfstate"
+  }
+
+  required_version = ">= 1.2.0"
+}

deployment/lambda/extractor_and_loader/vars.tf

@@ -0,0 +1,5 @@
+variable "lambda_image_tag" {
+  description = "Docker image tag (e.g. GitHub SHA)"
+  type        = string
+  default     = "local-dev-latest"
+}

deployment/lambda/lambda_example/docker/Dockerfile

@@ -0,0 +1,26 @@
+# AWS Lambda python pacakge
+FROM public.ecr.aws/lambda/python:3.12
+
+# Install Poetry (you could pin a version if you like)
+RUN curl -sSL https://install.python-poetry.org | python3 -
+
+# Add Poetry to PATH
+ENV PATH="/root/.local/bin:$PATH"
+
+# Set working directory
+WORKDIR /var/task
+
+# Copy Poetry files first to leverage Docker layer caching
+COPY pyproject.toml poetry.lock README.md ./
+COPY etl/ etl/
+
+
+# Install dependencies into /var/task
+RUN poetry config virtualenvs.create false \
+  && poetry install --only main --no-interaction --no-ansi
+
+# Copy app code
+COPY deployment/lambda/lambda_example/docker/app.py ./
+
+# Set the CMD to your handler
+CMD ["app.handler"]

deployment/lambda/lambda_example/docker/app.py

@@ -0,0 +1,9 @@
+from etl.hubSpotClient.hubspotClient import HubSpotClient
+
+def handler(event, context):
+    nums = [
+    ]
+
+    hubspot = HubSpotClient()
+    for num in nums:
+        hubspot.delete_line_item(num)

deployment/lambda/lambda_example/docker/ecr.tf

@@ -0,0 +1,61 @@
+# ECR repo for lambda_example
+resource "aws_ecr_repository" "lambda_example" {
+  name = "lambda_example"
+}
+
+# ECR policy to allow Lambda access
+resource "aws_ecr_repository_policy" "lambda_example_ecr_access" {
+  repository = aws_ecr_repository.lambda_example.name
+
+  policy = jsonencode({
+    Version = "2008-10-17",
+    Statement = [{
+      Sid       = "AllowLambdaPull",
+      Effect    = "Allow",
+      Principal = {
+        Service = "lambda.amazonaws.com"
+      },
+      Action = [
+        "ecr:GetDownloadUrlForLayer",
+        "ecr:BatchGetImage",
+        "ecr:BatchCheckLayerAvailability"
+      ]
+    }]
+  })
+}
+
+# ECR lifecycle policy to delete tagged images older than 14 days
+resource "aws_ecr_lifecycle_policy" "lambda_example_ecr_lifecycle" {
+  repository = aws_ecr_repository.lambda_example.name
+
+  policy = jsonencode({
+    "rules": [
+        {
+            "rulePriority": 2,
+            "description": "Expire images older than 14 days",
+            "selection": {
+                "tagStatus": "untagged",
+                "countType": "sinceImagePushed",
+                "countUnit": "days",
+                "countNumber": 1
+            },
+            "action": {
+                "type": "expire"
+            }
+        },
+        {
+            "rulePriority": 1,
+            "description": "Keep last 5 images",
+            "selection": {
+                "tagStatus": "tagged",
+                "tagPrefixList": ["feature"],
+                "countType": "imageCountMoreThan",
+                "countNumber": 5
+            },
+            "action": {
+                "type": "expire"
+            }
+        }
+    ]
+  })
+}

deployment/lambda/lambda_example/docker/output.tf

@@ -0,0 +1,3 @@
+output "ecr_repo_url" {
+  value = aws_ecr_repository.lambda_example.repository_url
+}

deployment/lambda/lambda_example/docker/provider.tf

@@ -0,0 +1,15 @@
+terraform {
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = "~> 6.3.0"
+    }
+  }
+  backend "s3" {
+    bucket  = "survey-extractor-tf-state"
+    region  = "eu-west-2"
+    key     = "env:/dev/lambda/ecr/lambda_example_ecr.tfstate"
+  }
+
+  required_version = ">= 1.2.0"
+}

deployment/lambda/lambda_example/lambda_example_and_config.tf

@@ -0,0 +1,73 @@
+# Reference existing IAM role
+data "aws_iam_role" "lambda_exec_role" {
+  name = "lambda-exec-role"
+}
+
+# Reference existing ECR repository
+data "aws_ecr_repository" "lambda_example" {
+  name = "lambda_example"
+}
+
+# SQS queue for lambda_example
+resource "aws_sqs_queue" "lambda_example_queue" {
+  name = "lambda-example-queue"
+}
+
+# Custom IAM policy specific to lambda_example
+resource "aws_iam_policy" "lambda_example_policy" {
+  name = "lambda-example-policy"
+
+  policy = jsonencode({
+    Version = "2012-10-17",
+    Statement = [
+      {
+        Effect = "Allow",
+        Action = [
+          "sqs:ReceiveMessage",
+          "sqs:DeleteMessage",
+          "sqs:GetQueueAttributes",
+          "sqs:GetQueueUrl",
+          "sqs:ChangeMessageVisibility"
+        ],
+        Resource = aws_sqs_queue.lambda_example_queue.arn
+      },
+      {
+        Effect = "Allow",
+        Action = [
+          "ecr:GetDownloadUrlForLayer",
+          "ecr:BatchGetImage",
+          "ecr:BatchCheckLayerAvailability"
+        ],
+        Resource = data.aws_ecr_repository.lambda_example.arn
+      },
+      {
+        Effect = "Allow",
+        Action = ["ecr:GetAuthorizationToken"],
+        Resource = "*"
+      }
+    ]
+  })
+}
+
+
+
+resource "aws_iam_role_policy_attachment" "lambda_example_policy_attach" {
+  role       = data.aws_iam_role.lambda_exec_role.name
+  policy_arn = aws_iam_policy.lambda_example_policy.arn
+}
+
+# Lambda function
+resource "aws_lambda_function" "lambda_example" {
+  function_name = "lambda-example"
+  role          = data.aws_iam_role.lambda_exec_role.arn
+  package_type  = "Image"
+  image_uri     = "${data.aws_ecr_repository.lambda_example.repository_url}:${var.lambda_image_tag}"
+  timeout       = 10
+}
+
+# SQS trigger
+resource "aws_lambda_event_source_mapping" "lambda_example_trigger" {
+  event_source_arn = aws_sqs_queue.lambda_example_queue.arn
+  function_name    = aws_lambda_function.lambda_example.arn
+  batch_size       = 1
+}

deployment/lambda/lambda_example/provider.tf

@@ -0,0 +1,15 @@
+terraform {
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = "~> 6.3.0"
+    }
+  }
+  backend "s3" {
+    bucket  = "survey-extractor-tf-state"
+    region  = "eu-west-2"
+    key     = "env:/dev/lambda/eachlambda/lambda_example.tfstate"
+  }
+
+  required_version = ">= 1.2.0"
+}

deployment/lambda/lambda_example/vars.tf

@@ -0,0 +1,5 @@
+variable "lambda_image_tag" {
+  description = "Docker image tag (e.g. GitHub SHA)"
+  type        = string
+  default     = "local-dev-latest"
+}

deployment/lambda/lambda_shared/lambda_shared_config.tf

@@ -0,0 +1,21 @@
+# IAM role for both Lambdas (can be shared)
+resource "aws_iam_role" "lambda_exec_role" {
+  name = "lambda-exec-role"
+
+  assume_role_policy = jsonencode({
+    Version = "2012-10-17",
+    Statement = [{
+      Effect = "Allow",
+      Principal = {
+        Service = "lambda.amazonaws.com"
+      },
+      Action = "sts:AssumeRole"
+    }]
+  })
+}
+
+
+resource "aws_iam_role_policy_attachment" "lambda_basic_execution" {
+  role       = aws_iam_role.lambda_exec_role.name
+  policy_arn = "arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole"
+}

deployment/lambda/lambda_shared/output.tf

@@ -0,0 +1,9 @@
+output "lambda_exec_role_arn" {
+  description = "The ARN of the IAM role used by the Lambda functions"
+  value       = aws_iam_role.lambda_exec_role.arn
+}
+
+output "lambda_exec_role_name" {
+  description = "The ARN of the IAM role used by the Lambda functions"
+  value       = aws_iam_role.lambda_exec_role.name
+}

deployment/lambda/lambda_shared/provider.tf

@@ -0,0 +1,15 @@
+terraform {
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = "~> 6.3.0"
+    }
+  }
+  backend "s3" {
+    bucket  = "survey-extractor-tf-state"
+    region  = "eu-west-2"
+    key     = "env:/dev/lambda/lambda_share_configuration.tfstate"
+  }
+
+  required_version = ">= 1.2.0"
+}

deployment/lambda/walthamforest_etl/docker/.dockerignore

@@ -0,0 +1,21 @@
+# Ignore junk and large files
+*.pdf
+*.csv
+*.xml
+*.parquet
+*.ipynb
+*.mp4
+*.mov
+*.jpg
+*.png
+*.zip
+*.tar.gz
+__pycache__/
+*.pyc
+*.pyo
+*.pyd
+build/
+dist/
+.etl_cache/
+tests/
+docs/

deployment/lambda/walthamforest_etl/docker/Dockerfile

@@ -0,0 +1,25 @@
+FROM public.ecr.aws/lambda/python:3.12
+
+# Install Poetry (you could pin a version if you like)
+RUN curl -sSL https://install.python-poetry.org | python3 -
+
+# Add Poetry to PATH
+ENV PATH="/root/.local/bin:$PATH"
+
+# Set working directory
+WORKDIR /var/task
+
+# Copy Poetry files first to leverage Docker layer caching
+COPY pyproject.toml poetry.lock README.md ./
+COPY etl/ etl/
+
+
+# Install dependencies into /var/task
+RUN poetry config virtualenvs.create false \
+  && poetry install --only main --no-interaction --no-ansi
+
+# Copy app code
+COPY deployment/lambda/extractor_and_loader/docker/app.py ./
+
+# Set Lambda handler
+CMD ["app.handler"]

deployment/lambda/walthamforest_etl/docker/app.py

@@ -0,0 +1,380 @@
+import pandas as pd
+import json
+from pprint import pprint
+import os
+import copy
+from collections import defaultdict
+from typing import List, Dict, Any, Union, Optional
+import boto3
+from urllib.parse import urlparse
+from decent_homes_pilot import decent_homes_calc
+import uuid
+from datetime import datetime, timezone, time, date
+from decimal import Decimal
+from sqlmodel import select
+from sqlalchemy import update
+from etl.models.topLevel import uploaded_files, ReportType
+from etl.db.db import get_db_session, init_db
+
+
+def process_complex(sheet_name, group_key="ADDRESS"):
+    df = pd.read_excel("../../../../../home/Downloads/data.xlsx", sheet_name=sheet_name)
+
+    element_cols = [
+        "ELEMENT GROUP", "ELEMENT CODE", "ELEMENT CODE DESCRIPTION",
+        "ATTRIBUTE CODE", "ATTRIBUTE CODE DESCRIPTION",
+        "ELEMENT DATE VALUE", "ELEMENT NUMERIC VALUE",
+        "ELEMENT TEXT VALUE", "QUANTITY",
+        "INSTALL DATE", "REMAINING LIFE", "ELEMENT COMMENTS"
+    ]
+
+    property_cols = [
+        "PROP REF", "ADDRESS", "OWNERSHIP",
+        "PROP STATUS", "PROP TYPE", "PROP SUB TYPE"
+    ]
+
+    # Prepare output
+    records = []
+
+    # Loop through unique values in group_key (ADDRESS or BLOCK_CODE)
+    for val in df[group_key].unique():
+        g = df[df[group_key] == val]   # subset
+        
+        property_info = g[property_cols].drop_duplicates().iloc[0].to_dict()
+        
+        # build elements dict keyed by ELEMENT CODE DESCRIPTION
+        elements_dict = {}
+        for _, row in g[element_cols].drop_duplicates().iterrows():
+            key = row["ELEMENT CODE DESCRIPTION"]  # could also use "ELEMENT CODE"
+            elements_dict[key] = row.to_dict()
+        
+        records.append({
+            group_key: val,
+            "property_info": property_info,
+            "elements": elements_dict
+        })
+
+    return records
+
+def process_simple(sheet_name):
+    df = pd.read_excel("../../../../../home/Downloads/data.xlsx", sheet_name=sheet_name)
+
+    records = []
+
+    for address in df["Address"].unique():
+        g = df[df["Address"] == address].drop_duplicates()   # subset for that address
+        row = g.iloc[0]  # take first row if multiple
+
+        # build dict of all columns except Address
+        elements_dict = row.drop(labels=["Address"]).to_dict()
+
+        records.append({
+            "ADDRESS": address,
+            "to_add": elements_dict
+        })
+
+    return records
+
+
+def combine_records_by_address(
+    asset_records: List[Dict[str, Any]],
+    simple_records: List[Dict[str, Any]],
+    dest_key: str = "to_add",
+    unique_identifier="Address"
+) -> List[Dict[str, Any]]:
+    """
+    Merge process_house_asset_data() and process_simple() results by ADDRESS.
+    All columns from simple_records['to_add'] will be merged under dest_key.
+    """
+    # Index inputs by ADDRESS
+    asset_by_addr = {r["ADDRESS"]: r for r in asset_records}
+    simple_by_addr = {r["ADDRESS"]: r for r in simple_records}
+
+    merged: List[Dict[str, Any]] = []
+
+    # Use union of addresses from both sources
+    all_addresses = set(asset_by_addr) | set(simple_by_addr)
+
+    for addr in sorted(all_addresses):
+        base = copy.deepcopy(asset_by_addr.get(addr, {"ADDRESS": addr}))
+        simple = simple_by_addr.get(addr)
+
+        if simple:
+            base[dest_key] = simple.get("to_add", {})
+
+        merged.append(base)
+
+    return merged
+
+def combine_records_for_flats(assets: dict, simple: list) -> dict:
+    """Attach BLOCK_INFO (from simple[0]) to each asset in assets."""
+    if not simple or not isinstance(simple[0], dict):
+        return assets  # nothing to add
+    
+    block_info = simple[0]
+
+    for record in assets:
+        # Make sure record is a dict
+        # record.update({"BLOCK_INFO": block_info})
+        for ele_desc in block_info["elements"]:
+            if ele_desc not in record["elements"]:
+                record["elements"].update({ele_desc:block_info["elements"][ele_desc]})
+
+    return assets
+
+def _json_default(o):
+    # datetimes → ISO 8601 strings
+    if isinstance(o, (datetime, date, time)):
+        return o.isoformat()
+    # decimals → float (or str if you need exactness)
+    if isinstance(o, Decimal):
+        return float(o)
+    # sets → lists
+    if isinstance(o, set):
+        return list(o)
+    # numpy/pandas types (optional)
+    try:
+        import numpy as np
+        import pandas as pd
+        if isinstance(o, (np.integer,)):
+            return int(o)
+        if isinstance(o, (np.floating,)):
+            return float(o)
+        if isinstance(o, (np.ndarray,)):
+            return o.tolist()
+        if isinstance(o, (pd.Timestamp,)):
+            return o.isoformat()
+    except Exception:
+        pass
+    # last resort: string
+    return str(o)
+
+def uprn_to_address():
+    df = pd.read_excel("../../../../../home/Downloads/data.xlsx", sheet_name="All Energy Breakdown ")
+    mapping = df.set_index('Address')['UPRN'].to_dict()
+    return mapping
+
+def stories_to_address():
+    df = pd.read_excel("../../../../../home/Downloads/data.xlsx", sheet_name="All Energy Breakdown ")
+    mapping = df.set_index('Address')['Storeys'].to_dict()
+    return mapping
+
+def parse_s3_uri(uri: str):
+    """
+    Parse an S3 URI or HTTPS S3 URL into bucket and key.
+    Supports formats:
+      - s3://bucket-name/path/to/file
+      - https://bucket-name.s3.region.amazonaws.com/path/to/file
+    """
+    parsed = urlparse(uri)
+
+    if parsed.scheme == "s3":
+        # s3://bucket/key
+        bucket = parsed.netloc
+        key = parsed.path.lstrip("/")
+    elif parsed.scheme in ("http", "https"):
+        # https://bucket-name.s3.region.amazonaws.com/key
+        host_parts = parsed.netloc.split(".")
+        if len(host_parts) >= 3 and host_parts[1] == "s3":
+            bucket = host_parts[0]
+        else:
+            raise ValueError("Not a valid S3 HTTPS URL format")
+        key = parsed.path.lstrip("/")
+    else:
+        raise ValueError("Unsupported URI scheme")
+
+    return bucket, key
+
+def upload_json_to_s3(json_obj, dest_uri: str, location="decent_homes/raw_data") -> str:
+    """
+    Upload a JSON-serializable object to S3 at the given s3:// or https S3 URL.
+    Returns the public-style HTTPS S3 URL (still private if bucket is private).
+    """
+    bucket, pdf_key = parse_s3_uri(dest_uri)
+    base_folder = os.path.dirname(pdf_key)  # e.g. ".../report"
+
+    # Build jsonified folder + timestamp filename
+    timestamp = datetime.utcnow().strftime("%Y%m%d_%H%M%S")
+    json_key = f"{base_folder}/{location}/jsonified/{timestamp}.json"
+
+    # Same region/creds you used for download
+    aws_access_key = "AKIAU5A36PPNJMZZ3KRW"
+    aws_secret_key = "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"
+    aws_region = "eu-west-2"
+
+    s3 = boto3.client(
+        "s3",
+        aws_access_key_id=aws_access_key,
+        aws_secret_access_key=aws_secret_key,
+        region_name=aws_region
+    )
+
+    body = json.dumps(json_obj, ensure_ascii=False, indent=2,  default=_json_default).encode("utf-8")
+
+    s3.put_object(
+        Bucket=bucket,
+        Key=json_key,
+        Body=body,
+        ContentType="application/json"
+        # Optional hardening:
+        # , ServerSideEncryption="AES256"
+    )
+
+    # Return an HTTPS-style S3 URL (matches your input style)
+    return f"https://{bucket}.s3.{aws_region}.amazonaws.com/{json_key}"
+
+
+
+def generate_file_uri(UPRN):
+    timestamp = datetime.utcnow().strftime("%Y%m%d_%H%M%S")
+    file_uri = f"https://retrofit-energy-assessments-dev.s3.eu-west-2.amazonaws.com/documents/{UPRN}/"
+    return file_uri
+
+def create_or_update_uploaded_file_entry(
+    db_session,
+    uprn: str,
+    doc_type: ReportType,
+    json_uri: str,
+    s3_file_uri: str
+):
+    """
+    Create or update an entry in uploaded_files.
+    - If a record with the same (uprn, doc_type) exists, update it.
+    - Otherwise, insert a new record.
+    Commits, refreshes, and returns the ORM object.
+    """
+    existing = (
+        db_session.query(uploaded_files)
+        .filter(uploaded_files.uprn == str(uprn), uploaded_files.doc_type == doc_type)
+        .one_or_none()
+    )
+
+    if existing:
+        # Update existing record
+        existing.s3_json_uri = json_uri
+        existing.s3_json_upload_timestamp = datetime.now(timezone.utc)
+        existing.s3_file_uri = s3_file_uri
+        obj = existing
+    else:
+        # Insert new record
+        obj = uploaded_files(
+            doc_type=doc_type,
+            s3_json_uri=json_uri,
+            s3_json_upload_timestamp=datetime.now(timezone.utc),
+            s3_file_uri=s3_file_uri,
+            uprn=str(uprn),
+        )
+        db_session.add(obj)
+
+    db_session.commit()
+    db_session.refresh(obj)
+    return obj
+
+def handler(event, context):
+
+    uprn_mapping = uprn_to_address()
+    flats_to_stories = stories_to_address() 
+
+    # read data for houses only
+    assets = process_complex("Houses Asset Data")
+    simple = process_simple("Houses")         
+    houses = combine_records_by_address(assets, simple, dest_key="EPC_DATA")
+
+    for house in houses:
+        pseudo_name = house["ADDRESS"].split(",")[0]
+        if pseudo_name.lower() in (k.lower() for k in uprn_mapping.keys()):
+            house.update({"UPRN": uprn_mapping[pseudo_name.upper()]})
+    
+    #upload to s3
+
+
+    for i,house in enumerate(houses):
+        uprn = house["UPRN"]
+        print(uprn)
+        json_uri = upload_json_to_s3(house, generate_file_uri(house["UPRN"]), location="decent_homes/raw_data")
+
+        # Save JSON locally
+        filename = f"{uprn}.json"
+        filepath = os.path.join("output", filename)  # saves inside an "output" folder
+        os.makedirs("output", exist_ok=True)         # make sure folder exists
+        
+        with open(filepath, "w", encoding="utf-8") as f:
+            json.dump(house, f, indent=2, ensure_ascii=False, default=_json_default)
+
+        property_decent_home, decent_home_meta = decent_homes_calc(filepath) 
+
+        json_uri_1 = upload_json_to_s3(property_decent_home, generate_file_uri(uprn), location="decent_homes/property_decent_home")
+        with get_db_session() as session:
+            create_or_update_uploaded_file_entry(
+                db_session=session,
+                uprn=uprn,
+                doc_type=ReportType.DECENT_HOMES_SUMMARY,
+                json_uri=json_uri_1,
+                s3_file_uri=json_uri,
+            )
+        json_uri_1 = upload_json_to_s3(decent_home_meta, generate_file_uri(uprn), location="decent_homes/decent_homes_meta")
+        with get_db_session() as session:
+            create_or_update_uploaded_file_entry(
+                db_session=session,
+                uprn=uprn,
+                doc_type=ReportType.DECENT_HOMES_PROPERTY_META,
+                json_uri=json_uri_1,
+                s3_file_uri=json_uri,
+            )
+    
+    # read data for flats 
+    assets = process_complex("Chingford Rd 236-256 Properties")        
+    simple = process_complex("CHINGFORD ROAD 236-254 Asset Bl", "BLOCK_CODE")
+    flats = combine_records_for_flats(assets, simple)
+
+    for house in flats:
+            pseudo_name = house["ADDRESS"].split(",")[0]
+            if pseudo_name.lower() in (k.lower() for k in uprn_mapping.keys()):
+                print(uprn_mapping[pseudo_name.upper()])
+
+                house.update({"UPRN": uprn_mapping[pseudo_name.upper()]})
+                house["property_info"].update({"FLAT LEVEL": flats_to_stories[pseudo_name.upper()]})
+
+
+
+    for i,house in enumerate(flats):
+        uprn = house["UPRN"]
+        print(uprn)
+        json_uri = upload_json_to_s3(house, generate_file_uri(house["UPRN"]))
+
+        # Save JSON locally
+        filename = f"{house['UPRN']}.json"
+        filepath = os.path.join("output", filename)  # saves inside an "output" folder
+        os.makedirs("output", exist_ok=True)         # make sure folder exists
+        
+        with open(filepath, "w", encoding="utf-8") as f:
+            json.dump(house, f, indent=2, ensure_ascii=False, default=_json_default)
+        
+        property_decent_home, decent_home_meta = decent_homes_calc(filepath) 
+
+        json_uri_1 = upload_json_to_s3(property_decent_home, generate_file_uri(uprn), location="decent_homes/property_decent_home")
+        with get_db_session() as session:
+            create_or_update_uploaded_file_entry(
+                db_session=session,
+                uprn=uprn,
+                doc_type=ReportType.DECENT_HOMES_SUMMARY,
+                json_uri=json_uri_1,
+                s3_file_uri=json_uri,
+            )
+        json_uri_1 = upload_json_to_s3(decent_home_meta, generate_file_uri(uprn), location="decent_homes/decent_homes_meta")
+        with get_db_session() as session:
+            create_or_update_uploaded_file_entry(
+                db_session=session,
+                uprn=uprn,
+                doc_type=ReportType.DECENT_HOMES_PROPERTY_META,
+                json_uri=json_uri_1,
+                s3_file_uri=json_uri,
+            )
+        
+        # Keep track of saved file path
+
+    
+
+# To Do:
+# [Jun-te] Spec of quesation that we have for waltham forest
+# [Khalim] A document that has our mapping and our understanding of our data

deployment/lambda/walthamforest_etl/docker/decent_homes_pilot.py

@@ -0,0 +1,811 @@
+import json
+import os
+
+import pandas as pd
+
+from datetime import datetime
+
+from docutils.nodes import table
+
+
+def years_between(d1, d2):
+    # precise year difference (accounts for months/days)
+    return (d1.year - d2.year) - ((d1.month, d1.day) < (d2.month, d2.day))
+
+
+def get_element(elements, label):
+    """Safely get an element dict by display label (your JSON keys)."""
+    return elements.get(label)
+
+
+def append_result(decent_homes_meta, criteria, variable, sub_variable, result, install_date=None, expiry_date=None):
+    decent_homes_meta.append({
+        "criteria": criteria,
+        "variable": variable,
+        "sub_variable": sub_variable,
+        "result": result,
+        "hhsrs_rank": None,
+        "hhsrs_score": None,
+        "install_date": install_date,
+        "expiry_date": expiry_date,
+    })
+
+
+def decent_homes_calc(one_property):
+    # Read in static json, which is transformed by Jun-te's script
+    folder = "../../../../../home/Downloads/"
+    fn = one_property
+
+    # filenames = ["flat 1.json", "house 1.json"]
+
+    houses_waltham_forest_data = pd.read_excel(
+        os.path.join(folder, "data.xlsx"),
+        sheet_name="Houses Asset Data"
+    )
+    flats_waltham_forest_data = pd.read_excel(
+        os.path.join(folder, "data.xlsx"),
+        sheet_name="CHINGFORD ROAD 236-254 Asset Bl"
+    )
+
+    # Standardised variables which will form the enums in the db
+    HHSRS_VARIABLES = [
+        "damp_and_mould_growth",
+        "excess_cold",
+        "excess_heat",
+        "asbestos_and_mm_fibres",
+        "biocides",
+        "carbon_monoxide_and_fuel_combustion_products",
+        "lead",
+        "radiation",
+        "uncombusted_fuel_gas",
+        "volatile_organic_compounds",
+        "crowding_and_space",
+        "entry_by_intruders",
+        "lighting",
+        "noise",
+        "domestic_hygiene_pests_and_refuse",
+        "food_safety",
+        "personal_hygiene_sanitation_and_drainage",
+        "water_supply",
+        "falls_associated_with_baths",
+        "falls_on_level_surfaces",
+        "falls_on_stairs_and_steps",
+        "falls_between_levels",
+        "electrical_hazards",
+        "fire",
+        "flames_hot_surfaces_and_materials",
+        "collision_and_entrapment",
+        "explosions",
+        "ergonomics",
+        "structural_collapse_and_falling_elements"
+    ]
+
+    ELEMENT_CODE_TO_DESCRIPTION = {
+        # One-to-one
+        "HHSRSDAMP": "damp_and_mould_growth",
+        "HHSRSCOLD": "excess_cold",
+        "HHSRSHEAT": "excess_heat",
+        "HHSRSASB": "asbestos_and_mm_fibres",
+        "HHSRSBIOC": "biocides",
+        "HHSRSLEAD": "lead",
+        "HHSRSRADIA": "radiation",
+        "HHSRSFUEL": "uncombusted_fuel_gas",
+        "HHSRSORGAN": "volatile_organic_compounds",
+        "HHSRSCROWD": "crowding_and_space",
+        "HHSRSENTRY": "entry_by_intruders",
+        "HHSRSLIGHT": "lighting",
+        "HHSRSNOISE": "noise",
+        "HHSRSDOMES": "domestic_hygiene_pests_and_refuse",
+        "HHSRSFOOD": "food_safety",
+        "HHSRSPERS": "personal_hygiene_sanitation_and_drainage",
+        "HHSRSWATER": "water_supply",
+        "HHSRSFBATH": "falls_associated_with_baths",
+        "HHSRSFLEVE": "falls_on_level_surfaces",
+        "HHSRSFSTAI": "falls_on_stairs_and_steps",
+        "HHSRSFBETW": "falls_between_levels",
+        "HHSRSELEC": "electrical_hazards",
+        "HHSRSFIRE": "fire",
+        "HHSRSFLAME": "flames_hot_surfaces_and_materials",
+        "HHSRSEXPLO": "explosions",
+        "HHSRSPOSI": "ergonomics",
+        "HHSRSSTRUC": "structural_collapse_and_falling_elements",
+
+        # One-to-many expansions
+        "HHSRSCO": "carbon_monoxide",
+        "HHSRSSO2": "sulphur_dioxide_and_smoke",
+        "HHSRSNO2": "nitrogen_dioxide",
+        "HHSRSENTRP": "collision_and_entrapment",
+        "HHSRSCLOW": "collision_hazards_and_low_headroom",
+    }
+
+    CRITERION_B_VARIABLES = [
+        "external_walls_structure", "lintels", "brickwork_spalling", "wall_finish", "roof_structure", "roof_finish",
+        "chimneys", "windows", "external_doors", "kitchens", "bathrooms", "central_heating_boiler",
+        "central_heating_distribution_system", "heating_other", "electrical_systems",
+    ]
+
+    CRITERION_C_VARIABLES = [
+        "kitchen_less_than_20_years_old", "kitchen_adequate_space_and_layout", "bathroom_less_than_30_years_old",
+        "bathroom_wc_appropriately_located", "adequate_external_noise_insulation", "adequate_common_entrance_areas",
+    ]
+
+    # Criterion C explicit age limits (different from component lifespans used elsewhere)
+    CRITERION_C_AGE_LIMITS = {
+        "kitchen_years_max": 20,
+        "bathroom_years_max": 30,
+    }
+
+    # Field labels as they appear in your JSON (based on your code)
+    LABEL_KITCHEN = "Adequacy of Kitchen and Type in Property"
+    LABEL_BATHROOM = "Adequacy of Bathroom Location in Property"
+    LABEL_NOISE = "Adequacy of Noise Insulation in Property"
+    LABEL_COMMON_CIRC = "Circulation Space in Common Area"  # flats only
+
+
+    STANDARD_HHSRS_MAPPING = {
+        "pass": ["TYPRISK"], 
+        "fail": ["MODRISK","SLIGHTRISK"], 
+        "no_data": ["TOBEASSESS"],
+    }
+
+    # Criterion A - mapping of HHSRS variables to Waltham forest element codes
+    HHSRS_MAPPING = {
+        "damp_and_mould_growth": {"HHSRSDAMP": STANDARD_HHSRS_MAPPING},
+        "excess_cold": {"HHSRSCOLD": STANDARD_HHSRS_MAPPING},
+        "excess_heat": {"HHSRSHEAT": STANDARD_HHSRS_MAPPING},
+        "asbestos_and_mm_fibres": {"HHSRSASB": STANDARD_HHSRS_MAPPING},
+        "biocides": {"HHSRSBIOC": STANDARD_HHSRS_MAPPING},
+        "carbon_monoxide_and_fuel_combustion_products": {
+            "HHSRSCO": STANDARD_HHSRS_MAPPING,
+            "HHSRSSO2": STANDARD_HHSRS_MAPPING,
+            "HHSRSNO2": STANDARD_HHSRS_MAPPING
+        },
+        "lead": {"HHSRSLEAD": STANDARD_HHSRS_MAPPING},
+        "radiation": {"HHSRSRADIA": STANDARD_HHSRS_MAPPING},
+        "uncombusted_fuel_gas": {"HHSRSFUEL": STANDARD_HHSRS_MAPPING},
+        "volatile_organic_compounds": {"HHSRSORGAN": STANDARD_HHSRS_MAPPING},
+        "crowding_and_space": {"HHSRSCROWD": STANDARD_HHSRS_MAPPING},
+        "entry_by_intruders": {"HHSRSENTRY": STANDARD_HHSRS_MAPPING},
+        "lighting": {"HHSRSLIGHT": STANDARD_HHSRS_MAPPING},
+        "noise": {"HHSRSNOISE": STANDARD_HHSRS_MAPPING},
+        "domestic_hygiene_pests_and_refuse": {"HHSRSDOMES": STANDARD_HHSRS_MAPPING},
+        "food_safety": {"HHSRSFOOD": STANDARD_HHSRS_MAPPING},
+        "personal_hygiene_sanitation_and_drainage": {"HHSRSPERS": STANDARD_HHSRS_MAPPING},
+        "water_supply": {"HHSRSWATER": STANDARD_HHSRS_MAPPING},
+        "falls_associated_with_baths": {"HHSRSFBATH": STANDARD_HHSRS_MAPPING},
+        "falls_on_level_surfaces": {"HHSRSFLEVE": STANDARD_HHSRS_MAPPING},
+        "falls_on_stairs_and_steps": {"HHSRSFSTAI": STANDARD_HHSRS_MAPPING},
+        "falls_between_levels": {"HHSRSFBETW": STANDARD_HHSRS_MAPPING},
+        "electrical_hazards": {"HHSRSELEC": STANDARD_HHSRS_MAPPING},
+        "fire": {"HHSRSFIRE": STANDARD_HHSRS_MAPPING},
+        "flames_hot_surfaces_and_materials": {"HHSRSFLAME": STANDARD_HHSRS_MAPPING},
+        "collision_and_entrapment": {"HHSRSENTRP": STANDARD_HHSRS_MAPPING, "HHSRSCLOW": STANDARD_HHSRS_MAPPING},
+        "explosions": {"HHSRSEXPLO": STANDARD_HHSRS_MAPPING},
+        "ergonomics": {"HHSRSPOSI": STANDARD_HHSRS_MAPPING},
+        "structural_collapse_and_falling_elements": {"HHSRSSTRUC": STANDARD_HHSRS_MAPPING}
+    }
+
+    # print(houses_waltham_forest_data[
+    #           houses_waltham_forest_data["ELEMENT CODE"] == "INTBTHADEQ"
+    #           ][["ATTRIBUTE CODE", "ATTRIBUTE CODE DESCRIPTION"]].drop_duplicates())
+
+    # print(flats_waltham_forest_data[
+    #           flats_waltham_forest_data["ELEMENT CODE"] == "INTBTHADEQ"
+    #           ][["ATTRIBUTE CODE", "ATTRIBUTE CODE DESCRIPTION"]].drop_duplicates())
+
+
+    # Criterion B
+    B_COMPONENT_LABELS = {
+        # Key components
+        "wall_structure": [
+            "Wall Structure in External Area",
+        ],
+        "lintels": [
+            "Lintels in External Area",
+        ],
+        "brickwork_spalling": [
+            "Wall Spalling in External Area",
+        ],
+        "wall_finish": [
+            "Wall Finish 1 in External Area",
+            "Wall Finish 2 in External Area",
+            "External Decorations in External Area",
+            "Brickwork Pointing in External Area",
+        ],
+        "roof_structure": [
+            "Roof Structure 1 in External Area",
+            "Roof Structure 2 in External Area",
+            "Roof Structure 3 in External Area",
+            "Garage Roof in External Area",
+            "Garage and Store Roofs in External Area",
+            "Store Roof in External Area",
+            "Fascia / Soffit / Bargeboard in External Area",
+            "Gutters in External Area",
+            "Downpipes in External Area",
+            "Internal Downpipes in External Area"
+        ],
+        "roof_finish": [
+            "Roof Covering 1 in External Area",
+            "Roof Covering 2 in External Area",
+            "Roof Covering 3 in External Area",
+        ],
+        "chimneys": [
+            "Chimneys in External Area",
+        ],
+        "windows": [
+            "Windows in Property",
+            "Windows 1 in External Area",
+            "Windows 2 in External Area",
+            "Garage and Store Windows in External Area",
+            "Garage Windows in External Area",
+            "Store Windows in External Area",
+        ],
+        "external_doors": [
+            "Type and Location of Front Door in Property",
+            "Front Door Fire Rating in Property",
+            "Patio and French Doors 1 in External Area",
+            "Back and Side Doors 1 in External Area",
+            "Back and Side Doors 2 in External Area",
+            "Garage and Store Doors in External Area",
+            "Garage Door in External Area",
+            "Store Door in External Area",
+        ],
+        "central_heating_boiler": [
+            # "Heating Improvement Required in Property",
+            "Boiler Fuel in Property",
+            "Type of Water Heating in Property",
+        ],
+        "heating_other": [
+            # "Heating Distribution System in Property"
+            "Boiler Fuel in Property",
+            "Type of Water Heating in Property",
+        ],
+        "electrical_systems": [
+            "Electrics Required in Property",
+        ],
+        # Other components
+        "kitchen": [
+            "Adequacy of Kitchen and Type in Property",
+        ],
+        "bathroom": [
+            "Adequacy of Bathroom Location in Property",
+        ],
+        "central_heating_distribution_system": [
+            "Heating Distribution System in Property",
+        ],
+    }
+
+    KEY_COMPONENTS = {
+        "wall_structure", "lintels", "brickwork_spalling", "wall_finish",
+        "roof_structure", "roof_finish", "chimneys", "windows",
+        "external_doors", "central_heating_boiler", "heating_other",
+        "electrical_systems",
+    }
+    OTHER_COMPONENTS = {
+        "kitchen", "bathroom", "central_heating_distribution_system",
+    }
+
+    # Criterion C
+    COMPONENT_LIFESPANS = {
+        # Key components
+        "wall_structure": {
+            "house": 80, "flat_below_6_storeys": 80, "flat_above_6_storeys": 80
+        },
+        "lintels": {
+            "house": 60, "flat_below_6_storeys": 60, "flat_above_6_storeys": 60
+        },
+        "brickwork_spalling": {
+            "house": 30, "flat_below_6_storeys": 30, "flat_above_6_storeys": 30
+        },
+        "wall_finish": {
+            "house": 60, "flat_below_6_storeys": 60, "flat_above_6_storeys": 30
+        },
+        "roof_structure": {
+            "house": 50, "flat_below_6_storeys": 30, "flat_above_6_storeys": 30
+        },
+        "roof_finish": {
+            "house": 50, "flat_below_6_storeys": 30, "flat_above_6_storeys": 30
+        },
+        "chimneys": {
+            "house": 50, "flat_below_6_storeys": 50, "flat_above_6_storeys": None  # N/A
+        },
+        "windows": {
+            "house": 40, "flat_below_6_storeys": 30, "flat_above_6_storeys": 30
+        },
+        "external_doors": {
+            "house": 40, "flat_below_6_storeys": 30, "flat_above_6_storeys": 30
+        },
+        "central_heating_boiler": {
+            "house": 15, "flat_below_6_storeys": 15, "flat_above_6_storeys": 15
+        },
+        "heating_other": {
+            "house": 30, "flat_below_6_storeys": 30, "flat_above_6_storeys": 30
+        },
+        "electrical_systems": {
+            "house": 30, "flat_below_6_storeys": 30, "flat_above_6_storeys": 30
+        },
+
+        # Other components
+        "kitchen": {
+            "house": 30, "flat_below_6_storeys": 30, "flat_above_6_storeys": 30
+        },
+        "bathroom": {
+            "house": 40, "flat_below_6_storeys": 40, "flat_above_6_storeys": 40
+        },
+        "central_heating_distribution_system": {
+            "house": 40, "flat_below_6_storeys": 40, "flat_above_6_storeys": 40
+        },
+    }
+
+    # Database design
+    # creation_date, uprn, variable, result (pass/fail/nodata), hhsrs_score (optional, numeric), hhsrs_rank (A-J),
+    # install_date (for components which expire, e.g. kitchen), remaining_life (for components which expire, e.g. kitchen),
+
+    # TODO: Add the criterion
+    decent_homes_meta = []
+    # Use to capture criterion A, B, C and D. Should be:
+    # {"uprn": int, "creation_date": datetime, "criterion_a": bool, "criterion_b": bool, "criterion_c": bool,
+    #  "criterion_d": bool, "decent_homes": bool"}
+    property_decent_homes = []
+    with open(os.path.join(fn), "rb") as f:
+        data = json.load(f)
+
+    today = pd.Timestamp.today().normalize()
+
+    property_info = data["property_info"]
+    if property_info["PROP TYPE"] in ["HOU"]:
+        property_type = "house"
+    elif property_info["PROP TYPE"] == "FLA":
+        if property_info["FLAT LEVEL"] < 6:
+            property_type = "flat_below_6_storeys"
+        else:
+            property_type = "flat_above_6_storeys"
+    else:
+        raise NotImplementedError("Unknown property type")
+
+    # ---------------- Criterion A ----------------
+    # Critrion A: pass/fail
+    # If fail, why?
+    for hhsrs_variable, mapping in HHSRS_MAPPING.items():
+        element_code = list(mapping.keys())[0]
+        # Find the data in the JSON within data["elements"]
+        check_pass = []
+        for k, v in data["elements"].items():
+            if v["ELEMENT CODE"] == element_code:
+                # We check the attribute code
+                # Check if pass
+                if v["ATTRIBUTE CODE"] in mapping[element_code]["pass"]:
+                    result = "pass"
+                elif v["ATTRIBUTE CODE"] in mapping[element_code]["fail"]:
+                    result = "fail"
+                elif v["ATTRIBUTE CODE"] in mapping[element_code]["no_data"]:
+                    result = "no_data"
+                else:
+                    raise ValueError(f"Unknown attribute code: '{v[element_code]}")
+                check_pass.append(result)
+                append_result(
+                    decent_homes_meta,
+                    criteria="A",
+                    variable=hhsrs_variable,
+                    sub_variable=ELEMENT_CODE_TO_DESCRIPTION[element_code],
+                    result=result,
+                    install_date=None,
+                    expiry_date=None,
+                )
+
+        # We check if we have a pass, fail or no_data
+        # if all([x == "pass" for x in check_pass]):
+        #     hhsrs_result = "pass"
+        # elif any([x == "fail" for x in check_pass]):
+        #     hhsrs_result = "fail"
+        # elif any([x == "no_data" for x in check_pass]):
+        #     hhsrs_result = "no_data"
+        # else:
+        #     raise NotImplementedError("Mixed results not implemented")
+
+    # ---------------- Criterion B ----------------
+    # Check each of the components
+
+    # ---------------- Criterion B ----------------
+    property_boiler = get_element(data["elements"], "Boiler Fuel in Property")
+
+    for component, labels in B_COMPONENT_LABELS.items():
+        for label in labels:
+            label_data = get_element(data["elements"], label)
+
+            # Handle no-data or not-applicable
+            if label_data["ATTRIBUTE CODE"] in ["UNKNOWN", "NONE", "UNKNOWNG", "UNKNOWNS", "UNKNOWNMAT"] and pd.isnull(label_data["INSTALL DATE"]):
+                continue
+
+            # Special skip conditions for heating
+            no_boiler_condition = (
+                property_boiler["ATTRIBUTE CODE"] in ["NONENOCH"]
+                and component == "central_heating_boiler"
+            )
+            other_heating_condition = (
+                label_data["ATTRIBUTE CODE"] in ["NONENOCH"]
+                and component == "heating_other"
+            )
+            if no_boiler_condition or other_heating_condition:
+                # append_result(
+                #     decent_homes_meta,
+                #     criteria="B",
+                #     variable=component,
+                #     sub_variable=label,
+                #     result="pass",
+                #     install_date=None,
+                #     expiry_date=None,
+                # )
+                continue
+
+            # Normal case: evaluate install date + lifetime + remaining life
+            install_date = pd.to_datetime(label_data["INSTALL DATE"])
+            if pd.isnull(install_date):
+                append_result(
+                    decent_homes_meta,
+                    criteria="B",
+                    variable=component,
+                    sub_variable=label,
+                    result="no_data",
+                    install_date=str(install_date),
+                    expiry_date=None,
+                )
+                continue
+            component_lifetime = COMPONENT_LIFESPANS[component][property_type]
+            is_old = years_between(today.to_pydatetime(), install_date.to_pydatetime()) > component_lifetime
+
+            if pd.isnull(label_data["REMAINING LIFE"]):
+                append_result(
+                    decent_homes_meta,
+                    criteria="B",
+                    variable=component,
+                    sub_variable=label,
+                    result="no_data",
+                    install_date=str(install_date),
+                    expiry_date=None,
+                )
+                continue
+            
+            has_failed = label_data["REMAINING LIFE"] < 0
+
+            expiry_date = today.to_pydatetime() + pd.DateOffset(years=label_data["REMAINING LIFE"])
+
+            component_result = "fail" if is_old and has_failed else "pass"
+
+            # Push into decent_homes_meta
+            append_result(
+                decent_homes_meta,
+                criteria="B",
+                variable=component,
+                sub_variable=label,
+                result=component_result,
+                install_date=str(install_date),
+                expiry_date=str(expiry_date),
+            )
+
+    # ---------------- Criterion C ----------------
+
+    # Guard: property type string already set earlier
+    is_flat = (property_info["PROP TYPE"] == "FLA")
+
+    # 1) Kitchen age ≤ 20 years
+    kitchen = get_element(data["elements"], LABEL_KITCHEN)
+    if kitchen:
+        kit_install_raw = kitchen["INSTALL DATE"]
+        kit_install = pd.to_datetime(kit_install_raw)
+        kit_age_years = years_between(today.to_pydatetime(), kit_install.to_pydatetime())
+        kitchen_age_result = "pass" if kit_age_years <= CRITERION_C_AGE_LIMITS["kitchen_years_max"] else "fail"
+        # For transparency, store next renewal as install + 20 years (criterion C perspective)
+        kit_next_due = today.to_pydatetime() + pd.DateOffset(years=kitchen["REMAINING LIFE"])
+    else:
+        raise NotImplementedError("Kitchen data missing - pls check")
+    append_result(
+        decent_homes_meta,
+        criteria="C",
+        variable="kitchen_less_than_20_years_old",
+        sub_variable="kitchen_less_than_20_years_old",
+        result=kitchen_age_result,
+        install_date=str(kit_install),
+        expiry_date=str(kit_next_due)
+    )
+
+    # 2) Kitchen adequate space/layout
+    # Prefer explicit codes if you have them, fall back to text in ATTRIBUTE CODE DESCRIPTION
+    if kitchen:
+        kit_attr_desc = kitchen["ATTRIBUTE CODE"]
+        if kit_attr_desc == "STDKITADQ":
+            kitchen_adequacy_result = "pass"
+        else:
+            raise NotImplementedError("No other observed codes yet")
+    else:
+        raise NotImplementedError("Kitchen data missing - pls check")
+    append_result(
+        decent_homes_meta,
+        criteria="C",
+        variable="kitchen_adequate_space_and_layout",
+        sub_variable="kitchen_adequate_space_and_layout",
+        result=kitchen_adequacy_result,
+    )
+
+    # 3) Bathroom age ≤ 30 years
+    bath = get_element(data["elements"], LABEL_BATHROOM)
+    if bath:
+        bth_install_raw = bath["INSTALL DATE"]
+        bth_install = pd.to_datetime(bth_install_raw)
+        bth_age_years = years_between(today.to_pydatetime(), bth_install.to_pydatetime())
+        bathroom_age_result = "pass" if bth_age_years <= CRITERION_C_AGE_LIMITS["bathroom_years_max"] else "fail"
+        bth_next_due = today.to_pydatetime() + pd.DateOffset(years=bath["REMAINING LIFE"])
+    else:
+        raise NotImplementedError("Bathroom data missing - pls check")
+    append_result(
+        decent_homes_meta,
+        criteria="C",
+        variable="bathroom_less_than_30_years_old",
+        sub_variable="bathroom_less_than_30_years_old",
+        result=bathroom_age_result,
+        install_date=str(bth_install),
+        expiry_date=bth_next_due
+    )
+
+    # 4) Bathroom/WC appropriately located
+    if bath:
+        bth_attr_code = bath["ATTRIBUTE CODE"]
+        if bth_attr_code in {"STDBTHADQ", "ADPBTHADQ"}:
+            bathroom_location_result = "pass"
+        elif bth_attr_code in {"STDBTHINAD"}:
+            bathroom_location_result = "fail"
+        else:
+            raise NotImplementedError(f"No other observed codes yet {bth_attr_code}")
+    else:
+        raise NotImplementedError("Bathroom data missing - pls check")
+
+    append_result(
+        decent_homes_meta,
+        criteria="C",
+        variable="bathroom_wc_appropriately_located",
+        sub_variable="bathroom_wc_appropriately_located",
+        result=bathroom_location_result
+    )
+
+    # 5) Adequate external noise insulation
+    noise = get_element(data["elements"], LABEL_NOISE)
+    if noise:
+        noise_code = noise["ATTRIBUTE CODE"]
+        if noise_code in {"ADEQUATE"}:
+            noise_result = "pass"
+        else:
+            raise NotImplementedError("No other observed codes yet")
+    else:
+        raise NotImplementedError("Noise insulation data missing - pls check")
+    append_result(
+        decent_homes_meta,
+        criteria="C",
+        variable="adequate_external_noise_insulation",
+        sub_variable="adequate_external_noise_insulation",
+        result=noise_result
+    )
+
+    # 6) Adequate common entrance areas (flats only)
+    if is_flat:
+        common = get_element(data["elements"], LABEL_COMMON_CIRC)
+        if common:
+            circ_desc = common["ATTRIBUTE CODE DESCRIPTION"]
+            if circ_desc in {"Adequate Circulation Space in Common Area"}:
+                common_areas_result = "pass"
+            else:
+                raise NotImplementedError(f"New description on common area {circ_desc}")
+        else:
+            common_areas_result = "no_data"
+        append_result(
+            decent_homes_meta=decent_homes_meta,
+            criteria="C",
+            variable="adequate_common_entrance_areas", 
+            sub_variable="adequate_common_entrance_areas", 
+            result=common_areas_result,
+        )
+
+    # ---------------- Criterion D ----------------
+    # heating system type
+    heating = get_element(data["elements"], "Heating Improvement Required in Property")
+    if heating:
+        heat_type_code = heating["ATTRIBUTE CODE"]
+        if heat_type_code in {"NOTAPPLIC"}:
+            heating_type_result = "pass"
+        elif heat_type_code in {"WETINSFULL"}:
+            heating_type_result = "fail"
+        else:
+            raise NotImplementedError("No other observed codes yet")
+    else:
+        raise NotImplementedError("Heating element missing in dataset")
+
+    append_result(
+        decent_homes_meta,
+        criteria="D",
+        variable="efficient_heating_system_type",
+        sub_variable="efficient_heating_system_type",
+        result=heating_type_result
+    )
+
+    # heating distribution
+    heating_dist = get_element(data["elements"], "Heating Distribution System in Property")
+    if heating_dist:
+        dist_code = heating_dist["ATTRIBUTE CODE"]
+        if dist_code == "UNKNOWN":
+            # For the observed case, there was no heating and wet heating needed to be installed in full so the value
+            # was unknown
+            heating_dist_result = "no_data"
+        elif dist_code in {"RADIATORS", "ELECWARMAR"}:
+            # Found one with heating distribution - check with Khalim if this is pass
+            heating_dist_result = "pass"
+        else:
+            print(f"heating_dist {heating_dist}")
+            print(f"dist-code {dist_code}")
+            raise NotImplementedError("No other observed codes yet")
+    else:
+        raise NotImplementedError("Heating distribution element missing in dataset")
+
+    append_result(
+        decent_homes_meta,
+        criteria="D",
+        variable="efficient_heating_distribution",
+        sub_variable="efficient_heating_distribution",
+        result=heating_dist_result
+    )
+
+    # insulation
+    loft = get_element(data["elements"], "Size in mm of Loft Insulation Thickness in Property")
+    wall = get_element(data["elements"], "Wall Insulation Improvement in External Area")
+    # To determine how much loft insulation is required
+
+    # Loft insulation check (example threshold: ≥ 270mm = pass)
+    if loft:
+        # We have a specific code, where further loft insulation is needed - It appears the heating type check has
+        # already been completed in this dataset and so we just need to check the code
+        loft_code = loft["ATTRIBUTE CODE"]
+        if loft_code == "LOFTINSRQD":
+            loft_result = "fail"
+        elif loft_code.isnumeric():
+            loft_result = "pass"
+        elif loft_code == "UNKNOWN":
+            loft_result = None
+        else:
+            raise NotImplementedError(f"Unknown loft insulation code - pls check {loft_code}")
+    else:
+        raise NotImplementedError("Loft insulation data missing - pls check")
+    
+    if loft_result:
+        append_result(
+            decent_homes_meta,
+            criteria="D",
+            variable="loft_insulation_sufficient",
+            sub_variable="loft_insulation_sufficient",
+            result=loft_result
+        )
+
+    # Wall insulation check
+    if wall:
+        wall_code = wall["ATTRIBUTE CODE"]
+        if wall_code in {"NONE"}:  # Means no insulation improvement required
+            wall_result = "pass"
+        elif wall_code in {"UNKNOWN"}:
+            wall_result = "no_data"
+        elif wall_code in {"SOLID"}:
+            wall_result = "fail"
+        else:
+            raise NotImplementedError(f"No other observed codes yet {wall_code}")
+    else:
+        raise NotImplementedError("Wall insulation data missing - pls check")
+    append_result(
+        decent_homes_meta,
+        criteria="D",
+        variable="wall_insulation_sufficient",
+        sub_variable="wall_insulation_sufficient",
+        result=wall_result
+    )
+
+    # ---------------- Criterion A overall ----------------
+    a_vars = set(HHSRS_MAPPING.keys())
+    latest_a_results = {r["variable"]: r["result"] for r in decent_homes_meta if r["variable"] in a_vars}
+
+    if any(v == "fail" for v in latest_a_results.values()):
+        criterion_a_result = "fail"
+    elif all(v == "pass" for v in latest_a_results.values()):
+        criterion_a_result = "pass"
+    else:
+        criterion_a_result = "no_data"
+
+    # ---------------- Criterion B overall ----------------
+
+    component_results = {}
+
+    for component in B_COMPONENT_LABELS.keys():
+        comp_rows = [r for r in decent_homes_meta if
+                    r["criteria"] == "B" and r["variable"] == component and r["sub_variable"] is not None]
+        comp_sub_results = [r["result"] for r in comp_rows]
+
+        if not comp_sub_results:  # no rows at all
+            comp_result = "no_data"
+        elif any(r == "fail" for r in comp_sub_results):
+            comp_result = "fail"
+        elif all(r == "pass" for r in comp_sub_results):
+            comp_result = "pass"
+        else:
+            comp_result = "no_data"
+
+        component_results[component] = comp_result
+
+    key_fails = [c for c, r in component_results.items() if c in KEY_COMPONENTS and r == "fail"]
+    other_fails = [c for c, r in component_results.items() if c in OTHER_COMPONENTS and r == "fail"]
+
+    if key_fails:
+        criterion_b_result = "fail"
+    elif len(other_fails) >= 2:
+        criterion_b_result = "fail"
+    elif any(r == "no_data" for r in component_results.values()):
+        criterion_b_result = "no_data"
+    else:
+        criterion_b_result = "pass"
+
+    # ---------------- Criterion C overall ----------------
+    criterion_c_vars = [
+        "kitchen_less_than_20_years_old",
+        "kitchen_adequate_space_and_layout",
+        "bathroom_less_than_30_years_old",
+        "bathroom_wc_appropriately_located",
+        "adequate_external_noise_insulation",
+    ]
+    if is_flat:
+        criterion_c_vars.append("adequate_common_entrance_areas")
+
+    latest_c_results = {r["variable"]: r["result"] for r in decent_homes_meta if r["variable"] in criterion_c_vars}
+
+    count_fails = sum(1 for v in latest_c_results.values() if v == "fail")
+    # optionally count no_data too if you want strict interpretation
+    criterion_c_result = "fail" if count_fails >= 3 else "pass"
+
+    # ---------------- Criterion D overall ----------------
+    # Needs to have both efficient geating and distribution so all should pass
+    criterion_d_vars = [
+        "efficient_heating_system_type",
+        "efficient_heating_distribution",
+        "loft_insulation_sufficient",
+        "wall_insulation_sufficient",
+    ]
+    latest_d_results = {r["variable"]: r["result"] for r in decent_homes_meta if r["variable"] in criterion_d_vars}
+
+    if any(v == "fail" for v in latest_d_results.values()):
+        criterion_d_result = "fail"
+    elif all(v == "pass" for v in latest_d_results.values()):
+        criterion_d_result = "pass"
+    else:
+        criterion_d_result = "no_data"
+
+    # ---------------- Append to property_decent_homes ----------------
+    check_pass = [
+        criterion_a_result,
+        criterion_b_result,
+        criterion_c_result,
+        criterion_d_result
+    ]
+    decent_homes_result = "no_data"
+
+    if all(v == "pass" for v in check_pass):
+        decent_homes_result = "pass"
+    elif any(v == "fail" for v in check_pass):
+        decent_homes_result = "fail"
+    elif any(v=="no_data" for v in check_pass):
+        decent_homes_result = "no_data"
+    
+
+    property_decent_homes.append({
+        "uprn": data.get("UPRN"),  # TODO: Need UPRN
+        "creation_date": datetime.now().date().isoformat(),
+        "criterion_a": criterion_a_result,
+        "criterion_b": criterion_b_result,
+        "criterion_c": criterion_c_result,
+        "criterion_d": criterion_d_result,
+        "decent_homes": decent_homes_result,
+    })
+
+    return property_decent_homes[0], decent_homes_meta,
+

deployment/lambda/walthamforest_etl/docker/ecr.tf

@@ -0,0 +1,63 @@
+# ECR repo
+resource "aws_ecr_repository" "walthamforest_etl_adhoc_ecr" {
+  name = "walthamforest_etl_adhoc_ecr"
+}
+
+# ECR policy to allow Lambda access
+resource "aws_ecr_repository_policy" "walthamforest_etl_adhoc_ecr_access" {
+  repository = aws_ecr_repository.walthamforest_etl_adhoc_ecr.name
+
+  policy = jsonencode({
+    Version = "2008-10-17",
+    Statement = [{
+      Sid       = "AllowLambdaPull",
+      Effect    = "Allow",
+      Principal = {
+        Service = "lambda.amazonaws.com"
+      },
+      Action = [
+        "ecr:GetDownloadUrlForLayer",
+        "ecr:BatchGetImage",
+        "ecr:BatchCheckLayerAvailability"
+      ]
+    }]
+  })
+}
+
+
+
+# ECR lifecycle policy to delete tagged images older than 14 days
+resource "aws_ecr_lifecycle_policy" "walthamforest_etl_adhoc_loader_lifecycle" {
+  repository = aws_ecr_repository.walthamforest_etl_adhoc_ecr.name
+
+  policy = jsonencode({
+    "rules": [
+        {
+            "rulePriority": 2,
+            "description": "Expire images older than 14 days",
+            "selection": {
+                "tagStatus": "untagged",
+                "countType": "sinceImagePushed",
+                "countUnit": "days",
+                "countNumber": 1
+            },
+            "action": {
+                "type": "expire"
+            }
+        },
+        {
+            "rulePriority": 1,
+            "description": "Keep last 5 images",
+            "selection": {
+                "tagStatus": "tagged",
+                "tagPrefixList": ["feature"],
+                "countType": "imageCountMoreThan",
+                "countNumber": 5
+            },
+            "action": {
+                "type": "expire"
+            }
+        }
+    ]
+  })
+}

deployment/lambda/walthamforest_etl/docker/provider.tf

@@ -0,0 +1,15 @@
+terraform {
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = "~> 6.3.0"
+    }
+  }
+  backend "s3" {
+    bucket  = "survey-extractor-tf-state"
+    region  = "eu-west-2"
+    key     = "env:/dev/lambda/ecr/walthamforest_etl.tfstate"
+  }
+
+  required_version = ">= 1.2.0"
+}

deployment/lambda/walthamforest_etl/provider.tf

@@ -0,0 +1,15 @@
+terraform {
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = "~> 6.3.0"
+    }
+  }
+  backend "s3" {
+    bucket  = "survey-extractor-tf-state"
+    region  = "eu-west-2"
+    key     = "env:/dev/lambda/eachlambda/walthamforest_etl_lambda.tfstate"
+  }
+
+  required_version = ">= 1.2.0"
+}

deployment/lambda/walthamforest_etl/vars.tf

@@ -0,0 +1,5 @@
+variable "lambda_image_tag" {
+  description = "Docker image tag (e.g. GitHub SHA)"
+  type        = string
+  default     = "local-dev-latest"
+}

deployment/lambda/walthamforest_etl/walthamforest_etl_lambda.tf

@@ -0,0 +1,83 @@
+# Reference existing IAM role
+data "aws_iam_role" "lambda_exec_role" {
+  name = "lambda-exec-role"
+}
+
+# Reference existing ECR repository
+data "aws_ecr_repository" "walthamforest_etl_adhoc_ecr" {
+  name = "walthamforest_etl_adhoc_ecr"
+}
+
+# SQS queue
+resource "aws_sqs_queue" "walthamforest_etl_adhoc_queue" {
+  name = "walthamforest_etl_adhoc-queue"
+  visibility_timeout_seconds  = 1800  # 30 minutes (>= 300s and ~6x Lambda timeout)
+}
+
+
+# Custom IAM policy specific to lambda_example
+resource "aws_iam_policy" "walthamforest_etl_adhoc_policy" {
+  name = "walthamforest_adhoc_policy_lambda"
+
+  policy = jsonencode({
+    Version = "2012-10-17",
+    Statement = [
+      {
+        Effect = "Allow",
+        Action = [
+          "sqs:ReceiveMessage",
+          "sqs:DeleteMessage",
+          "sqs:GetQueueAttributes",
+          "sqs:GetQueueUrl",
+          "sqs:ChangeMessageVisibility"
+        ],
+        Resource = aws_sqs_queue.walthamforest_etl_adhoc_queue.arn
+      },
+      {
+        Effect = "Allow",
+        Action = [
+          "ecr:GetDownloadUrlForLayer",
+          "ecr:BatchGetImage",
+          "ecr:BatchCheckLayerAvailability"
+        ],
+        Resource = data.aws_ecr_repository.walthamforest_etl_adhoc_ecr.arn
+      },
+      {
+        Effect = "Allow",
+        Action = ["ecr:GetAuthorizationToken"],
+        Resource = "*"
+      }
+    ]
+  })
+}
+
+resource "aws_iam_role_policy_attachment" "walthamforest_etl_adhoc_policy_attach" {
+  role       = data.aws_iam_role.lambda_exec_role.name
+  policy_arn = aws_iam_policy.walthamforest_etl_adhoc_policy.arn
+}
+
+# Lambda function
+resource "aws_lambda_function" "walthamforest_etl_adhoc" {
+  function_name = "walthamforest_etl_adhoc"
+  role          = data.aws_iam_role.lambda_exec_role.arn
+  package_type  = "Image"
+  image_uri     = "${data.aws_ecr_repository.walthamforest_etl_adhoc_ecr.repository_url}:${var.lambda_image_tag}"
+  # Increase timeout (max 900 sec / 15 min)
+  # timeout       = 300   # e.g. 5 minutes
+
+  # Increase memory (default 128 MB)
+  memory_size   = 2048  # try 1024 or 2048 MB to start
+
+  # environment {
+  #   variables = {
+  #     DATABASE_URL = "postgresql://postgres:makingwarmhomes@terraform-20250331175522503500000002.cdgzupxvdyp0.eu-west-2.rds.amazonaws.com:5432/surveyDB"
+  #   }
+  # }
+}
+
+# SQS trigger
+resource "aws_lambda_event_source_mapping" "walthamforest_etl_adhoc_trigger" {
+  event_source_arn = aws_sqs_queue.walthamforest_etl_adhoc_queue.arn
+  function_name    = aws_lambda_function.walthamforest_etl_adhoc.arn
+  batch_size       = 1
+}

devcontainer.sh

@@ -0,0 +1,102 @@
+#!/usr/bin/env bash
+#
+# devcontainer.sh — devcontainer helper for this repo
+#
+# Usage:
+#   ./devcontainer.sh <command>
+#
+# Commands:
+#   up        build + start the devcontainer (idempotent)
+#   shell     attach a bash shell; auto-ups if not running
+#   down      stop the devcontainer
+#   rebuild   remove + rebuild from scratch, no cache
+#
+# Examples:
+#   ./devcontainer.sh shell     # one-shot: up if needed, then bash
+#   ./devcontainer.sh rebuild
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)"
+REPO_ROOT="${SCRIPT_DIR}"
+CONFIG_PATH="${REPO_ROOT}/.devcontainer/devcontainer.json"
+
+VALID_COMMANDS=(up shell down rebuild)
+
+# --- helpers ---------------------------------------------------------------
+
+usage() {
+  sed -n '3,15p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//'
+  exit "${1:-0}"
+}
+
+die() {
+  echo "error: $*" >&2
+  exit 1
+}
+
+in_list() {
+  local needle="$1"
+  shift
+  local item
+  for item in "$@"; do
+    [[ "${item}" == "${needle}" ]] && return 0
+  done
+  return 1
+}
+
+container_id() {
+  # Find the running container for this repo via devcontainer labels.
+  docker ps -q \
+    --filter "label=devcontainer.local_folder=${REPO_ROOT}" \
+    --filter "label=devcontainer.config_file=${CONFIG_PATH}"
+}
+
+# --- argument parsing ------------------------------------------------------
+
+[[ $# -eq 1 ]] || usage 1
+
+COMMAND="$1"
+
+in_list "${COMMAND}" "${VALID_COMMANDS[@]}" \
+  || die "invalid command '${COMMAND}' (expected: ${VALID_COMMANDS[*]})"
+
+[[ -f "${CONFIG_PATH}" ]] || die "config not found: ${CONFIG_PATH}"
+
+DC_ARGS=(--workspace-folder "${REPO_ROOT}")
+
+# --- dispatch --------------------------------------------------------------
+
+case "${COMMAND}" in
+  up)
+    echo ">> bringing up devcontainer"
+    devcontainer up "${DC_ARGS[@]}"
+    ;;
+
+  shell)
+    # Auto-up if not already running. `devcontainer up` is idempotent —
+    # it reuses an existing container, so this is cheap on warm starts.
+    if [[ -z "$(container_id)" ]]; then
+      echo ">> devcontainer not running, bringing it up first"
+      devcontainer up "${DC_ARGS[@]}"
+    fi
+    echo ">> attaching shell"
+    devcontainer exec "${DC_ARGS[@]}" bash 2>/dev/null \
+      || devcontainer exec "${DC_ARGS[@]}" sh
+    ;;
+
+  down)
+    cid="$(container_id)"
+    if [[ -z "${cid}" ]]; then
+      echo ">> devcontainer not running, nothing to stop"
+      exit 0
+    fi
+    echo ">> stopping devcontainer"
+    docker stop "${cid}"
+    ;;
+
+  rebuild)
+    echo ">> rebuilding devcontainer from scratch"
+    devcontainer up "${DC_ARGS[@]}" --remove-existing-container --build-no-cache
+    ;;
+esac

etl/MonthEndUploader.py

@@ -0,0 +1,50 @@
+import os
+os.environ["SHAREPOINT_CLIENT_ID"] = "0e28c4f9-3e77-4571-8d63-df1857f4266a"
+os.environ["SHAREPOINT_CLIENT_SECRET"] = "2s48Q~t8.pI-~rbtQaSCFcYY97Z3LiMYhuo0GaOb"
+os.environ["SHAREPOINT_TENANT_ID"] = "6f080c63-8a66-4bbc-9d72-b85d5df30555"
+
+from etl.scraper.scraper import SharePointScraper, SharePointInstaller
+from datetime import datetime
+
+
+def upload_to_month_end_folder(file_name_on_sp, local_file_path, add_to_path):
+    sharepoint = SharePointScraper(SharePointInstaller.OSMOSIS_ACD)
+    sharepoint.get_folders_in_path("/")
+
+    parent_folder = "General/Junte Kim/month end"
+    today = datetime.today()
+    # Format as "Month YYYY"
+    formatted_date = today.strftime("%B %Y")
+    sharepoint.create_dir(formatted_date, parent_folder)
+    sharepoint_path = parent_folder + "/" + formatted_date
+
+
+    # Make day month year folder
+    formatted_date = today.strftime("%d-%m-%y")
+    sharepoint.create_dir(formatted_date, sharepoint_path)
+    sharepoint_path += "/" + formatted_date
+
+    # Make company folder
+    sharepoint.create_dir(add_to_path, sharepoint_path)
+    sharepoint_path += "/" + add_to_path 
+
+    print("Uploading to sharepoint...")
+    sharepoint.upload_file(local_file_path, sharepoint_path, file_name_on_sp)
+    print(f"Finished upload of {local_file_path} to sharepoint. It's found under {sharepoint_path}/{file_name_on_sp}")
+
+
+def upload_to_nick_folder(file_name_on_sp, local_file_path, add_to_path=None):
+    sharepoint = SharePointScraper(SharePointInstaller.OSMOSIS_ACD)
+    parent_folder = "General/Junte Kim/For Nick"
+    today = datetime.today()
+    formatted_date = today.strftime("%Y-%m-%d%H-%M-S")
+
+    sharepoint.create_dir(formatted_date, parent_folder)
+    sharepoint_path = parent_folder + "/" + formatted_date
+    if add_to_path:
+        sharepoint.create_dir(add_to_path, sharepoint_path)
+        sharepoint_path += "/" + add_to_path 
+
+    print("Uploading to sharepoint...")
+    sharepoint.upload_file(local_file_path, sharepoint_path, file_name_on_sp)
+    print(f"Finished upload of {local_file_path} to sharepoint. It's found under {sharepoint_path}/{file_name_on_sp}")

etl/condition_report_etl.py

@@ -0,0 +1,14 @@
+from pprint import pprint
+from etl.db.db import get_db_session, init_db
+
+from etl.surveyedData.surveryedData import surveyedDataProcessor
+condition_report_file_path = "/workspaces/survey-extractor/etl/files/osmosis_condition_report.pdf"
+sdp = surveyedDataProcessor("123 Fake Street", [condition_report_file_path])
+pprint(sdp.condition_report.master_obj)
+
+init_db()
+with get_db_session() as db_session:
+    sdp.load_condition_report(db_session)
+
+# TODO: add the ability to add document type, and sharepoint or s3 link so we can process access it again
+# Terraform lambda set up to start this job from a s3 link

etl/daily_script.py

@@ -1,5 +1,5 @@
 import os
-from pdfReader.pdfReaderToText import pdfReaderToText
+from fileReader.pdfReaderToText import pdfReaderToText
 from etl.scraper.scraper import SharePointScraper, SharePointInstaller, WEEK_COMMENCING
 from pprint import pprint, pformat
 import logging

etl/db/db.py

@@ -1,6 +1,7 @@
 from sqlmodel import SQLModel, create_engine, Session
 from pydantic_settings import BaseSettings
 from typing import Optional, List
+from sqlalchemy.pool import QueuePool
 
 class Settings(BaseSettings):
     DATABASE_URL: Optional[str] = None  # Default to None if not set
@@ -9,8 +10,20 @@ class Settings(BaseSettings):
         env_file = ".env"  # Load from an optional .env file
 
 settings = Settings()
-engine = create_engine(settings.DATABASE_URL) if settings.DATABASE_URL else None
-
+# engine to the dabatase, currently set up to connect via settings. database
+engine = (
+    create_engine(
+        settings.DATABASE_URL,
+        poolclass=QueuePool,          # use standard connection pool
+        pool_pre_ping=True,           # test connection before use
+        pool_recycle=300,             # reconnect every 5 minutes
+        pool_size=5,                  # limit pool size for CI/CD or serverless
+        max_overflow=2,               # allow brief overuse
+        connect_args={"sslmode": "require"},  # enforce SSL for cloud DBs
+    )
+    if settings.DATABASE_URL
+    else None
+)
 
 def get_db_session():
     if engine is None:
@@ -19,4 +32,5 @@ def get_db_session():
 
 def init_db():
     if engine:
+        # Links SQLModel and metadata defined in sqlmodel instance
         SQLModel.metadata.create_all(engine)

etl/db/hubSpotLoad.py

@@ -1,136 +1,305 @@
-from etl.hubSpotClient.hubspot import HubSpotClient, DealStage
-from etl.surveyPrice.surveyPrice import SurveyPrice
-from etl.surveyedData.surveryedData import surveyedDataProcessor
-from etl.scraper.scraper import SharePointScraper, SharePointInstaller
 from etl.db.db import get_db_session, init_db
-import pandas as pd
-from etl.db.db import get_db_session, init_db
-from etl.utils.utils import get_sharepoint_path
+from etl.models.topLevel import HubspotDealData, HubspotCommpanyData
+from sqlmodel import select
+from etl.s3.s3_uploader import S3Uploader
+import hashlib
+import os
 
-class HubspotTodb():
+
+class HubspotTodb:
     def __init__(self):
         init_db()
-        self.hubspot = HubSpotClient()
-        self.deals_in_hubspot = None
-        self.data_in_sharepoint = []
-        self.sp = SurveyPrice()
+        self.s3 = S3Uploader()
 
-    def get_all_deals(self):
-        self.deals_in_hubspot = self.sp.get_all_surveys_from_hubspot()
-        return self.deals_in_hubspot
-    
-            
-    def get_sharepoint_scraper(self, installer):
-        sp = None
-        if installer.upper() == "J & J CRUMP":
-            sp = SharePointScraper(SharePointInstaller.JJC)
-        elif installer.upper() == "SCIS":
-            sp = SharePointScraper(SharePointInstaller.SOUTH_COAST_INSULATION)
-        elif installer.upper() == "SGEC":
-            sp = SharePointScraper(SharePointInstaller.SGEC)
-        else:
-            sp = None
-
-        return sp
-    
-    def create_files_locally(self, sp, path, address):
-        address_paths = {}
-        file_names_to_download = {}
-        avoid = [".jpg",".mov", ".JPG", ".heic", ".HEIC", ".png", ".PNG", ".jpeg", ".JPEG", ".mov", ".MOV", ".mp4", ".MP4"]
-
-
-        microsoft_graph_data = sp.get_folders_in_path(path)
-        for file in microsoft_graph_data['value']:
-            if 'file' in file:
-                if any(file["name"].endswith(ext) for ext in avoid):
-                    continue
-                file_names_to_download.update({file["name"]: file['@microsoft.graph.downloadUrl']})
-        
-        each_file = []
-        for file_name, url in file_names_to_download.items():
-            content = sp.get_file_content(url)
-            file_path = sp.create_temp_file(content, f"{address}/{file_name}")
-            each_file.append(file_path)
-        address_paths.update({address: each_file})
-        return address_paths
-    
-    def string_to_installer(self, installer):
-        if installer.upper() == "J & J CRUMP":
-            return SharePointInstaller.JJC
-        elif installer.upper() == "SCIS":
-            return SharePointInstaller.SOUTH_COAST_INSULATION
-        elif installer.upper() == "SGEC":
-            return SharePointInstaller.SGEC
-        else:
-            return None
-
-    def work_out_invoice(self, row):
-        survey = self.gather_data_from_sharepoint_url(row)
-        installer = self.string_to_installer(row["HUBSPOT_INSTALLER"])
-        survey_pd = pd.DataFrame([self.sp.survey_to_pandas_format(surveyInfo=survey, installer=installer)])
-        hubspot_data = pd.DataFrame([row])
-        merged_df = self.sp.merge_hub_spot_and_survey_information_from_sharepoint_url(hubspot_data, survey_pd)
-        return self.sp.calculate_all_price(merged_df)
-
-
-
-        # self.sp.calculate_one_price_with_sharepoint_url(row, )
-
-    def gather_data_from_sharepoint_url(self, row):
-        sp = self.get_sharepoint_scraper(row["HUBSPOT_INSTALLER"])
-        path = get_sharepoint_path(row["HUBSPOT_SHAREPOINT_PATH"])
-        data_loc = self.create_files_locally(sp, path, row["HUBSPOT_DEAL_ADDRESS"])
-
-        for add, file_loc in data_loc.items():
-            sdp = surveyedDataProcessor(add, file_loc)
-            sdp.hubspot_deal_id = row["HUBSPOT_DEAL_ID"]
-            with  get_db_session() as session:
-                self.load_one_pre_site_note(session, sdp, row)
-            return sdp
-
-
-    def gather_data_from_each_sharepoint(self):
-        self.get_all_deals()
-        for _, row in self.deals_in_hubspot.iterrows():
-            sp = self.get_sharepoint_scraper(row["HUBSPOT_INSTALLER"])
-            path = self.get_sharepoint_path(row["HUBSPOT_SHAREPOINT_PATH"])
-            data_loc = self.create_files_locally(sp, path, row["HUBSPOT_DEAL_ADDRESS"])
-
-            for add, file_loc in data_loc.items():
-                sdp = surveyedDataProcessor(add, file_loc)
-                sdp.hubspot_deal_id = row["HUBSPOT_DEAL_ID"]
-                self.data_in_sharepoint.append(sdp)
-
-    
-    def load_all(self, fast=False):
-        if fast is False:
-            self.gather_data_from_each_sharepoint()
+    def new_record_company(self, company_data):
+        """Adds a new record to the hubspot_company_data table."""
         with get_db_session() as session:
-            self.load_all_pre_site_note(session)
+            new_record = HubspotCommpanyData(
+                company_id=company_data.get("hs_object_id"),
+                company_name=company_data.get("name"),
+            )
+            session.add(new_record)
             session.commit()
+            session.refresh(new_record)
+            return new_record
 
-    def load_one_pre_site_note(self, db_session, surveyedData, hubspot_data):
-        df = hubspot_data
-        assessor = surveyedData.load_assessor_table(db_session)
+    def new_record_to_hubspot_data(self, deal_data, company, listing, hubspot_client):
+        print("⚠️ Deprecated — use the new interface instead.")
+        return self.upsert_hubspot_deal(deal_data, company, listing, hubspot_client)
 
-        # Loads the pre site summary information
-        summary_info = surveyedData.load_pre_site_notes_summary_table(db_session)
+    def find_all_deals_with_company_id(self, company_id):
+        """Returns a list of deals for a given company_id."""
+        with get_db_session() as session:
+            return (
+                session.query(HubspotDealData)
+                .filter(HubspotDealData.company_id == company_id)
+                .all()
+            )
 
-        property_description = surveyedData.load_property_description(db_session)
+    def find_deal_with_deal_id(self, deal_id):
+        with get_db_session() as session:
+            return (
+                session.query(HubspotDealData)
+                .filter(HubspotDealData.deal_id == deal_id)
+                .one()
+            )
 
-        # Creates the a final pre site note table that links all information
-        presitenote = surveyedData.create_pre_site_note_table(db_session, assessor, summary_info, property_description)
+    def _sha256(self, file_path: str) -> str:
+        """Compute SHA-256 checksum of a file."""
+        sha256 = hashlib.sha256()
+        with open(file_path, "rb") as f:
+            for chunk in iter(lambda: f.read(8192), b""):
+                sha256.update(chunk)
+        return sha256.hexdigest()
 
-        building_table = surveyedData.create_buildings_table(
-            db_session,
-            df["HUBSPOT_LANDLORD_ID"],
-            df["HUBSPOT_DOMNA_ID"],
+    def update_deal(self, deal_in_db, hubspot_client):
+        """
+        Checks if a deal needs updating and syncs it with HubSpot.
+        Also handles major_condition_issue_photos file upload to S3 with integrity check.
+        """
+
+        def soft_assert(condition, message="Assertion Failed"):
+            if not condition:
+                print(f"⚠️ Soft Assert Failed: {message}")
+                return False
+            return True
+
+        print(f"🔍 Checking if deal needs updating (deal_id={deal_in_db.deal_id})")
+
+        hs_deal, hs_company_id, hs_listing = hubspot_client.get_deal_info_for_db(
+            deal_in_db.deal_id
         )
-        documents = surveyedData.create_document_table_via_pre_site_note(db_session, presitenote, assessor, building_table)
 
-    def load_all_pre_site_note(self, db_session):
-        # Loads all pre
-        for surveyedData in self.data_in_sharepoint:
-            self.load_one_pre_site_note(surveyedData=surveyedData, db_session=db_session)
+        # Soft compare key fields
+        checks = [
+            soft_assert(
+                deal_in_db.deal_id == hs_deal.get("hs_object_id"), "deal_id mismatch"
+            ),
+            soft_assert(deal_in_db.company_id == hs_company_id, "company_id mismatch"),
+            soft_assert(
+                deal_in_db.landlord_property_id == hs_listing.get("owner_property_id"),
+                "landlord_property_id mismatch",
+            ),
+            soft_assert(
+                deal_in_db.outcome == hs_deal.get("outcome"), "outcome mismatch"
+            ),
+            soft_assert(
+                deal_in_db.dealstage == hs_deal.get("dealstage"), "dealstage mismatch"
+            ),
+            soft_assert(
+                deal_in_db.dealname == hs_deal.get("dealname"), "dealname mismatch"
+            ),
+            soft_assert(
+                deal_in_db.project_code == hs_deal.get("project_code"),
+                "project_code mismatch",
+            ),
+            soft_assert(
+                deal_in_db.uprn == hs_listing.get("national_uprn"), "uprn mismatch"
+            ),
+            soft_assert(
+                deal_in_db.outcome_notes == hs_deal.get("outcome_notes"),
+                "outcome_notes mismatch",
+            ),
+            soft_assert(
+                deal_in_db.major_condition_issue_description
+                == hs_deal.get("major_condition_issue_description"),
+                "major condition description mismatch",
+            ),
+            soft_assert(
+                deal_in_db.major_condition_issue_photos
+                == hs_deal.get("major_condition_issue_photos"),
+                "major condition issue photos mismatch",
+            ),
+            soft_assert(
+                deal_in_db.coordination_status
+                == hs_deal.get("coordination_status__stage_1_"),
+                "coordination stage 1 status mismatch",
+            ),
+            soft_assert(
+                deal_in_db.design_status == hs_deal.get("retrofit_design_status"),
+                "retrofit design mismatch",
+            ),
+        ]
 
+        # If discrepancies found, update from HubSpot
+        if not all(checks):
+            print(
+                f"❗ Discrepancies found for deal_id {deal_in_db.deal_id} — syncing with HubSpot."
+            )
+            self.upsert_hubspot_deal(hs_deal, hs_company_id, hs_listing, hubspot_client)
+            return False
 
+        # Handle photo upload if it exists but S3 URL is missing
+        if (
+            deal_in_db.major_condition_issue_photos
+            and not deal_in_db.major_condition_issue_evidence_s3_url
+        ):
+            print(
+                f"🖼️ Found photo for deal_id {deal_in_db.deal_id} — uploading to S3..."
+            )
+
+            photo_url = hs_deal.get("major_condition_issue_photos")
+            if photo_url:
+                try:
+                    # Download from HubSpot using fresh URL from hs_deal (not stale DB URL)
+                    local_file = hubspot_client.download_file_from_url(photo_url)
+
+                    # Upload to S3
+                    bucket = "retrofit-data-dev"
+                    s3_url = self.s3.upload_file(
+                        local_file, bucket, prefix="hubspot/awaabs_law_evidence/"
+                    )
+
+                    # Download again to verify integrity
+                    downloaded = self.s3.download_from_url(s3_url)
+                    if self._sha256(local_file) == self._sha256(downloaded):
+                        print("✅ SHA256 match verified — upload successful.")
+                    else:
+                        print("❌ SHA256 mismatch — integrity check failed.")
+                        raise ValueError("File integrity check failed after S3 upload.")
+
+                    # Update DB record with S3 URL
+                    with get_db_session() as session:
+                        db_record = session.get(HubspotDealData, deal_in_db.id)
+                        db_record.major_condition_issue_evidence_s3_url = s3_url
+                        session.add(db_record)
+                        session.commit()
+                        print(
+                            f"✅ Updated DB with S3 URL for deal_id={deal_in_db.deal_id}"
+                        )
+                    return False
+                except Exception as e:
+                    print(
+                        f"⚠️ Failed to download/upload photo for deal_id {deal_in_db.deal_id}: {e}"
+                    )
+                    # Continue without the file — don't crash the entire update
+            else:
+                print(f"⚠️ Photo URL missing for deal_id {deal_in_db.deal_id}")
+
+        else:
+            print(f"✅ No update or upload required for deal_id {deal_in_db.deal_id}.")
+
+        return True
+
+    def upsert_hubspot_deal(self, deal_data, company, listing, hubspot_client):
+        """
+        Inserts or updates a deal record.
+        Also uploads photos if present and adds S3 URL.
+        """
+        with get_db_session() as session:
+            print(deal_data)
+            deal_id = deal_data.get("hs_object_id")
+
+            statement = select(HubspotDealData).where(
+                HubspotDealData.deal_id == deal_id
+            )
+            existing = session.exec(statement).first()
+
+            if existing:
+                print(f"🔄 Updating existing deal (deal_id={deal_id})")
+
+                for attr, value in {
+                    "dealname": deal_data.get("dealname"),
+                    "dealstage": deal_data.get("dealstage"),
+                    "landlord_property_id": listing.get("owner_property_id"),
+                    "uprn": listing.get("national_uprn"),
+                    "outcome": deal_data.get("outcome"),
+                    "outcome_notes": deal_data.get("outcome_notes"),
+                    "project_code": deal_data.get("project_code"),
+                    "company_id": company,
+                    "major_condition_issue_description": deal_data.get(
+                        "major_condition_issue_description"
+                    ),
+                    "major_condition_issue_photos": deal_data.get(
+                        "major_condition_issue_photos"
+                    ),
+                    "major_condition_issue_description": deal_data.get(
+                        "major_condition_issue_description"
+                    ),
+                    "major_condition_issue_photos": deal_data.get(
+                        "major_condition_issue_photos"
+                    ),
+                    "coordination_status": deal_data.get(
+                        "coordination_status__stage_1_"
+                    ),
+                    "design_status": deal_data.get("retrofit_design_status"),
+                }.items():
+                    setattr(existing, attr, value or getattr(existing, attr))
+
+                # Upload if photo exists but S3 link missing
+                if (
+                    existing.major_condition_issue_photos
+                    and not existing.major_condition_issue_evidence_s3_url
+                ):
+                    # Fetch fresh URL from HubSpot instead of using potentially expired stored URL
+                    fresh_deal = hubspot_client.from_deal_get_info(existing.deal_id)
+                    photo_url = fresh_deal.get("major_condition_issue_photos")
+
+                    if photo_url:
+                        try:
+                            local_file = hubspot_client.download_file_from_url(
+                                photo_url
+                            )
+                            s3_url = self.s3.upload_file(
+                                local_file,
+                                "retrofit-data-dev",
+                                prefix="hubspot/awaabs_law_evidence/",
+                            )
+                            existing.major_condition_issue_evidence_s3_url = s3_url
+                        except Exception as e:
+                            print(
+                                f"⚠️ Failed to download photo for deal_id {existing.deal_id}: {e}"
+                            )
+                            # Continue without the file — don't crash the update
+                    else:
+                        print(f"⚠️ Photo URL missing for deal_id {existing.deal_id}")
+
+                session.add(existing)
+                session.commit()
+                session.refresh(existing)
+                return existing
+
+            else:
+                print(f"🆕 Inserting new deal (deal_id={deal_id})")
+                new_record = HubspotDealData(
+                    deal_id=deal_id,
+                    dealname=deal_data.get("dealname"),
+                    dealstage=deal_data.get("dealstage"),
+                    landlord_property_id=listing.get("owner_property_id"),
+                    uprn=listing.get("national_uprn"),
+                    outcome=deal_data.get("outcome"),
+                    outcome_notes=deal_data.get("outcome_notes"),
+                    project_code=deal_data.get("project_code"),
+                    company_id=company,
+                    major_condition_issue_description=deal_data.get(
+                        "major_condition_issue_description"
+                    ),
+                    major_condition_issue_photos=deal_data.get(
+                        "major_condition_issue_photos"
+                    ),
+                    coordination_status=deal_data.get("coordination_status__stage_1_"),
+                    design_status=deal_data.get("retrofit_design_status"),
+                )
+
+                # Handle upload at insert time
+                if new_record.major_condition_issue_photos:
+                    try:
+                        local_file = hubspot_client.download_file_from_url(
+                            new_record.major_condition_issue_photos
+                        )
+                        s3_url = self.s3.upload_file(
+                            local_file,
+                            "retrofit-data-dev",
+                            prefix="hubspot/awaabs_law_evidence/",
+                        )
+                        new_record.major_condition_issue_evidence_s3_url = s3_url
+                    except Exception as e:
+                        print(
+                            f"⚠️ Failed to download photo for deal_id {new_record.deal_id}: {e}"
+                        )
+                        # Continue without the file — don't crash the insert
+
+                session.add(new_record)
+                session.commit()
+                session.refresh(new_record)
+                return new_record

etl/development.py

@@ -1,6 +1,6 @@
 from etl.scraper.scraper import SharePointScraper, SharePointInstaller
 from pprint import pformat
-from etl.pdfReader.pdfReaderToText import pdfReaderToText
+from etl.fileReader.pdfReaderToText import pdfReaderToText
 from etl.surveyedData.surveryedData import surveyedDataProcessor
 import pandas as pd
 

etl/dimitra_hubspot_notes_gather.py

@@ -6,7 +6,7 @@ from bs4 import BeautifulSoup
 from openpyxl import Workbook
 from openpyxl.styles import Font
 from etl.scraper.scraper import SharePointScraper, SharePointInstaller, previous_monday
-from etl.hubSpotClient.hubspot import HubSpotClient, DealStage
+from etl.hubSpotClient.hubspotClient import HubSpotClient, DealStage
 from collections import defaultdict
 import time
 # Auth credentials
@@ -63,6 +63,7 @@ for pipeline in pipelines.results:
         for stage in pipeline.stages:
             if stage.label.upper().strip() not in [s.upper() for s in exclude_stage.get(pipeline_name, [])]:
                 for deals in hubspot.get_all_deals_from_stage_id(stage.id):
+                    print(f"Scraping deal {deals['deal_id']}")
                     time.sleep(1)
                     deal_notes_by_week = {"Week 1": [], "Week 2": [], "Week 3": []}
                     notes = hubspot.get_notes_from_deals_id(deals["deal_id"])
@@ -71,7 +72,12 @@ for pipeline in pipelines.results:
                         week_label = get_week_label(note["created_at"])
                         if not week_label:
                             continue
-                        html_body = note['note']
+                        html_body = note.get('note')
+                        if not html_body:
+                            print(f"Skipping note with missing 'note' field: {note}")
+                            continue
+
+                        print(f"Debugging purposes html_body looks like {html_body}")
                         soup = BeautifulSoup(html_body, "html.parser")
                         plain_text = soup.get_text(separator="\n")
                         deal_notes_by_week[week_label].append(plain_text)
@@ -85,11 +91,12 @@ for pipeline in pipelines.results:
                             except:
                                 owner_name = "Couldn't find owner information"
 
+                        # Unique identifier to Domna Homes' hubspot
                         portal_id = 145275138
 
                         notes_data[pipeline_name].append({
                             "Deal Name": deal_name.upper(),
-                            "Deal URL": f"https://app-eu1.hubspot.com/contacts/{portal_id}/record/0-3/{deals["deal_id"]}/",
+                            "Deal URL": f"https://app-eu1.hubspot.com/contacts/{portal_id}/record/0-3/{deals['deal_id']}/",
                             "Deal Owner": owner_name,
                             "Deal Stage": stage.label.upper(),
                             "Value": deals["value"],
@@ -150,10 +157,7 @@ for pipeline, deals in notes_data.items():
 
 
 # Generate file name with next Monday’s date
-days_ahead = (7 - today.weekday()) % 7
-days_ahead = 7 if days_ahead == 0 else days_ahead
-next_monday = today + timedelta(days=days_ahead)
-formatted = next_monday.strftime("%d-%m-%Y Monday")
+formatted = previous_monday()
 file_name = f"{formatted} DEAL_NOTES_FROM_HUBSPOT.xlsx"
 output_path = os.path.abspath(file_name)
 wb.save(output_path)
@@ -165,3 +169,6 @@ sharepoint_client.upload_file(
     f"02. Sales and Marketing/02. Deal Notes from Hubspot/{formatted}",
     file_name
 )
+
+
+print("hello world")

etl/epr_etl_example.py

@@ -0,0 +1,9 @@
+from etl.surveyedData.surveryedData import surveyedDataProcessor
+
+files  = [
+    # "/tmp/sharepoint/Sandwell/SANDWELL-001/26 Willow close B64 6EG/Content (13).pdf",
+    "/tmp/sharepoint/Livewest/Livewest-001/12 Birch End/Summary Information 12 Birch End.pdf"
+]
+
+from sqlalchemy.dialects.postgresql import UUID
+sdp = surveyedDataProcessor("fake address", files)

etl/fileReader/pdfReaderToText.py

@@ -1,8 +1,17 @@
 from etl.utils.logger import Logger
 import logging
 import pymupdf
-from etl.pdfReader.sitenotes import QuidosSiteNotesExtractor, CSR
-from etl.pdfReader.reportType import ReportType
+from etl.fileReader.sitenotes import (
+    QuidosSiteNotesExtractor,
+    CSR,
+    WarmHomesConditionReport,
+    ECOConditionReport,
+    EnergyPerformanceReportWithData,
+    EnergyPerformanceReportSummaryInformation,
+    SmartEpcSiteNote,
+)
+from etl.fileReader.reportType import ReportType
+from pprint import pprint
 
 class pdfReaderToText():
 
@@ -40,6 +49,16 @@ class pdfReaderToText():
                 self.type = ReportType.ENERGY_PERFORMANCE_REPORT
             elif "Chartered Surveyor Report: Recommending Extraction of Defective Cavity Wall Insulation " in self.text_list:
                 self.type = ReportType.CHARTED_SURVEYOR_REPORT
+            elif "Osmosis ACD NEW PAS 2035 Condition Report".lower() in self.text_list[0].lower():
+                self.type = ReportType.WARM_HOMES_CONDITION_REPORT
+            elif "Domna NEW PAS 2035 ECO Condition Report".lower() in self.text_list[0].lower():
+                self.type = ReportType.ECO_CONDITION_REPORT
+            elif "ENERGY REPORT".lower() == self.text_list[0].lower() and "Data inputs" in self.text_list:
+                self.type = ReportType.ENERGY_PERFORMANCE_REPORT_WITH_DATA
+            elif "Summary Information".lower() == self.text_list[0].lower():
+                self.type = ReportType.ENERGY_PERFORMANCE_REPORT_SUMMARY_INFORMATION
+            elif "SMART EPC: Record of" in self.all_text and "Inspection & Site Notes" in self.all_text:
+                self.type = ReportType.SMART_EPC_SITE_NOTE
             else:
                 pass
             return self.type
@@ -51,4 +70,13 @@ class pdfReaderToText():
             return QuidosSiteNotesExtractor(self.text_list)
         elif self.type == ReportType.CHARTED_SURVEYOR_REPORT:
             return CSR(self.text_list) 
-        
+        elif self.type == ReportType.WARM_HOMES_CONDITION_REPORT:
+            return WarmHomesConditionReport(self.text_list)
+        elif self.type == ReportType.ECO_CONDITION_REPORT:
+            return ECOConditionReport(self.text_list)
+        elif self.type == ReportType.ENERGY_PERFORMANCE_REPORT_WITH_DATA:
+            return EnergyPerformanceReportWithData(self.text_list)
+        elif self.type == ReportType.ENERGY_PERFORMANCE_REPORT_SUMMARY_INFORMATION:
+            return EnergyPerformanceReportSummaryInformation(self.text_list)
+        elif self.type == ReportType.SMART_EPC_SITE_NOTE:
+            return SmartEpcSiteNote(self.text_list)

etl/fileReader/reportType.py

@@ -0,0 +1,16 @@
+from enum import Enum
+
+
+class ReportType(Enum):
+    QUIDOS_PRESITE_NOTE = "quidos_presite_note"
+    CHARTED_SURVEYOR_REPORT = "charted_surveyor_report"
+    U_VALUE_CALCULATOR_REPORT = "u_value_calculator_report"
+    OVERWRITING_U_VALUE_DECLARATION_FORM = "overwriting_u_value_declaration_form"
+    ECO_CONDITION_REPORT = "osmosis_condition_pas_2035_report"
+    WARM_HOMES_CONDITION_REPORT = "warm_homes_condition_pas_2035_report"
+    SMART_EPC_SITE_NOTE = "smart_epc_site_note"
+    ENERGY_PERFORMANCE_REPORT_WITH_DATA = "energy_performance_report_with_data"
+    ENERGY_PERFORMANCE_REPORT_SUMMARY_INFORMATION = "energy_performance_report_summary_information"
+    LIG_XML = "lodgement_xml_needed_for_lodgement_to_like_trademark"
+    RDSAP_XML = "reduce_xml_needed_to_generate_full_sap_xml"
+    FULLSAP_XML = "full_xml_needed_for_co_ordination"

etl/fileReader/xmlReader.py

@@ -0,0 +1,43 @@
+from etl.utils.logger import Logger
+import logging
+from xml.dom.minidom import parse
+import os
+from etl.fileReader.reportType import ReportType
+
+class xmlReader():
+    def __init__(self, file_path):
+        self.source_path = file_path
+        self.logger = Logger(name='xmlReader', level=logging.INFO).get_logger()
+        self.xml_obj = None 
+        self.type = None
+        self.get_xml_obj()
+
+ 
+    def get_xml_obj(self):
+        try:
+            if not os.path.exists(self.source_path):
+                self.logger.error(f"File not found: {self.source_path}")
+                return None
+
+            with open(self.source_path, 'r', encoding='utf-8') as file:
+                self.xml_obj = parse(file)
+                self.get_type()
+                return self.xml_obj
+
+        except Exception as e:
+            self.logger.error(f"Failed to parse XML file {self.source_path}: {e}")
+            self.xml_obj = None
+            return self.xml_obj
+        
+    def get_type(self):
+        xmlHeaderName = self.xml_obj.documentElement.tagName
+        xmlHeaderName = xmlHeaderName.lower()
+        if xmlHeaderName == 'RdSap-Report'.lower():
+            self.type = ReportType.LIG_XML
+        elif xmlHeaderName == "SurveyRec".lower():
+            self.type = ReportType.RDSAP_XML
+        elif xmlHeaderName == "ImportExportRecord".lower():
+            self.type = ReportType.FULLSAP_XML
+        else:
+            pass
+        return self.type

etl/hubSpotClient/hubspot.py

@@ -1,252 +0,0 @@
-import hubspot
-from enum import Enum
-from hubspot.crm.deals import PublicObjectSearchRequest
-from hubspot.crm.deals.models import SimplePublicObjectInput
-from etl.hubSpotClient.types import SubmissionInfoFromDeal
-import time
-from pydantic import ValidationError
-from etl.utils.logger import Logger
-import logging
-
-
-
-class DealStage(Enum):
-    SURVEYED_COMPLETE_NEEDS_SIGN_OFF = "1617223914"
-    SURVEYED_NO_ACCESS_NEED_SIGN_OFF = "1617223915"
-    CUSTOMER_CONTACTED = "888730834"
-    SURVEYED_COMPLETED_SIGNED_OFF = "1617223916"
-    NEEDS_ADDITIONAL_INFORMATION_FROM_ASSESSOR = "1887736000"
-
-class HubSpotClient():
-    def __init__(self):
-        self.access_token = "pat-eu1-064f7f5c-a7d8-4d93-a9b2-b604da6164a6"
-        self.client = hubspot.Client.create(access_token=self.access_token)
-        self.logger = Logger(name='HubSpotClient', level=logging.INFO).get_logger()
-
-    def get_all_deals(self):
-        return self.client.crm.deals.get_all()
-
-    def get_owner_name_from_id(self, owner_id):
-        owner = self.client.crm.owners.owners_api.get_by_id(owner_id)
-        time.sleep(1)
-        first_name = owner.first_name or ""
-        last_name = owner.last_name or ""
-        return f"{first_name} {last_name}".strip()
-        
-    def get_deal_name_by_id(self, deal_id):
-        try:
-            deal = self.client.crm.deals.basic_api.get_by_id(deal_id)
-            time.sleep(1)
-            return deal.properties.get("dealname", "No deal name")
-        except Exception as e:
-            return "Unknown Deal"  # Fallback if the deal name is not found
-        
-    def get_listings_from_deals_id(self, deals_id):
-        from hubspot.crm.objects import PublicObjectSearchRequest
-        found_notes = []
-        after = None
-        while True:
-            # Correct filter for notes associated with the given deal ID
-            search_request = PublicObjectSearchRequest(
-                filter_groups=[{
-                    "filters": [{
-                        "propertyName": "associations.deal",  # Filter by association to the deal
-                        "operator": "EQ",
-                        "value": deals_id,
-                    }]
-                }],
-                properties=["domna_property_id", "owner_property_id", 'national_uprn'],  # Properties of the note you need
-                limit=200,
-                after=after,
-            )
-            # Call the search API
-            response = self.client.crm.objects.search_api.do_search(object_type="0-420", public_object_search_request=search_request)
-            time.sleep(1)
-
-            # Add the results to the found_notes list
-            found_notes.extend(response.results)
-
-            # Handle pagination if more results are available
-            if not response.paging or not response.paging.next:
-                break
-            after = response.paging.next.after
-
-        if found_notes:
-            return found_notes[0]
-        return None
-    
-    def get_domna_and_landlord_id(self, deals_id):
-        data = self.get_listings_from_deals_id(deals_id)
-        return data.properties['domna_property_id'], data.properties['owner_property_id'], data.properties['national_uprn']
-
-    def get_notes_from_deals_id(self, deals_id):
-        from hubspot.crm.objects import PublicObjectSearchRequest
-        found_notes = []
-        after = None
-        while True:
-            # Correct filter for notes associated with the given deal ID
-            search_request = PublicObjectSearchRequest(
-                filter_groups=[{
-                    "filters": [{
-                        "propertyName": "associations.deal",  # Filter by association to the deal
-                        "operator": "EQ",
-                        "value": deals_id,
-                    }]
-                }],
-                properties=["hs_note_body", "hubspot_owner_id"],  # Properties of the note you need
-                limit=200,
-                after=after,
-            )
-            # Call the search API
-            response = self.client.crm.objects.search_api.do_search(object_type="notes", public_object_search_request=search_request)
-            time.sleep(1)
-
-            # Add the results to the found_notes list
-            found_notes.extend(response.results)
-
-            # Handle pagination if more results are available
-            if not response.paging or not response.paging.next:
-                break
-            after = response.paging.next.after
-
-        all_notes = []
-        for note in found_notes:
-            # Extract note content and author information
-            note_body = note.properties.get("hs_note_body", "No content")
-
-            # Collect note details in a dictionary
-            all_notes.append({
-                "note_id": note.id,
-                "note": note_body,
-                "created_at": note.created_at.strftime("%Y-%m-%d %H:%M:%S"),
-            })
-        return all_notes
-    
-    
-    def get_all_deals_from_stage_id(self, stage_id):
-        found_deals = []
-        after = None
-        while True:
-            search_request = PublicObjectSearchRequest(
-                filter_groups=[{
-                    "filters": [{
-                        "propertyName": "dealstage",
-                        "operator": "EQ",
-                        "value": stage_id,
-                    }]
-                }],
-                properties=[
-                    "dealname",
-                    "amount",
-                    "hubspot_owner_id",
-                ],
-                limit=200,
-                after=after,
-            )
-            response = self.client.crm.deals.search_api.do_search(search_request)
-            time.sleep(1)
-            found_deals.extend(response.results)
-            if not response.paging or not response.paging.next:
-                break
-            after = response.paging.next.after
-
-        all_deals = []
-        for deal in found_deals:
-            all_deals.append({
-                "deal_id": deal.id,
-                "value": deal.properties["amount"],
-                "deal_owner": deal.properties.get("hubspot_owner_id"),
-            })
-        return all_deals
-    
-    def get_associations_for_deal(self, deal_id, to_object_type):
-        """
-        Returns a list of associated object IDs of type `to_object_type`
-        (e.g. "contacts", "companies", "notes", etc.)
-        """
-        assoc_resp = self.client.crm.deals.associations_api.get_all(
-            deal_id=deal_id,
-            to_object_type=to_object_type
-        )
-        return [assoc.id for assoc in assoc_resp.results]
-
-    def get_deals_from_deal_stage(self, deal_stage: DealStage):
-        found_deals = []
-        after = None
-        while True:
-            search_request = PublicObjectSearchRequest(
-                filter_groups=[{
-                    "filters": [{
-                        "propertyName": "dealstage",
-                        "operator": "EQ",
-                        "value": deal_stage.value,
-                    }]
-                }],
-                properties=[
-                    "dealname",
-                    "number_of_wet_rooms_needing_ventilation",
-                    "work_type",
-                    "property_needs_trickle_vents",
-                    "domna_survey_post_sap",
-                    "existing_wall_insulation",
-                    "installer",
-                    "submission_folder",
-                ],
-                limit=200,
-                after=after,
-            )
-            response = self.client.crm.deals.search_api.do_search(search_request)
-            found_deals.extend(response.results)
-            if not response.paging or not response.paging.next:
-                break
-            after = response.paging.next.after
-
-        all_deals = []
-        for deal in found_deals:
-            domna_id, landlord_id, uprn = self.get_domna_and_landlord_id(deal.id)
-            try:
-                all_deals.append(SubmissionInfoFromDeal(
-                    deal_id= deal.properties["hs_object_id"],
-                    deal_name=deal.properties["dealname"],
-                    work_type=deal.properties["work_type"],
-                    needs_trickle_ventilation=True if deal.properties.get("property_needs_trickle_vents", "NO").upper() == "YES" else False,
-                    post_sap_score=int(deal.properties["domna_survey_post_sap"]),
-                    existing_wall_insulation=deal.properties.get("existing_wall_insulation") if deal.properties.get("existing_wall_insulation") else "None",
-                    no_of_wet_rooms=int(deal.properties["number_of_wet_rooms_needing_ventilation"]),
-                    installer=deal.properties["installer"],
-                    submission_folder_path = deal.properties["submission_folder"],
-                    landlord_id = landlord_id,
-                    domna_id = domna_id,
-                    uprn = uprn,
-                ))
-            except Exception as e:
-                deal_id = deal.properties['hs_object_id']
-                self.logger.info(f"Deal <{deal_id}> not valid")
-                self.move_deals_to_different_stage([deal_id], DealStage.NEEDS_ADDITIONAL_INFORMATION_FROM_ASSESSOR.value)               
-                
-
-
-
-
-        return all_deals
-    
-    def print_all_pipeline_ids(self):
-        pipelines = self.client.crm.pipelines.pipelines_api.get_all(object_type="deals")
-        for pipeline in pipelines.results:
-            print(f"Pipeline: {pipeline.label}")
-            for stage in pipeline.stages:
-                print(f" - Label: {stage.label}")
-                print(f"   ID: {stage.id}")
-
-    def move_deals_to_different_stage(self, list_of_deals_id, to_stage_id):
-        deal_properties = SimplePublicObjectInput(
-            properties={
-                "dealstage": to_stage_id
-            }
-        )
-        for deal_id in list_of_deals_id:
-            self.client.crm.deals.basic_api.update(
-                deal_id,
-                simple_public_object_input=deal_properties
-            )
-            self.logger.info(f"Deal {deal_id} moved to stage with ID {to_stage_id}.")

etl/hubSpotClient/hubspotClient.py

@@ -0,0 +1,447 @@
+import hubspot
+from enum import Enum
+from etl.utils.logger import Logger
+import logging
+from hubspot.crm.associations import ApiException
+import os
+import requests
+
+from hubspot.crm.objects import SimplePublicObjectInput
+from hubspot.crm.associations.v4 import AssociationSpec
+from hubspot.crm.associations import ApiException
+
+
+class Companies(Enum):
+    ABRI = "237615001799"
+    SOUTHERN_HOUSING_GROUP = "109343619305"
+    LIVEWEST = "86205872354"
+    SURESERVE = "301745289413"
+    HOMEGROUP = "94946071794"
+    APPLE = "184769046716"
+    THE_GUINESS_PARTNERSHIP = "86970043613"
+
+
+class DealStage(Enum):
+    SURVEYED_COMPLETE_NEEDS_SIGN_OFF = "1617223914"
+    SURVEYED_NO_ACCESS_NEED_SIGN_OFF = "1617223915"
+    CUSTOMER_CONTACTED = "888730834"
+    SURVEYED_COMPLETED_SIGNED_OFF = "1617223916"
+    FILES_MISSING_FROM_ASSESSOR = "1887736000"
+
+
+class Pipeline(Enum):
+    OPERATIONS_SOCIAL_HOUSING = "1167582403"
+
+
+class HubSpotClient:
+    def __init__(self):
+        self.access_token = "pat-eu1-064f7f5c-a7d8-4d93-a9b2-b604da6164a6"
+        self.client = hubspot.Client.create(access_token=self.access_token)
+        self.logger = Logger(name="HubSpotClient", level=logging.INFO).get_logger()
+        self.all_deals = None
+
+    def get_all_deals(self):
+        self.all_deals = self.client.crm.deals.get_all()
+        return self.all_deals
+
+    def get_deal_ids_by_pipeline(self, pipeline_id):
+        """
+        Get all deal IDs associated with a given pipeline.
+        """
+        if self.all_deals is None:
+            self.get_all_deals()
+
+        # Filter deals where properties['pipeline'] matches the given pipeline_id
+        filtered_deals = [
+            deal
+            for deal in self.all_deals
+            if deal.properties["pipeline"] == str(pipeline_id)
+        ]
+
+        # Extract and return only the deal IDs
+        deal_ids = [deal.id for deal in filtered_deals]
+
+        return deal_ids
+
+    def get_deals_from_company(self, company_id: str) -> list[str]:
+        associations_api = self.client.crm.associations.v4.basic_api
+
+        deal_ids = []
+        after = None
+
+        while True:
+            response = associations_api.get_page(
+                object_type="companies",
+                object_id=company_id,
+                to_object_type="deals",
+                limit=100,
+                after=after,
+            )
+
+            deal_ids.extend(assoc.to_object_id for assoc in response.results)
+
+            if not response.paging or not response.paging.next:
+                break
+
+            after = response.paging.next.after
+
+        return deal_ids
+
+    def from_deal_get_associated_listing(self, deal_id: str):
+        """
+        Get the associated listing information for a given deal.
+        Returns a dictionary of listing properties, or None if not found.
+        """
+        associations_api = self.client.crm.associations.v4.basic_api
+        listings_api = (
+            self.client.crm.objects.basic_api
+        )  # works for custom objects like "listing"
+
+        # Fetch associated listing(s)
+        response = associations_api.get_page(
+            object_type="deals",
+            object_id=deal_id,
+            to_object_type="0-420",  # <-- use your exact custom object name slug here
+            limit=1,
+        )
+
+        if not response.results:
+            self.logger.info(f"No listing association found for deal {deal_id}")
+            return None
+
+        listing_id = response.results[0].to_object_id
+        self.logger.info(f"Associated listing ID for deal {deal_id}: {listing_id}")
+
+        # Fetch listing details (the "listing information")
+        listing = listings_api.get_by_id(
+            object_type="0-420",  # again, must match your HubSpot object name
+            object_id=listing_id,
+            properties=[
+                "national_uprn",
+                "domna_property_id",
+                "owner_property_id",
+            ],
+        )
+
+        listing_info = listing.properties
+        self.logger.info(f"Listing info for deal {deal_id}: {listing_info}")
+        return listing_info
+
+    def from_deal_get_info(self, deal_id):
+        deal = self.client.crm.deals.basic_api.get_by_id(
+            deal_id,
+            properties=[
+                "dealname",
+                "dealstage",
+                "pipeline",
+                "outcome",  # outcome,
+                "outcome_notes",  # outcome notes
+                "project_code",
+                "major_condition_issue_description",
+                "major_condition_issue_photos",
+                "coordination_status__stage_1_",  # Coordiantion Status (Stage 1),
+                "retrofit_design_status",  # Retrofit Design Status
+            ],
+        )
+
+        return deal.properties
+    
+    def from_deal_get_associated_company_id(self, deal_id: str):
+            """
+            Get the associated company ID from a given deal ID.
+            Returns the associated company ID, or None if not found.
+            """
+            try:
+                associations_api = self.client.crm.associations.v4.basic_api
+
+                # Fetch associations for this specific deal only
+                response = associations_api.get_page(
+                    object_type="deals",
+                    object_id=deal_id,
+                    to_object_type="companies",
+                    limit=1  # Expect only one associated company
+                )
+
+                if not response.results:
+                    self.logger.info(f"No company association found for deal {deal_id}")
+                    return None
+
+                company_id = response.results[0].to_object_id
+                self.logger.info(f"Associated company ID for deal {deal_id}: {company_id}")
+                return company_id
+
+            except ApiException as e:
+                self.logger.error(f"Error fetching associated company for deal {deal_id}: {e}")
+                return None
+
+    def get_deal_info_for_db(self, deal_id):
+        deal = self.from_deal_get_info(deal_id)
+        company = self.from_deal_get_associated_company_id(deal_id)
+        listing = self.from_deal_get_associated_listing(deal_id)
+
+        return deal, company, listing
+
+    def get_company_information(self, company_id):
+        company = self.client.crm.companies.basic_api.get_by_id(
+            company_id,
+            properties=[
+                "name",
+            ],
+        )
+        company_info = company.properties
+        return company_info
+
+    def get_all_pipelines(self):
+        """
+        Retrieve all pipelines for deals, returning a list of dicts with pipeline names and IDs.
+        """
+        try:
+            pipelines_api = self.client.crm.pipelines.pipelines_api
+            response = pipelines_api.get_all(object_type="deals")
+
+            pipelines = [
+                {"name": pipeline.label, "id": pipeline.id}
+                for pipeline in response.results
+            ]
+
+            self.logger.info(f"Retrieved {len(pipelines)} pipelines.")
+            return pipelines
+
+        except Exception as e:
+            self.logger.error(f"Error retrieving pipelines: {e}")
+            return []
+
+    def get_deal_stages(self, pipeline_id=None):
+        """
+        Retrieve all deal stages for a given pipeline.
+        If no pipeline_id is provided, retrieves all stages for all pipelines.
+        Returns a list of dicts with pipeline name, stage name, and stage ID.
+        """
+        try:
+            pipelines_api = self.client.crm.pipelines.pipelines_api
+            response = pipelines_api.get_all(object_type="deals")
+
+            all_stages = []
+
+            for pipeline in response.results:
+                # Skip other pipelines if a specific one is requested
+                if pipeline_id and pipeline.id != str(pipeline_id):
+                    continue
+
+                stages = [
+                    {
+                        "pipeline_name": pipeline.label,
+                        "pipeline_id": pipeline.id,
+                        "stage_name": stage.label,
+                        "stage_id": stage.id,
+                    }
+                    for stage in pipeline.stages
+                ]
+
+                all_stages.extend(stages)
+
+            if not all_stages:
+                self.logger.info(
+                    f"No deal stages found for pipeline {pipeline_id if pipeline_id else 'ALL'}"
+                )
+            else:
+                self.logger.info(f"Retrieved {len(all_stages)} deal stages.")
+
+            return all_stages
+
+        except Exception as e:
+            self.logger.error(f"Error retrieving deal stages: {e}")
+            return []
+
+    def download_file_from_url(self, download_url: str, save_path: str = None) -> str:
+        """
+        Download a file from a HubSpot file URL (public or private), keeping its original file type.
+        Includes retry logic for transient failures and normalization of URL-encoded special characters.
+        """
+        import mimetypes
+        import requests
+        import os
+        import time
+        import re
+        from urllib.parse import urlparse, urlunparse, unquote, quote
+
+        # Strip signature and expiration from CDN URLs to get a fresh one with response-content-disposition
+        # This is how HubSpot UI generates longer-lived URLs
+        if "cdnp1.hubspotusercontent" in download_url:
+            from urllib.parse import urlparse, urlunparse
+            parsed = urlparse(download_url)
+            # Keep only the path, strip all query params to force CDN to generate fresh signature
+            download_url = f"{parsed.scheme}://{parsed.netloc}{parsed.path}?response-content-disposition=attachment"
+            self.logger.info(
+                f"Requesting fresh CDN signature with response-content-disposition: {download_url}"
+            )
+
+        # Normalize URL-encoded special characters in the path (not query string)
+        # to avoid signature validation issues with special characters like %2c, %20
+        parsed = urlparse(download_url)
+        clean_path = quote(unquote(parsed.path), safe="/@:")
+        download_url = urlunparse(parsed._replace(path=clean_path))
+
+        max_attempts = 3
+        retry_delays = [1, 2, 4]  # exponential backoff in seconds
+        last_exception = None
+
+        for attempt in range(max_attempts):
+            try:
+                headers = {}
+                # Add auth token for API endpoints (not direct CDN URLs)
+                if "hubspot.com/form-integrations" in download_url or "api-eu1.hubspot.com" in download_url:
+                    headers["Authorization"] = f"Bearer {self.access_token}"
+
+                self.logger.info(
+                    f"Downloading HubSpot file (attempt {attempt + 1}/{max_attempts}): {download_url}"
+                )
+                response = requests.get(
+                    download_url,
+                    headers=headers,
+                    stream=True,
+                    allow_redirects=True,
+                    timeout=(10, 30),
+                )
+                response.raise_for_status()
+
+                # Try to infer filename from Content-Disposition header
+                content_disposition = response.headers.get("content-disposition")
+                if content_disposition and "filename=" in content_disposition:
+                    filename = content_disposition.split("filename=")[1].strip('"')
+                else:
+                    # fallback: extract from URL or content-type
+                    filename = (
+                        os.path.basename(download_url.split("?")[0]) or "hubspot_download"
+                    )
+                    if "." not in filename:
+                        content_type = response.headers.get("content-type")
+                        ext = (
+                            mimetypes.guess_extension(content_type.split(";")[0])
+                            if content_type
+                            else None
+                        )
+                        if ext:
+                            filename += ext
+
+                # Make sure save_path is valid
+                if save_path is None:
+                    save_path = os.path.abspath(filename)
+                elif os.path.isdir(save_path):
+                    save_path = os.path.join(save_path, filename)
+                else:
+                    # if user passes a file path directly, leave it
+                    save_path = os.path.abspath(save_path)
+
+                with open(save_path, "wb") as f:
+                    for chunk in response.iter_content(chunk_size=8192):
+                        f.write(chunk)
+
+                self.logger.info(f"File downloaded successfully → {save_path}")
+                return save_path
+
+            except requests.exceptions.HTTPError as e:
+                # Don't retry on 404 — file genuinely doesn't exist
+                if e.response is not None and e.response.status_code == 404:
+                    self.logger.error(f"Failed to download file from HubSpot: {e}")
+                    raise
+                last_exception = e
+                if attempt < max_attempts - 1:
+                    delay = retry_delays[attempt]
+                    self.logger.warning(
+                        f"HTTP error (attempt {attempt + 1}/{max_attempts}): {e} — retrying in {delay}s"
+                    )
+                    time.sleep(delay)
+            except (
+                requests.exceptions.ConnectionError,
+                requests.exceptions.Timeout,
+            ) as e:
+                last_exception = e
+                if attempt < max_attempts - 1:
+                    delay = retry_delays[attempt]
+                    self.logger.warning(
+                        f"Connection/timeout error (attempt {attempt + 1}/{max_attempts}): {e} — retrying in {delay}s"
+                    )
+                    time.sleep(delay)
+            except requests.exceptions.RequestException as e:
+                # Other request errors (e.g., invalid URL) — don't retry
+                self.logger.error(f"Failed to download file from HubSpot: {e}")
+                raise
+
+        # If we got here, all retries failed
+        self.logger.error(
+            f"Failed to download file after {max_attempts} attempts: {last_exception}"
+        )
+        raise last_exception
+
+    def create_line_item_from_product(self, product_id: str, quantity: int = 1):
+        # Fetch product mapping
+        product = self.client.crm.products.basic_api.get_by_id(
+            product_id, properties=["name", "price", "hs_price"]
+        )
+
+        name = product.properties.get("name")
+        price = (
+            product.properties.get("price") or product.properties.get("hs_price") or "0"
+        )
+
+        # Build line item payload
+        line_item_input = SimplePublicObjectInput(
+            properties={
+                "hs_product_id": product_id,
+                "name": name,
+                "quantity": str(quantity),
+                "price": price,
+                "amount": str(float(price) * quantity),
+                "invoiced": "Outstanding",
+            }
+        )
+
+        # Create line item
+        line_item = self.client.crm.line_items.basic_api.create(line_item_input)
+        return line_item.id
+
+    def associate_line_item_to_deal(self, line_item_id: str, deal_id: str):
+        self.logger.info(f"Associating line item {line_item_id} → deal {deal_id}")
+
+        association_api = self.client.crm.associations.v4.basic_api
+
+        association_api.create(
+            "0-3",  # to object type
+            deal_id,  # to object id
+            "line_items",  # from object type
+            line_item_id,  # from object id
+            [
+                AssociationSpec(
+                    association_category="HUBSPOT_DEFINED",
+                    association_type_id=19,  # line_item → deal
+                )
+            ],
+        )
+
+    def add_product_line_item_to_deal(
+        self, deal_id: str, product_id: str, quantity: int = 1
+    ):
+        # Step 1: Create the line item from product mapping
+        line_item_id = self.create_line_item_from_product(product_id, quantity)
+
+        # Step 2: Associate the created line item to the deal
+        self.associate_line_item_to_deal(line_item_id, deal_id)
+
+        return line_item_id
+
+    def delete_line_item(self, line_item_id: str):
+        """
+        Delete (archive) a line item in HubSpot by its ID.
+        """
+        try:
+            self.logger.info(f"Deleting line item {line_item_id}...")
+
+            self.client.crm.line_items.basic_api.archive(line_item_id)
+
+            self.logger.info(f"Line item {line_item_id} deleted successfully.")
+            return True
+
+        except ApiException as e:
+            self.logger.error(f"Failed to delete line item {line_item_id}: {e}")
+            return False

etl/hubSpotClient/hubspot_types.py

@@ -0,0 +1,15 @@
+# from sqlmodel import Field, SQLModel
+# from sqlalchemy import Column
+# from sqlalchemy.dialects.postgresql import UUID
+# import uuid
+# from pydantic import  Field
+
+
+
+# class BaseModel(SQLModel):
+#     id: uuid.UUID = Field(
+#         default_factory=uuid.uuid4,
+#         sa_column=Column(UUID(as_uuid=True), primary_key=True)
+#     )
+
+

etl/hubSpotClient/scripts/README.md

@@ -0,0 +1,325 @@
+# HubSpot Client Scripts - Onboarding Guide
+
+## Overview
+
+The scripts in this directory form a **3-stage ETL pipeline** for syncing HubSpot data (companies and deals) into the local database:
+
+1. **Stage 0 (Seed)**: `hubspot_company.py` — Load company master records
+2. **Stage 1 (Bulk Load)**: `hubspot_gather_all_deals.py` — Initial load of all deals for a company
+3. **Stage 2 (Sync/Update)**: `hubspot_update_script.py` — Ongoing synchronization (runs on a schedule)
+
+These scripts work together to keep your local database in sync with HubSpot while handling photo uploads to S3 and maintaining data integrity.
+
+---
+
+## Onboarding a New Client
+
+Follow these steps in order when adding a new company/client:
+
+### Step 1: Add the Company to the `Companies` Enum
+
+Edit `../hubspotClient.py` and add your new company to the `Companies` enum class:
+
+```python
+class Companies(Enum):
+    ABRI = "237615001799"
+    SOUTHERN_HOUSING_GROUP = "109343619305"
+    LIVEWEST = "86205872354"
+    SURESERVE = "301745289413"
+    HOMEGROUP = "94946071794"
+    APPLE = "184769046716"
+    THE_GUINESS_PARTNERSHIP = "86970043613"
+    YOUR_NEW_COMPANY = "YOUR_HUBSPOT_COMPANY_ID"  # ← Add here
+```
+
+**How to find your HubSpot Company ID:**
+- Log into HubSpot
+- Navigate to **Contacts** → **Companies**
+- Click on the company name
+- The URL will be: `https://app.hubspot.com/crm/xxx/objects/companies/COMPANY_ID` — copy that ID
+
+### Step 2: Update Each Script to Include Your Company
+
+After adding the enum, update the company lists in all three scripts:
+
+#### `hubspot_company.py` (line ~6)
+```python
+companies = [
+    Companies.THE_GUINESS_PARTNERSHIP,
+    Companies.YOUR_NEW_COMPANY  # ← Add here
+]
+```
+
+#### `hubspot_gather_all_deals.py` (line ~7)
+```python
+valuable_companies = [
+    Companies.THE_GUINESS_PARTNERSHIP.value,
+    Companies.YOUR_NEW_COMPANY.value  # ← Add here
+]
+```
+
+#### `hubspot_update_script.py` (line ~12)
+```python
+companies = [
+    Companies.THE_GUINESS_PARTNERSHIP,
+    Companies.YOUR_NEW_COMPANY  # ← Add here
+]
+```
+
+### Step 3: Run `hubspot_company.py` (One-time setup)
+
+This script seeds the company record into the `hubspot_company_data` table. Run it once:
+
+```bash
+python etl/hubSpotClient/scripts/hubspot_company.py
+```
+
+**What it does:**
+- Connects to HubSpot and fetches company information (name, ID)
+- Inserts the company record into the local database
+
+**Output:** You'll see the company added to `hubspot_company_data` table.
+
+### Step 3.5: Update Group ID in Database (Manual)
+
+After the company record is created, you need to manually update the **group ID** for the new company. This is done via DBeaver or pgAdmin:
+
+**Steps:**
+1. Open DBeaver or pgAdmin and connect to the database
+2. Navigate to the `hubspot_company_data` table
+3. Find the row with your new company (search by `company_name` or `company_id`)
+4. Edit the **`group_id`** column to the portfolio/group ID you want to track for this company
+5. Save the changes
+
+**Example Query** (if you prefer SQL):
+```sql
+UPDATE hubspot_company_data
+SET group_id = 'YOUR_GROUP_ID'
+WHERE company_id = 'YOUR_COMPANY_ID';
+```
+
+**What is Group ID?**
+- The group ID identifies which portfolio/group in your system this company belongs to
+- Each company can be associated with one group ID for tracking and organization
+- This field is used for tracking and reporting across your survey data
+
+### Step 4: Run `hubspot_gather_all_deals.py` (One-time bulk load)
+
+This script performs the initial load of all deals for your company, filtered by the `OPERATIONS_SOCIAL_HOUSING` pipeline. Run it once per company:
+
+```bash
+python etl/hubSpotClient/scripts/hubspot_gather_all_deals.py
+```
+
+**What it does:**
+- Fetches all deal IDs associated with your company from HubSpot
+- For each deal, retrieves detailed properties:
+  - `dealname`, `dealstage`, `pipeline`, `outcome`, `outcome_notes`, `project_code`
+  - `major_condition_issue_description`, `major_condition_issue_photos`
+  - `coordination_status__stage_1_`, `retrofit_design_status`
+- Filters to only deals in the `OPERATIONS_SOCIAL_HOUSING` pipeline
+- Fetches the associated listing (UPRN, property IDs)
+- Inserts each deal into the `hubspot_data` table
+- **Downloads photo evidence files** and uploads them to S3 (bucket: `retrofit-data-dev`)
+
+**⚠️ Note:** This script can take a long time if your company has many deals. It processes deals serially with progress reporting via `tqdm`.
+
+**Output:** Deals appear in `hubspot_data` table; photos appear in S3 at `s3://retrofit-data-dev/hubspot/awaabs_law_evidence/`.
+
+### Step 5: `hubspot_update_script.py` (Automatic scheduling)
+
+After the initial setup, **no manual action is needed**. This script runs automatically every 15 minutes during working hours as a scheduled job.
+
+**What it does:**
+- Queries the local database for all stored deals for each company
+- Compares each deal's stored fields against the live HubSpot data (13 fields checked)
+- Updates the database if any values have changed in HubSpot
+- **Uploads newly available photos** to S3 (with SHA-256 integrity verification)
+- Prints a summary report of changes, updates, and any failures
+
+---
+
+## Script Reference
+
+### `hubspot_company.py`
+
+**Stage:** Seed (one-time setup)
+**Frequency:** Run once per new company
+**Speed:** Fast
+
+**Purpose:** Load company master data into the database.
+
+**Database Output:**
+- Table: `hubspot_company_data`
+- Fields: `company_id`, `company_name`
+
+**Code Flow:**
+```
+For each company in config:
+  1. Call HubSpot API: get_company_information(company_id)
+  2. Insert record into hubspot_company_data table
+```
+
+---
+
+### `hubspot_gather_all_deals.py`
+
+**Stage:** Bulk Load (one-time per company)
+**Frequency:** Run once per company (manually triggered)
+**Speed:** Slow (serial processing of all deals)
+
+**Purpose:** Perform initial load of all deals for target companies.
+
+**Database Output:**
+- Table: `hubspot_data`
+- Fields: `deal_id`, `deal_name`, `company_id`, `stage`, `outcome`, `photos_s3_url`, and others
+
+**S3 Output:**
+- Bucket: `retrofit-data-dev`
+- Path: `hubspot/awaabs_law_evidence/{filename}`
+
+**Code Flow:**
+```
+For each company in config:
+  1. Fetch all deal IDs from HubSpot
+  2. For each deal:
+     a. Get deal properties from HubSpot
+     b. Filter by OPERATIONS_SOCIAL_HOUSING pipeline
+     c. Fetch associated listing data (UPRN, property IDs)
+     d. Insert deal into hubspot_data table
+     e. If photos exist: download from HubSpot URL, upload to S3, save S3 URL to DB
+     f. Print progress: "Uploaded deal_id {id} to db"
+```
+
+**Error Handling:** None — script will abort on first error. Re-run to retry.
+
+---
+
+### `hubspot_update_script.py`
+
+**Stage:** Sync/Update (ongoing maintenance)
+**Frequency:** Every 15 minutes during working hours (automated schedule)
+**Speed:** Fast (only processes stored deals, compares, updates deltas)
+
+**Purpose:** Keep database synchronized with live HubSpot data; handle new/updated photos.
+
+**Database Operations:**
+- Reads: All deals from `hubspot_data` for each company
+- Writes: Updates only when fields differ from HubSpot
+- S3 Uploads: New or previously missing photos
+
+**Summary Report:**
+After completion, prints a table of per-company statistics:
+```
+Company | Checked | Updated | Up-to-date | Failed
+```
+
+Plus detailed error messages for any failed updates.
+
+**Code Flow:**
+```
+1. Initialize HubSpot client (warm-up: get_deal_stages)
+2. For each company:
+   a. Query DB for all deals with company_id
+   b. For each deal:
+      - Fetch live deal data from HubSpot
+      - Compare 13 fields: deal_id, company_id, landlord_property_id, outcome,
+                          dealstage, dealname, project_code, uprn,
+                          outcome_notes, major_condition_issue_description,
+                          major_condition_issue_photos, coordination_status,
+                          design_status
+      - If any field differs: call upsert_hubspot_deal() to update DB
+      - If photos exist in HubSpot but not yet in S3:
+        * Download file from HubSpot URL
+        * Upload to S3
+        * Verify SHA-256 hash integrity
+        * Save S3 URL back to DB
+      - Collect success/failure counts
+   c. Print per-company summary
+3. Print all failures (if any) with error messages
+```
+
+**Error Handling:** Wrapped in try/except per deal. Failures are logged, and the script continues to the next deal.
+
+---
+
+## Common Tasks
+
+### I added a new company but deals aren't showing up
+
+**Checklist:**
+- [ ] Company added to `Companies` enum in `hubspotClient.py`
+- [ ] Company added to the `companies` list in **all three scripts**
+- [ ] Ran `hubspot_company.py` successfully
+- [ ] Ran `hubspot_gather_all_deals.py` and watched for "Uploaded deal_id" messages
+- [ ] Check database: `SELECT COUNT(*) FROM hubspot_data WHERE company_id = 'YOUR_ID'`
+- [ ] Check HubSpot: Does the company have any deals in the OPERATIONS_SOCIAL_HOUSING pipeline?
+
+### Deals exist in HubSpot but aren't syncing
+
+The `hubspot_gather_all_deals.py` script only loads deals in the `OPERATIONS_SOCIAL_HOUSING` pipeline. If deals are in a different pipeline, they won't be loaded. Check the deal's pipeline in HubSpot.
+
+### Photos aren't uploading
+
+- First run of `hubspot_gather_all_deals.py` should upload photos at import time
+- Subsequent runs of `hubspot_update_script.py` will upload newly available photos
+- Check S3 bucket `retrofit-data-dev` under `hubspot/awaabs_law_evidence/`
+- Check DB field `major_condition_issue_photos` (photo S3 URL is stored here)
+
+### I need to re-sync everything for a company
+
+1. Clear the deals from the database:
+   ```sql
+   DELETE FROM hubspot_data WHERE company_id = 'YOUR_COMPANY_ID';
+   ```
+2. Clear the company:
+   ```sql
+   DELETE FROM hubspot_company_data WHERE company_id = 'YOUR_COMPANY_ID';
+   ```
+3. Re-run from **Step 3** above (run `hubspot_company.py`, then `hubspot_gather_all_deals.py`)
+
+---
+
+## Dependencies
+
+All scripts depend on:
+- `HubSpotClient` from `../hubspotClient.py` — Handles HubSpot API calls
+- `HubspotTodb` from `../../db/hubSpotLoad.py` — Handles database operations (insert/upsert/query)
+- `tqdm` — Progress bars
+- Python `requests` — HTTP downloads for photo files
+
+Environment Requirements:
+- Valid HubSpot API token (configured in `HubSpotClient.__init__()`)
+- Database connection (configured in `HubspotTodb`)
+- S3 credentials (for photo uploads)
+- Network access to HubSpot API and S3
+
+---
+
+## Notes & Tips
+
+1. **Idempotency:** `hubspot_gather_all_deals.py` and `hubspot_update_script.py` use upsert logic, so they can be run multiple times without creating duplicates.
+
+2. **Large Portfolios:** If a company has thousands of deals, `hubspot_gather_all_deals.py` will take a while. Use `tqdm` progress indicators to monitor.
+
+3. **Error Handling:** `hubspot_update_script.py` has error handling per deal. `hubspot_company.py` and `hubspot_gather_all_deals.py` do not — any failure aborts the script. If interrupted, simply re-run.
+
+4. **Schedule:** `hubspot_update_script.py` is scheduled to run every 15 minutes during business hours (typically configured as a cron job or similar scheduler).
+
+5. **Photo Integrity:** The `hubspot_update_script.py` verifies downloaded photos using SHA-256 hashing before committing the S3 URL to the database.
+
+6. **Unused Fields:** The scripts populate `deals_to_add` and `deal_to_companies` dicts in `hubspot_gather_all_deals.py` but don't use them downstream. This is harmless but could be cleaned up in future refactors.
+
+---
+
+## Troubleshooting
+
+| Issue | Likely Cause | Solution |
+|-------|--------------|----------|
+| "Company not found" error | Company enum not added or typo in name | Double-check `Companies` enum in `hubspotClient.py` |
+| Deal count mismatch | Company wasn't added to the script's companies list | Ensure company is in `valuable_companies` / `companies` in all 3 scripts |
+| Slow script execution | Large portfolio or network latency | Normal for first run; `hubspot_update_script.py` is faster on subsequent runs |
+| Photos not uploading | Deal doesn't have `major_condition_issue_photos` property | Photos only upload if HubSpot deal has photos attached |
+| S3 upload fails | Credentials or bucket issues | Check IAM permissions and bucket name (`retrofit-data-dev`) |
+| Update script reports failures | Stale data or missing DB fields | Check error messages in summary report; may need to re-sync company |
+

etl/hubSpotClient/scripts/debug_one_address.py

@@ -0,0 +1,19 @@
+from etl.hubSpotClient.hubspotClient import HubSpotClient
+from etl.db.hubSpotLoad import HubspotTodb
+
+
+
+working_deal_id = "319174821072"
+
+deal_id = "484368267483"
+
+hubspot = HubSpotClient()
+
+
+db = HubspotTodb()
+deal = db.find_deal_with_deal_id(deal_id)
+deal2 = db.find_deal_with_deal_id(working_deal_id)
+
+
+db.update_deal(deal2, hubspot)
+db.update_deal(deal, hubspot)

etl/hubSpotClient/scripts/hubspot_company.py

@@ -0,0 +1,16 @@
+from etl.hubSpotClient.hubspotClient import HubSpotClient, Companies, Pipeline
+from tqdm import tqdm
+from etl.db.hubSpotLoad import HubspotTodb
+
+hubspot = HubSpotClient()
+
+companies = [
+    Companies.THE_GUINESS_PARTNERSHIP,
+    Companies.SOUTHERN_HOUSING_GROUP,
+]
+# All deals from a pipeline_id via filter
+
+for company in companies:
+    new_company_info = hubspot.get_company_information(company.value)
+    loader = HubspotTodb()
+    loader.new_record_company(new_company_info)