junte/survey-extraction
1
0
Fork 0
mirror of https://github.com/Hestia-Homes/survey-extraction.git synced 2026-07-12 13:29:08 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Compare commits

base: junte:main
Branches Tags
junte:main
junte:feautre/fix_month_end
junte:feature/proof_of_site_notes
junte:solar-sharepoint-site
junte:feature/do_not_merge_this_branch
junte:feature/new
junte:feature/month_end_january
junte:feature/sureurve
junte:feature/tasks
junte:feature/images_in_s3
junte:feature/scrape_livewest_and_southern
junte:feature/ui_for_db
junte:feature/hubspot_sync
junte:feautre/walthamforest_etl
junte:feautre/adhoc_script_for_nick
junte:feature/month_end_automation_of_all
junte:feautre/additional_features_in_condition_report_extraction
junte:feature/document_upload
junte:feature/seperate_terraform_with_different_states
junte:feature/elmhurst_epr
junte:feature/energy_report_etl
junte:feature/hubspot_to_verification
junte:feature/condition_report_etl
junte:feature/nextjs
junte:feature/hubspot-as_the_source-of_truth
junte:feature/osmosis_complaince_automation
junte:feature/hubspot_to_db
junte:feature/hubspot_to_compelte_deem
junte:feature/hubspot_magic
junte:feautre/wave_3_make_something_on_share_point
junte:feature/terraform_to_aws
junte:feature/scary_asyncio_changes
junte:1.0.0
..
compare: junte:1.0.0
Branches Tags
junte:main
junte:feautre/fix_month_end
junte:feature/proof_of_site_notes
junte:solar-sharepoint-site
junte:feature/do_not_merge_this_branch
junte:feature/new
junte:feature/month_end_january
junte:feature/sureurve
junte:feature/tasks
junte:feature/images_in_s3
junte:feature/scrape_livewest_and_southern
junte:feature/ui_for_db
junte:feature/hubspot_sync
junte:feautre/walthamforest_etl
junte:feautre/adhoc_script_for_nick
junte:feature/month_end_automation_of_all
junte:feautre/additional_features_in_condition_report_extraction
junte:feature/document_upload
junte:feature/seperate_terraform_with_different_states
junte:feature/elmhurst_epr
junte:feature/energy_report_etl
junte:feature/hubspot_to_verification
junte:feature/condition_report_etl
junte:feature/nextjs
junte:feature/hubspot-as_the_source-of_truth
junte:feature/osmosis_complaince_automation
junte:feature/hubspot_to_db
junte:feature/hubspot_to_compelte_deem
junte:feature/hubspot_magic
junte:feautre/wave_3_make_something_on_share_point
junte:feature/terraform_to_aws
junte:feature/scary_asyncio_changes
junte:1.0.0

No commits in common. "main" and "1.0.0" have entirely different histories.
main ... 1.0.0

182 changed files with 2597 additions and 21848 deletions
Show stats Expand all files Collapse all files

8
.db-env
View file

@ -1,10 +1,10 @@
OSTGRES_USER=postgres
POSTGRES_USER=postgres
PGDATABASE=surveyDB
POSTGRES_PASSWORD=makingwarmhomes
POSTGRES_HOST=localhost
POSTGRES_PORT=5432
PGADMIN_DEFAULT_EMAIL=junte@domna.homes
PGADMIN_DEFAULT_PASSWORD=makingwarmhomes
# DATABASE_URL=postgresql://postgres:makingwarmhomes@db:5432/postgres
# Prod(dev-aws) Database Don't use
DATABASE_URL=postgresql://postgres:makingwarmhomes@terraform-20250331175522503500000002.cdgzupxvdyp0.eu-west-2.rds.amazonaws.com:5432/surveyDB
DATABASE_URL=postgresql://postgres:makingwarmhomes@db:5432/postgres
# Prod(dev-aws) Database Don't use!!!!
#DATABASE_URL=postgresql://postgres:makingwarmhomes@terraform-20250331175522503500000002.cdgzupxvdyp0.eu-west-2.rds.amazonaws.com:5432/surveyDB

29
.devcontainer/Dockerfile
View file

@ -3,12 +3,6 @@ FROM library/python:3.12-bullseye
ARG USER=vscode
ARG DEBIAN_FRONTEND=noninteractive
# DO NOT PUSH IMAGE TO ECR!!! as anyone with access to image can log on to our aws
# Will log on as aws Jun-te account, change in the future to development account
ENV AWS_ACCESS_KEY_ID=AKIAU5A36PPNK7RXX52V
ENV AWS_SECRET_ACCESS_KEY=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
ENV AWS_DEFAULT_REGION=eu-west-2
# Install system dependencies in a single layer
RUN apt update && apt install -y --no-install-recommends \
sudo jq vim curl\
@ -16,19 +10,28 @@ RUN apt update && apt install -y --no-install-recommends \
&& rm -rf /var/lib/apt/lists/*
# Create the user and grant sudo privileges
RUN useradd -m -s /bin/bash ${USER} \
RUN useradd -m -s /usr/bin/bash ${USER} \
&& echo "${USER} ALL=(ALL) NOPASSWD: ALL" >/etc/sudoers.d/${USER} \
&& chmod 0440 /etc/sudoers.d/${USER}
# Install Poetry
RUN pip install --no-cache-dir poetry
# Download and install nvm:
# RUN curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.3/install.sh | bash
# Install Node.js 22 (from NodeSource)
RUN curl -fsSL https://deb.nodesource.com/setup_22.x | bash - \
&& apt install -y nodejs \
&& node -v \
&& npm -v
# # in lieu of restarting the shell
# RUN \. "$HOME/.nvm/nvm.sh"
# # Download and install Node.js:
# RUN nvm install 22
# # Verify the Node.js version:
# RUN node -v # Should print "v22.16.0".
# RUN nvm current # Should print "v22.16.0".
# # Verify npm version:
# RUN npm -v # Should print "10.9.2".
# Install aws
@ -50,4 +53,4 @@ RUN terraform -install-autocomplete
# Set the working directory
WORKDIR /workspaces/survey-extractor
WORKDIR /workspaces/survey-extractor:q

16
.devcontainer/devcontainer.json
View file

@ -5,16 +5,10 @@
"remoteUser": "vscode",
"workspaceFolder": "/workspaces/survey-extractor",
"postStartCommand": "bash .devcontainer/post-install.sh",
"features": {
// "ghcr.io/devcontainers/features/ssh-agent:1": {}
},
"mounts": [
// Optional convenience mount
"source=${localEnv:HOME},target=/workspaces/home,type=bind"
"source=${localEnv:HOME},target=/workspaces/home,type=bind",
"source=${localEnv:HOME}/.aws/,target=/home/vscode/.aws/,type=bind"
],
"customizations": {
"vscode": {
"settings": {
@ -28,12 +22,8 @@
"lindacong.vscode-book-reader",
"4ops.terraform",
"fabiospampinato.vscode-todo-plus",
"jgclark.vscode-todo-highlight",
"corentinartaud.pdfpreview",
"GrapeCity.gc-excelviewer",
"anthropic.claude-code"
"jgclark.vscode-todo-highlight"
]
}
}
}

53
.devcontainer/docker-compose.yml
View file

@ -11,40 +11,39 @@ services:
- ../.db-env
volumes:
- ..:/workspaces/survey-extractor
# depends_on:
# - db
depends_on:
- db
networks:
- survey-net
# db:
# image: postgres:17.4
# restart: unless-stopped
# ports:
# - 5432:5432
# env_file:
# - ../.db-env
# volumes:
# - postgres-data-two:/var/lib/postgresql/data
# networks:
# - survey-net
db:
image: postgres:17.4
restart: unless-stopped
ports:
- 5432:5432
env_file:
- ../.db-env
volumes:
- postgres-data:/var/lib/postgresql/data
networks:
- survey-net
# pgadmin:
# image: dpage/pgadmin4
# hostname: pgadmin
# ports:
# - 5555:80
# env_file:
# - ../.db-env
# restart: unless-stopped
# depends_on:
# - db
# networks:
# - survey-net
pgadmin:
image: dpage/pgadmin4
hostname: pgadmin
ports:
- 5555:80
env_file:
- ../.db-env
restart: unless-stopped
depends_on:
- db
networks:
- survey-net
networks:
survey-net:
driver: bridge
volumes:
postgres-data-two:
postgres-data:

86
.github/workflows/actions/lambda-deploy/action.yml vendored
View file

@ -1,86 +0,0 @@
name: "Build and Push Lambda Image to ECR"
description: "Reusable action for building and pushing lambda Docker image to ECR"
inputs:
ecr_name:
description: "Lambda name / ECR repo name"
required: true
dockerfile_path:
description: "Path to Dockerfile"
required: true
ecr_tf_dir:
description: "Path to ECR terraform directory"
required: true
lambda_tf_dir:
description: "Path to Lambda terraform directory"
required: true
aws-access-key-id:
description: "AWS access key"
required: true
aws-secret-access-key:
description: "AWS secret key"
required: true
aws-region:
description: "AWS region"
required: true
git-sha:
description: "Git commit SHA"
required: true
git-ref:
description: "Git ref name"
required: true
runs:
using: "composite"
steps:
- uses: actions/checkout@v4
- name: Configure AWS credentials
uses: aws-actions/configure-aws-credentials@v4
with:
aws-access-key-id: ${{ inputs.aws-access-key-id }}
aws-secret-access-key: ${{ inputs.aws-secret-access-key }}
aws-region: ${{ inputs.aws-region }}
- name: Log in to Amazon ECR
id: login-ecr
uses: aws-actions/amazon-ecr-login@v2
- name: Deploy ECR
uses: ./.github/workflows/actions/terraform-deploy
with:
working_directory: ${{ inputs.ecr_tf_dir }}
aws-access-key-id: ${{ inputs.aws-access-key-id }}
aws-secret-access-key: ${{ inputs.aws-secret-access-key }}
aws-region: ${{ inputs.aws-region }}
- name: Set Docker image tag
id: set_tag
shell: bash
run: |
SHORT_SHA=$(echo "${{ inputs.git-sha }}" | cut -c1-7)
BRANCH=$(echo "${{ inputs.git-ref }}" | tr '/' '-')
TAG="${BRANCH}-${SHORT_SHA}"
echo "IMAGE_TAG=${TAG}" >> $GITHUB_ENV
echo "tag=$TAG" >> $GITHUB_OUTPUT
- name: Build and push Docker image
shell: bash
run: |
IMAGE_URI=${{ steps.login-ecr.outputs.registry }}/${{ inputs.ecr_name }}:${{ steps.set_tag.outputs.tag }}
echo "Building Docker image for ${{ inputs.ecr_name }}..."
docker build -t $IMAGE_URI -f ${{ inputs.dockerfile_path }} .
echo "Pushing to ECR..."
docker push $IMAGE_URI
- name: Deploy Lambda
uses: ./.github/workflows/actions/terraform-deploy
with:
working_directory: ${{ inputs.lambda_tf_dir }}
aws-access-key-id: ${{ inputs.aws-access-key-id }}
aws-secret-access-key: ${{ inputs.aws-secret-access-key }}
aws-region: ${{ inputs.aws-region }}
lambda-image-tag: ${{ steps.set_tag.outputs.tag }}

55
.github/workflows/actions/terraform-deploy/action.yml vendored
View file

@ -1,55 +0,0 @@
name: "Terraform Plan Shared Config"
description: "Plans shared Terraform config for Lambdas"
inputs:
working_directory:
description: "Directory containing Terraform config"
required: true
aws-access-key-id:
description: "AWS access key"
required: true
aws-secret-access-key:
description: "AWS secret key"
required: true
aws-region:
description: "AWS region"
required: true
lambda-image-tag:
description: "Tag of the Lambda image (e.g., GitHub SHA)"
required: false
runs:
using: "composite"
steps:
- uses: actions/checkout@v4
- name: Configure AWS credentials
uses: aws-actions/configure-aws-credentials@v4
with:
aws-access-key-id: ${{ inputs.aws-access-key-id }}
aws-secret-access-key: ${{ inputs.aws-secret-access-key }}
aws-region: ${{ inputs.aws-region }}
- name: Setup Terraform
uses: hashicorp/setup-terraform@v3
- name: Terraform Init
working-directory: ${{ inputs.working_directory }}
shell: bash
run: terraform init -reconfigure
- name: Terraform Plan
working-directory: ${{ inputs.working_directory }}
shell: bash
run: |
if [ -n "${{ inputs.lambda-image-tag }}" ]; then
terraform plan -out=tfplan -var="lambda_image_tag=${{ inputs.lambda-image-tag }}"
else
terraform plan -out=tfplan
fi
- name: Terraform Apply
working-directory: ${{ inputs.working_directory }}
shell: bash
run: terraform apply -auto-approve tfplan

29
.github/workflows/hubspot_deal_notes.yml vendored Normal file
View file

@ -0,0 +1,29 @@
name: Deal Notes From HubSpot Scraper
on:
schedule:
- cron: '0 19 * * 0'
workflow_dispatch:
jobs:
sharepoint-validator:
runs-on: ubuntu-22.04
steps:
- uses: actions/checkout@v3
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: '3.12'
- name: Install dependencies
run: |
pip install poetry
poetry install --no-root
- name: run script
run: |
pwd
ls -la
poetry run python etl/dimitra_hubspot_notes_gather.py
env:
PYTHONPATH: ${{ github.workspace }}

51
.github/workflows/hubspot_sync.yml vendored
View file

@ -1,51 +0,0 @@
# name: Hubspot Sync
# on:
# schedule:
# # Every 15 minutes, 07:00–18:59, Monday–Friday (UTC)
# - cron: '0 7-18/2 * * 1-5'
# # Once on Saturday at 09:00 UTC
# - cron: '0 9 * * 6'
# # Once on Sunday at 09:00 UTC
# - cron: '0 9 * * 0'
# workflow_dispatch:
# jobs:
# hubspot-sync:
# runs-on: [self-hosted, mist]
# steps:
# - uses: actions/checkout@v4
# - name: Set up Python
# uses: actions/setup-python@v5
# with:
# python-version: '3.12'
# - name: Install dependencies
# run: |
# pip install poetry
# poetry install --no-root
# # - name: Run scripts
# # env:
# # PYTHONPATH: ${{ github.workspace }}
# # DATABASE_URL: ${{ secrets.PROD_DATABASE_URL }}
# # run: |
# # pwd
# # ls -la
# # poetry run python etl/hubSpotClient/scripts/hubspot_gather_all_deals.py
# - name: Run scripts
# env:
# PYTHONPATH: ${{ github.workspace }}
# DATABASE_URL: ${{ secrets.PROD_DATABASE_URL }}
# run: |
# pwd
# ls -la
# poetry run python etl/hubSpotClient/scripts/hubspot_update_script.py

37
.github/workflows/hubspot_to_invoice.yml vendored Normal file
View file

@ -0,0 +1,37 @@
name: HubSpot Deals to DB loading and Invoice Calculator
on:
# schedule:
# - cron: '0 19 * * 0'
workflow_dispatch:
jobs:
hubspot-deals-to-db:
runs-on: ubuntu-22.04
steps:
- uses: actions/checkout@v3
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: '3.12'
- name: Install dependencies
run: |
pip install poetry
poetry install --no-root
- name: run script
run: |
pwd
ls -la
poetry run python etl/hubspot_to_invoice_rewrite.py
env:
PYTHONPATH: ${{ github.workspace }}
DATABASE_URL: postgresql://postgres:makingwarmhomes@terraform-20250331175522503500000002.cdgzupxvdyp0.eu-west-2.rds.amazonaws.com:5432/surveyDB
SOUTH_COAST_INSULATION_SERVICE_SHAREPOINT_ID: ${{ secrets.SOUTH_COAST_INSULATION_SERVICE_SHAREPOINT_ID }}
JJC_SERVICE_SHAREPOINT_ID: ${{ secrets.JJC_SERVICE_SHAREPOINT_ID }}
BAXTER_KELLY_SERVICE_SHAREPOINT_ID: ${{ secrets.BAXTER_KELLY_SERVICE_SHAREPOINT_ID }}
SGEC_SERVICE_SHAREPOINT_ID: ${{ secrets.SGEC_SERVICE_SHAREPOINT_ID }}
SHAREPOINT_CLIENT_ID: ${{ secrets.SHAREPOINT_CLIENT_ID }}
SHAREPOINT_CLIENT_SECRET: ${{ secrets.SHAREPOINT_CLIENT_SECRET }}
SHAREPOINT_TENANT_ID: ${{ secrets.SHAREPOINT_TENANT_ID }}

92
.github/workflows/lambda_main.yml vendored
View file

@ -1,92 +0,0 @@
name: Lambda Main Workflow
on:
push:
branches: [main, feautre/walthamforest_etl]
env:
AWS_REGION: eu-west-2
jobs:
shared-lambda-terraform:
runs-on: ubuntu-latest
steps:
- name: Checkout repo
uses: actions/checkout@v4
- name: Deploy shared Lambda Config Terraform
uses: ./.github/workflows/actions/terraform-deploy
with:
working_directory: ./deployment/lambda/lambda_shared
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
aws-region: ${{ secrets.AWS_REGION }}
lambda-ecr-example:
runs-on: ubuntu-latest
needs: shared-lambda-terraform
permissions:
id-token: write
contents: read
steps:
- name: Checkout repo
uses: actions/checkout@v4
- name: Build and deploy Lambda example
uses: ./.github/workflows/actions/lambda-deploy
with:
ecr_name: lambda_example
dockerfile_path: ./deployment/lambda/lambda_example/docker/Dockerfile
ecr_tf_dir: ./deployment/lambda/lambda_example/docker/
lambda_tf_dir: ./deployment/lambda/lambda_example/
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
aws-region: ${{ secrets.AWS_REGION }}
git-sha: ${{ github.sha }}
git-ref: ${{ github.ref_name }}
extractor-and-loader:
runs-on: ubuntu-latest
needs: shared-lambda-terraform
permissions:
id-token: write
contents: read
steps:
- name: Checkout repo
uses: actions/checkout@v4
- name: Build and deploy Extractor & Loader Lambda
uses: ./.github/workflows/actions/lambda-deploy
with:
ecr_name: extractor_and_loader
dockerfile_path: ./deployment/lambda/extractor_and_loader/docker/Dockerfile
ecr_tf_dir: ./deployment/lambda/extractor_and_loader/docker/
lambda_tf_dir: ./deployment/lambda/extractor_and_loader/
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
aws-region: ${{ secrets.AWS_REGION }}
git-sha: ${{ github.sha }}
git-ref: ${{ github.ref_name }}
walthamforest-etl:
runs-on: ubuntu-latest
needs: shared-lambda-terraform
permissions:
id-token: write
contents: read
steps:
- name: Checkout repo
uses: actions/checkout@v4
- name: Build and deploy WalthamForest ETL
uses: ./.github/workflows/actions/lambda-deploy
with:
ecr_name: walthamforest_etl_adhoc_ecr
dockerfile_path: ./deployment/lambda/walthamforest_etl/docker/Dockerfile
ecr_tf_dir: ./deployment/lambda/walthamforest_etl/docker/
lambda_tf_dir: ./deployment/lambda/walthamforest_etl/
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
aws-region: ${{ secrets.AWS_REGION }}
git-sha: ${{ github.sha }}
git-ref: ${{ github.ref_name }}

39
.github/workflows/months_end.yml vendored
View file

@ -1,39 +0,0 @@
name: Months End
on:
schedule:
- cron: '0 7 23-31 * *' # Every day from the 23rd to end of month at 07:00 UTC
workflow_dispatch:
jobs:
surveyed-needs-sign-off:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: '3.12'
- name: Install dependencies
run: |
pip install poetry
poetry install --no-root
- name: Run scripts
env:
PYTHONPATH: ${{ github.workspace }}
run: |
pwd
ls -la
poetry run python etl/month_end_automation_wave_2_layout.py # Done
poetry run python etl/month_end_automation_wave_2_no_3.py # Done
poetry run python etl/month_end_automation_wave_2_no_4.py # Done
poetry run python etl/month_end_automation_wave_2_no_6.py # Check with Matt if this can be deleted
poetry run python etl/month_end_automation_wave_2_no_7.py # Done
poetry run python etl/month_end_automation_wave_2_no_8.py # Done
poetry run python etl/month_end_automation_wave_2_no_11.py # Done
poetry run python etl/month_end_automation_wave_2_no_15.py # Done
poetry run python etl/month_end_automation_wave_accent_housing.py # Done
poetry run python etl/month_end_automation_wave_3_layout.py # Done

42
.github/workflows/pytest.yml vendored Normal file
View file

@ -0,0 +1,42 @@
name: Run Pytest
on:
push:
branches:
- '**' # Run on all branches
pull_request:
branches:
- main
jobs:
etl-unit-tests:
runs-on: ubuntu-22.04
steps:
- name: Checkout Repository
uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: '3.12'
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: '3.12'
- name: Install dependencies
run: |
pip install poetry
poetry install --no-root
- name: Run Tests
run: |
poetry run pytest -W ignore::DeprecationWarning
env:
PYTHONPATH: ${{ github.workspace }}
continue-on-error: ${{ github.event_name == 'push' && github.ref != 'refs/heads/main' }}

39
.github/workflows/scis_invoice_calculator.yml vendored Normal file
View file

@ -0,0 +1,39 @@
name: SCIS Invoice Calculator
on:
schedule:
- cron: '0 6 * * *'
workflow_dispatch:
jobs:
scis_invoice_calculator:
runs-on: ubuntu-22.04
steps:
- uses: actions/checkout@v3
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: '3.12'
- name: Install dependencies
run: |
pip install poetry
poetry install --no-root
- name: run script
run: |
bash scis_invoice.sh
env:
PYTHONPATH: ${{ github.workspace }}
SOUTH_COAST_INSULATION_SERVICE_SHAREPOINT_ID: ${{ secrets.SOUTH_COAST_INSULATION_SERVICE_SHAREPOINT_ID }}
JJC_SERVICE_SHAREPOINT_ID: ${{ secrets.JJC_SERVICE_SHAREPOINT_ID }}
BAXTER_KELLY_SERVICE_SHAREPOINT_ID: ${{ secrets.BAXTER_KELLY_SERVICE_SHAREPOINT_ID }}
SGEC_SERVICE_SHAREPOINT_ID: ${{ secrets.SGEC_SERVICE_SHAREPOINT_ID }}
SHAREPOINT_CLIENT_ID: ${{ secrets.SHAREPOINT_CLIENT_ID }}
SHAREPOINT_CLIENT_SECRET: ${{ secrets.SHAREPOINT_CLIENT_SECRET }}
SHAREPOINT_TENANT_ID: ${{ secrets.SHAREPOINT_TENANT_ID }}
- name: Upload Excel file
uses: actions/upload-artifact@v4
with:
name: my-excel-file
path: survey_data.xlsx

36
.github/workflows/sharepoint_validator.yml vendored Normal file
View file

@ -0,0 +1,36 @@
name: SharePoint Validator
on:
schedule:
- cron: '0 6 * * *'
workflow_dispatch:
jobs:
sharepoint-validator:
runs-on: ubuntu-22.04
steps:
- uses: actions/checkout@v3
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: '3.12'
- name: Install dependencies
run: |
pip install poetry
poetry install --no-root
- name: run script
run: |
pwd
ls -la
bash run_daily_script.sh
env:
PYTHONPATH: ${{ github.workspace }}
SOUTH_COAST_INSULATION_SERVICE_SHAREPOINT_ID: ${{ secrets.SOUTH_COAST_INSULATION_SERVICE_SHAREPOINT_ID }}
JJC_SERVICE_SHAREPOINT_ID: ${{ secrets.JJC_SERVICE_SHAREPOINT_ID }}
BAXTER_KELLY_SERVICE_SHAREPOINT_ID: ${{ secrets.BAXTER_KELLY_SERVICE_SHAREPOINT_ID }}
SGEC_SERVICE_SHAREPOINT_ID: ${{ secrets.SGEC_SERVICE_SHAREPOINT_ID }}
SHAREPOINT_CLIENT_ID: ${{ secrets.SHAREPOINT_CLIENT_ID }}
SHAREPOINT_CLIENT_SECRET: ${{ secrets.SHAREPOINT_CLIENT_SECRET }}
SHAREPOINT_TENANT_ID: ${{ secrets.SHAREPOINT_TENANT_ID }}

4
.gitignore vendored
View file

@ -4,6 +4,4 @@ __pycache__/
data/
*ipynb
etl/survery_data.csv
foo.env.py
*.xlsx
*.csv
foo.env.py

23
.vscode/settings.json vendored
View file

@ -3,7 +3,6 @@
"python.REPL.sendToNativeREPL": true,
"notebook.output.scrolling": true,
"terminal.integrated.defaultProfile.linux": "bash",
"editor.rulers": [67],
"terminal.integrated.profiles.linux": {
"bash": {
"path": "/bin/bash"
@ -15,27 +14,5 @@
// "%load_ext autoreload", "%autoreload 2"
// ]
"vim.enableNeovim": false,
// Allow VSCode native keybindings to override Vim when needed
"vim.handleKeys": {
"<C-p>": false,
"<C-P>": false,
"<C-S-p>": false,
"<C-c>": false,
"<C-v>": false,
"<C-S-v>": false,
"<C-S-e>": false,
"<C-b>": false,
"<C-j>": false,
"<C-S-c>": false
},
// Terminal copy/paste via Ctrl+Shift+C / Ctrl+Shift+V
"terminal.integrated.copyOnSelection": false,
"terminal.integrated.commandsToSkipShell": [
"workbench.action.terminal.copySelection",
"workbench.action.terminal.paste"
],
}

BIN
RdSAP_SiteNote_95053636_V3_Assessment (1).pdf
View file

Binary file not shown.

10
alembic/env.py
View file

@ -5,10 +5,8 @@ from sqlalchemy import pool
from alembic import context
from sqlmodel import SQLModel
from etl.models.topLevel import *
from etl.models.preSiteNoteTypes import *
from etl.models.conditionReport import *
from etl.fileReader.reportType import ReportType
from etl.load.topLevel import *
from etl.load.preSiteNoteTypes import *
import os
@ -34,8 +32,6 @@ def run_migrations_offline() -> None:
target_metadata=target_metadata,
literal_binds=True,
dialect_opts={"paramstyle": "named"},
compare_type=True,
compare_server_default=True,
)
with context.begin_transaction():
context.run_migrations()
@ -48,8 +44,6 @@ def run_migrations_online() -> None:
context.configure(
connection=connection,
target_metadata=target_metadata,
compare_type=True,
compare_server_default=True,
)
with context.begin_transaction():
context.run_migrations()

19
alembic/things_alembic_doesn't_automate_for.txt
View file

@ -1,19 +0,0 @@
Source: https://alembic.sqlalchemy.org/en/latest/autogenerate.html
Autogenerate can not detect:
Changes of table name. These will come out as an add/drop of two different tables, and should be hand-edited into a name change instead.
Changes of column name. Like table name changes, these are detected as a column add/drop pair, which is not at all the same as a name change.
Anonymously named constraints. Give your constraints a name, e.g. UniqueConstraint('col1', 'col2', name="my_name"). See the section The Importance of Naming Constraints for background on how to configure automatic naming schemes for constraints.
Special SQLAlchemy types such as Enum when generated on a backend which doesn’t support ENUM directly - this because the representation of such a type in the non-supporting database, i.e. a CHAR+ CHECK constraint, could be any kind of CHAR+CHECK. For SQLAlchemy to determine that this is actually an ENUM would only be a guess, something that’s generally a bad idea. To implement your own “guessing” function here, use the sqlalchemy.events.DDLEvents.column_reflect() event to detect when a CHAR (or whatever the target type is) is reflected, and change it to an ENUM (or whatever type is desired) if it is known that that’s the intent of the type. The sqlalchemy.events.DDLEvents.after_parent_attach() can be used within the autogenerate process to intercept and un-attach unwanted CHECK constraints.
Autogenerate can’t currently, but will eventually detect:
Some free-standing constraint additions and removals may not be supported, including PRIMARY KEY, EXCLUDE, CHECK; these are not necessarily implemented within the autogenerate detection system and also may not be supported by the supporting SQLAlchemy dialect.
Sequence additions, removals - not yet implemented.

46
alembic/versions/1f1d3d560ccb_db_defaults_enum_binding_for_uploaded_.py
View file

@ -1,46 +0,0 @@
"""db defaults + enum binding for uploaded_files
Revision ID: 1f1d3d560ccb
Revises: 270ba252bc11
Create Date: 2025-08-14 17:11:41.866908
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import postgresql
# revision identifiers, used by Alembic.
revision: str = '1f1d3d560ccb'
down_revision: Union[str, None] = '270ba252bc11'
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
"""Upgrade schema."""
# ### commands auto generated by Alembic - please adjust! ###
op.alter_column('uploaded_files', 's3_file_upload_timestamp',
existing_type=postgresql.TIMESTAMP(),
type_=sa.DateTime(timezone=True),
existing_nullable=False)
op.alter_column('uploaded_files', 's3_json_upload_timestamp',
existing_type=postgresql.TIMESTAMP(),
type_=sa.DateTime(timezone=True),
existing_nullable=True)
# ### end Alembic commands ###
def downgrade() -> None:
"""Downgrade schema."""
# ### commands auto generated by Alembic - please adjust! ###
op.alter_column('uploaded_files', 's3_json_upload_timestamp',
existing_type=sa.DateTime(timezone=True),
type_=postgresql.TIMESTAMP(),
existing_nullable=True)
op.alter_column('uploaded_files', 's3_file_upload_timestamp',
existing_type=sa.DateTime(timezone=True),
type_=postgresql.TIMESTAMP(),
existing_nullable=False)
# ### end Alembic commands ###

68
alembic/versions/20c418a7d5ec_add_company_id_in_hubspot_data_and_.py
View file

@ -1,68 +0,0 @@
"""add company id in hubspot data and rename table
Revision ID: 20c418a7d5ec
Revises: e72e15f7e0c3
Create Date: 2025-10-27 16:20:11.362657
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import postgresql
# revision identifiers, used by Alembic.
revision: str = '20c418a7d5ec'
down_revision: Union[str, None] = 'e72e15f7e0c3'
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
import sqlmodel
def upgrade() -> None:
"""Upgrade schema."""
# ### commands auto generated by Alembic - please adjust! ###
op.create_table('hubspot_deal_data',
sa.Column('id', sa.Uuid(), nullable=False),
sa.Column('deal_id', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
sa.Column('dealname', sqlmodel.sql.sqltypes.AutoString(), nullable=True),
sa.Column('dealstage', sqlmodel.sql.sqltypes.AutoString(), nullable=True),
sa.Column('company_id', sqlmodel.sql.sqltypes.AutoString(), nullable=True),
sa.Column('landlord_property_id', sqlmodel.sql.sqltypes.AutoString(), nullable=True),
sa.Column('uprn', sqlmodel.sql.sqltypes.AutoString(), nullable=True),
sa.Column('outcome', sqlmodel.sql.sqltypes.AutoString(), nullable=True),
sa.Column('outcome_notes', sqlmodel.sql.sqltypes.AutoString(), nullable=True),
sa.Column(
'created_at',
sa.DateTime(timezone=True),
server_default=sa.text('(CURRENT_TIMESTAMP AT TIME ZONE \'UTC\')'),
nullable=False,
),
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=True),
sa.PrimaryKeyConstraint('id')
)
op.create_index(op.f('ix_hubspot_deal_data_deal_id'), 'hubspot_deal_data', ['deal_id'], unique=False)
op.drop_index('ix_hubspot_data_deal_id', table_name='hubspot_data')
op.drop_table('hubspot_data')
# ### end Alembic commands ###
def downgrade() -> None:
"""Downgrade schema."""
# ### commands auto generated by Alembic - please adjust! ###
op.create_table('hubspot_data',
sa.Column('id', sa.UUID(), autoincrement=False, nullable=False),
sa.Column('deal_id', sa.VARCHAR(), autoincrement=False, nullable=False),
sa.Column('dealname', sa.VARCHAR(), autoincrement=False, nullable=True),
sa.Column('dealstage', sa.VARCHAR(), autoincrement=False, nullable=True),
sa.Column('landlord_property_id', sa.VARCHAR(), autoincrement=False, nullable=True),
sa.Column('uprn', sa.VARCHAR(), autoincrement=False, nullable=True),
sa.Column('outcome', sa.VARCHAR(), autoincrement=False, nullable=True),
sa.Column('outcome_notes', sa.VARCHAR(), autoincrement=False, nullable=True),
sa.Column('created_at', postgresql.TIMESTAMP(timezone=True), server_default=sa.text("(CURRENT_TIMESTAMP AT TIME ZONE 'UTC'::text)"), autoincrement=False, nullable=False),
sa.Column('updated_at', postgresql.TIMESTAMP(timezone=True), autoincrement=False, nullable=True),
sa.PrimaryKeyConstraint('id', name='hubspot_data_pkey')
)
op.create_index('ix_hubspot_data_deal_id', 'hubspot_data', ['deal_id'], unique=False)
op.drop_index(op.f('ix_hubspot_deal_data_deal_id'), table_name='hubspot_deal_data')
op.drop_table('hubspot_deal_data')
# ### end Alembic commands ###

32
alembic/versions/23a4e2cc5467_added_project_code.py
View file

@ -1,32 +0,0 @@
"""added project code
Revision ID: 23a4e2cc5467
Revises: 20c418a7d5ec
Create Date: 2025-10-27 16:23:16.984274
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
import sqlmodel
# revision identifiers, used by Alembic.
revision: str = '23a4e2cc5467'
down_revision: Union[str, None] = '20c418a7d5ec'
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
"""Upgrade schema."""
# ### commands auto generated by Alembic - please adjust! ###
op.add_column('hubspot_deal_data', sa.Column('project_code', sqlmodel.sql.sqltypes.AutoString(), nullable=True))
# ### end Alembic commands ###
def downgrade() -> None:
"""Downgrade schema."""
# ### commands auto generated by Alembic - please adjust! ###
op.drop_column('hubspot_deal_data', 'project_code')
# ### end Alembic commands ###

48
alembic/versions/2409147995c5_added_hubspot_table.py
View file

@ -1,48 +0,0 @@
"""added hubspot table
Revision ID: 2409147995c5
Revises: 4c67501b7451
Create Date: 2025-10-27 15:05:01.552689
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import postgresql
import sqlmodel
# revision identifiers, used by Alembic.
revision: str = '2409147995c5'
down_revision: Union[str, None] = '4c67501b7451'
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
op.create_table(
'hubspot_data',
sa.Column('id', sa.Uuid(), nullable=False),
sa.Column('deal_id', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
sa.Column('dealname', sqlmodel.sql.sqltypes.AutoString(), nullable=True),
sa.Column('dealstage', sqlmodel.sql.sqltypes.AutoString(), nullable=True),
sa.Column('landlord_property_id', sqlmodel.sql.sqltypes.AutoString(), nullable=True),
sa.Column('uprn', sqlmodel.sql.sqltypes.AutoString(), nullable=True),
sa.Column('outcome', sqlmodel.sql.sqltypes.AutoString(), nullable=True),
sa.Column('outcome_notes', sqlmodel.sql.sqltypes.AutoString(), nullable=True),
sa.Column('raw_data', postgresql.JSON(astext_type=sa.Text()), nullable=True),
sa.Column(
'created_at',
sa.DateTime(timezone=True),
server_default=sa.text('(CURRENT_TIMESTAMP AT TIME ZONE \'UTC\')'),
nullable=False,
),
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=True),
sa.PrimaryKeyConstraint('id')
)
def downgrade() -> None:
"""Downgrade schema."""
# ### commands auto generated by Alembic - please adjust! ###
op.drop_index(op.f('ix_hubspot_data_deal_id'), table_name='hubspot_data')
op.drop_table('hubspot_data')
# ### end Alembic commands ###

46
alembic/versions/253a1047c623_auto_generate_id.py
View file

@ -1,46 +0,0 @@
"""auto generate id
Revision ID: 253a1047c623
Revises: e8507a27795a
Create Date: 2025-08-14 17:25:54.010315
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import postgresql
# revision identifiers, used by Alembic.
revision: str = '253a1047c623'
down_revision: Union[str, None] = 'e8507a27795a'
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
"""Upgrade schema."""
# ### commands auto generated by Alembic - please adjust! ###
op.alter_column('uploaded_files', 'id',
existing_type=sa.UUID(),
server_default=sa.text('gen_random_uuid()'),
existing_nullable=False)
op.alter_column('uploaded_files', 's3_file_upload_timestamp',
existing_type=postgresql.TIMESTAMP(timezone=True),
server_default=sa.text("NOW() AT TIME ZONE 'utc'"),
existing_nullable=False)
# ### end Alembic commands ###
def downgrade() -> None:
"""Downgrade schema."""
# ### commands auto generated by Alembic - please adjust! ###
op.alter_column('uploaded_files', 's3_file_upload_timestamp',
existing_type=postgresql.TIMESTAMP(timezone=True),
server_default=None,
existing_nullable=False)
op.alter_column('uploaded_files', 'id',
existing_type=sa.UUID(),
server_default=None,
existing_nullable=False)
# ### end Alembic commands ###

30
alembic/versions/270ba252bc11_update_enum.py
View file

@ -1,30 +0,0 @@
"""update enum
Revision ID: 270ba252bc11
Revises: a6e4562797e4
Create Date: 2025-08-14 16:52:17.473370
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision: str = '270ba252bc11'
down_revision: Union[str, None] = 'a6e4562797e4'
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
op.execute(
"ALTER TYPE reporttype ADD VALUE IF NOT EXISTS 'osmosis_condition_pas_2035_report'"
)
def downgrade() -> None:
"""Downgrade schema."""
# ### commands auto generated by Alembic - please adjust! ###
pass
# ### end Alembic commands ###

34
alembic/versions/29113d69989e_enum_things.py
View file

@ -1,34 +0,0 @@
"""enum things
Revision ID: 29113d69989e
Revises: 2cf02c9f71f8
Create Date: 2025-08-19 11:40:52.712131
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision: str = '29113d69989e'
down_revision: Union[str, None] = '2cf02c9f71f8'
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade():
op.execute("ALTER TYPE reporttype ADD VALUE IF NOT EXISTS 'osmosis_condition_pas_2035_report'")
op.execute("ALTER TYPE reporttype ADD VALUE IF NOT EXISTS 'warm_homes_condition_pas_2035_report'")
op.execute("ALTER TYPE reporttype ADD VALUE IF NOT EXISTS 'energy_performance_report_with_data'")
op.execute("ALTER TYPE reporttype ADD VALUE IF NOT EXISTS 'energy_performance_report_summary_information'")
op.execute("ALTER TYPE reporttype ADD VALUE IF NOT EXISTS 'lodgement_xml_needed_for_lodgement_to_like_trademark'")
op.execute("ALTER TYPE reporttype ADD VALUE IF NOT EXISTS 'reduce_xml_needed_to_generate_full_sap_xml'")
op.execute("ALTER TYPE reporttype ADD VALUE IF NOT EXISTS 'full_xml_needed_for_co_ordination'")
def downgrade() -> None:
"""Downgrade schema."""
# ### commands auto generated by Alembic - please adjust! ###
pass
# ### end Alembic commands ###

41
alembic/versions/2cf02c9f71f8_add_missing_report_type.py
View file

@ -1,41 +0,0 @@
"""add missing report type
Revision ID: 2cf02c9f71f8
Revises: 253a1047c623
Create Date: 2025-08-19 11:36:16.006276
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import postgresql
from sqlalchemy.dialects import postgresql as psql
# revision identifiers, used by Alembic.
revision: str = '2cf02c9f71f8'
down_revision: Union[str, None] = '253a1047c623'
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
op.alter_column(
"uploaded_files",
"s3_json_uri",
type_=psql.JSON(), # or psql.JSONB()
postgresql_using="s3_json_uri::json", # or ::jsonb
existing_type=sa.VARCHAR(),
existing_nullable=True,
)
def downgrade() -> None:
"""Downgrade schema."""
# ### commands auto generated by Alembic - please adjust! ###
op.alter_column('uploaded_files', 's3_json_uri',
existing_type=postgresql.JSON(astext_type=sa.Text()),
type_=sa.VARCHAR(),
existing_nullable=True)
# ### end Alembic commands ###

354
alembic/versions/427e65da69c1_initial_table.py Normal file
View file

@ -0,0 +1,354 @@
"""Initial table
Revision ID: 427e65da69c1
Revises:
Create Date: 2025-05-14 15:36:08.611971
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
import sqlmodel
# revision identifiers, used by Alembic.
revision: str = '427e65da69c1'
down_revision: Union[str, None] = None
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
"""Upgrade schema."""
# ### commands auto generated by Alembic - please adjust! ###
op.create_table('buildings',
sa.Column('id', sa.Uuid(), nullable=False),
sa.Column('address', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
sa.Column('postcode', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
sa.Column('UPRN', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
sa.Column('landlord_id', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
sa.Column('domna_id', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
sa.PrimaryKeyConstraint('id')
)
op.create_table('companyinfo',
sa.Column('id', sa.Uuid(), nullable=False),
sa.Column('address', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
sa.Column('trading_name', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
sa.Column('post_code', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
sa.Column('fax_number', sqlmodel.sql.sqltypes.AutoString(), nullable=True),
sa.Column('related_party_disclosure', sqlmodel.sql.sqltypes.AutoString(), nullable=True),
sa.PrimaryKeyConstraint('id')
)
op.create_table('door',
sa.Column('id', sa.Uuid(), nullable=False),
sa.Column('no_of_doors', sa.Integer(), nullable=False),
sa.Column('no_of_insulated_doors', sa.Integer(), nullable=False),
sa.Column('u_value_w_m2_k', sqlmodel.sql.sqltypes.AutoString(), nullable=True),
sa.PrimaryKeyConstraint('id')
)
op.create_table('floors',
sa.Column('id', sa.Uuid(), nullable=False),
sa.Column('floor_type', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
sa.Column('ground_floor_construction', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
sa.Column('ground_floor_insulation_type', sqlmodel.sql.sqltypes.AutoString(), nullable=True),
sa.Column('floor_insulation_thickness_mm', sa.Float(), nullable=True),
sa.Column('u_value_known', sa.Boolean(), nullable=False),
sa.PrimaryKeyConstraint('id')
)
op.create_table('fluegasheatrecoverysystem',
sa.Column('id', sa.Uuid(), nullable=False),
sa.Column('fghrs_present', sa.Boolean(), nullable=False),
sa.PrimaryKeyConstraint('id')
)
op.create_table('heatingsystemcontrols',
sa.Column('id', sa.Uuid(), nullable=False),
sa.Column('control_type', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
sa.Column('flue_type', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
sa.Column('fan_assisted_flue', sa.Boolean(), nullable=False),
sa.Column('heat_emitter_type', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
sa.Column('electricity_meter_type', sqlmodel.sql.sqltypes.AutoString(), nullable=True),
sa.Column('mains_gas_available', sa.Boolean(), nullable=True),
sa.PrimaryKeyConstraint('id')
)
op.create_table('heatingtype',
sa.Column('id', sa.Uuid(), nullable=False),
sa.Column('heating_type', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
sa.Column('fuel_type', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
sa.PrimaryKeyConstraint('id')
)
op.create_table('hotwatercylinder',
sa.Column('id', sa.Uuid(), nullable=False),
sa.Column('volume', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
sa.Column('insulation_type', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
sa.Column('insulation_thickness', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
sa.Column('thermostat', sa.Boolean(), nullable=False),
sa.PrimaryKeyConstraint('id')
)
op.create_table('insulation',
sa.Column('id', sa.Uuid(), nullable=False),
sa.Column('type', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
sa.PrimaryKeyConstraint('id')
)
op.create_table('lighting',
sa.Column('id', sa.Uuid(), nullable=False),
sa.Column('total_no_of_light_fittings', sa.Integer(), nullable=False),
sa.Column('total_no_of_lel_fittings', sa.Integer(), nullable=False),
sa.PrimaryKeyConstraint('id')
)
op.create_table('otherdetails',
sa.Column('id', sa.Uuid(), nullable=False),
sa.Column('electricity_meter_type', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
sa.Column('main_gas_avalible', sa.Boolean(), nullable=False),
sa.PrimaryKeyConstraint('id')
)
op.create_table('photovoltaicpanel',
sa.Column('id', sa.Uuid(), nullable=False),
sa.Column('pvs_are_connected_to_dwelling_electricity_meter', sa.Boolean(), nullable=False),
sa.Column('percentage_of_external_roof_area_with_pvs', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
sa.PrimaryKeyConstraint('id')
)
op.create_table('presitenotessummaryinfo',
sa.Column('id', sa.Uuid(), nullable=False),
sa.Column('reference_number', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
sa.Column('epc_language', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
sa.Column('uprn', sqlmodel.sql.sqltypes.AutoString(), nullable=True),
sa.Column('postcode', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
sa.Column('region', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
sa.Column('address', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
sa.Column('town', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
sa.Column('county', sqlmodel.sql.sqltypes.AutoString(), nullable=True),
sa.Column('property_tenure', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
sa.Column('transaction_type', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
sa.Column('inspection_date', sa.DateTime(), nullable=False),
sa.Column('current_sap', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
sa.Column('potential_sap', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
sa.Column('current_ei', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
sa.Column('potential_ei', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
sa.Column('current_annual_emissions', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
sa.Column('current_annual_emission_including_0925_multiplayer', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
sa.Column('current_annual_energy_costs', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
sa.PrimaryKeyConstraint('id')
)
op.create_table('roofs',
sa.Column('id', sa.Uuid(), nullable=False),
sa.Column('construction', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
sa.Column('insulation_type', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
sa.Column('insulation_thickness', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
sa.Column('u_value_known', sa.Boolean(), nullable=False),
sa.PrimaryKeyConstraint('id')
)
op.create_table('showerandbaths',
sa.Column('id', sa.Uuid(), nullable=False),
sa.Column('no_of_rooms_with_baths_and_or_shower', sa.Integer(), nullable=False),
sa.Column('no_of_rooms_with_mixer_shower_and_no_baths', sa.Integer(), nullable=False),
sa.Column('no_of_rooms_with_mixer_shower_and_baths', sa.Integer(), nullable=False),
sa.PrimaryKeyConstraint('id')
)
op.create_table('solarwaterheating',
sa.Column('id', sa.Uuid(), nullable=False),
sa.Column('solar_water_heating_details_known', sa.Boolean(), nullable=False),
sa.PrimaryKeyConstraint('id')
)
op.create_table('ventilationandcooling',
sa.Column('id', sa.Uuid(), nullable=False),
sa.Column('no_of_open_fireplaces', sa.Integer(), nullable=False),
sa.Column('ventilation_type', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
sa.Column('space_cooling_system_present', sa.Boolean(), nullable=False),
sa.PrimaryKeyConstraint('id')
)
op.create_table('walls',
sa.Column('id', sa.Uuid(), nullable=False),
sa.Column('construction', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
sa.Column('insulation', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
sa.Column('insulation_thickness_mm', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
sa.Column('wall_thickness_measured', sa.Boolean(), nullable=False),
sa.Column('wall_thickness_mm', sa.Integer(), nullable=True),
sa.Column('u_value_known', sa.Boolean(), nullable=False),
sa.Column('u_value_w_m2_k', sa.Float(), nullable=True),
sa.Column('dry_lining', sa.Boolean(), nullable=False),
sa.Column('alternative_wall_present', sa.Boolean(), nullable=False),
sa.PrimaryKeyConstraint('id')
)
op.create_table('waterheating',
sa.Column('id', sa.Uuid(), nullable=False),
sa.Column('heating_type', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
sa.Column('fuel_type', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
sa.PrimaryKeyConstraint('id')
)
op.create_table('windturbine',
sa.Column('id', sa.Uuid(), nullable=False),
sa.Column('wind_turbine', sa.Boolean(), nullable=False),
sa.PrimaryKeyConstraint('id')
)
op.create_table('assessorinfo',
sa.Column('id', sa.Uuid(), nullable=False),
sa.Column('accreditation_number', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
sa.Column('name', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
sa.Column('phone_number', sqlmodel.sql.sqltypes.AutoString(), nullable=True),
sa.Column('email_address', sqlmodel.sql.sqltypes.AutoString(), nullable=True),
sa.Column('company_id', sa.Uuid(), nullable=True),
sa.ForeignKeyConstraint(['company_id'], ['companyinfo.id'], ),
sa.PrimaryKeyConstraint('id')
)
op.create_table('heating',
sa.Column('id', sa.Uuid(), nullable=False),
sa.Column('type', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
sa.Column('heating_source', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
sa.Column('efficiency_source', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
sa.Column('heating_fuel', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
sa.Column('brand_name', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
sa.Column('model_name', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
sa.Column('model_qualifer', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
sa.Column('sap_2009_table', sqlmodel.sql.sqltypes.AutoString(), nullable=True),
sa.Column('percentage_of_heated_floor_area_served', sqlmodel.sql.sqltypes.AutoString(), nullable=True),
sa.Column('controls_id', sa.Uuid(), nullable=True),
sa.ForeignKeyConstraint(['controls_id'], ['heatingsystemcontrols.id'], ),
sa.PrimaryKeyConstraint('id')
)
op.create_table('propertydetail',
sa.Column('id', sa.Uuid(), nullable=False),
sa.Column('age_band', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
sa.Column('wall_id', sa.Uuid(), nullable=True),
sa.Column('roof_id', sa.Uuid(), nullable=True),
sa.Column('floor_id', sa.Uuid(), nullable=True),
sa.ForeignKeyConstraint(['floor_id'], ['floors.id'], ),
sa.ForeignKeyConstraint(['roof_id'], ['roofs.id'], ),
sa.ForeignKeyConstraint(['wall_id'], ['walls.id'], ),
sa.PrimaryKeyConstraint('id')
)
op.create_table('dimension',
sa.Column('id', sa.Uuid(), nullable=False),
sa.Column('floor_area_m2', sa.Float(), nullable=False),
sa.Column('room_height_m', sa.Float(), nullable=False),
sa.Column('loss_perimeter_m', sa.Float(), nullable=False),
sa.Column('party_wall_length_m', sa.Float(), nullable=False),
sa.Column('property_detail_id', sa.Uuid(), nullable=True),
sa.ForeignKeyConstraint(['property_detail_id'], ['propertydetail.id'], ),
sa.PrimaryKeyConstraint('id')
)
op.create_table('documents',
sa.Column('id', sa.Uuid(), nullable=False),
sa.Column('assessor_id', sa.Uuid(), nullable=False),
sa.Column('created_at', sa.DateTime(), nullable=False),
sa.Column('document_type', sa.Enum('QUIDOS_PRESITE_NOTE', 'CHARTED_SURVEYOR_REPORT', 'ENERGY_PERFORMANCE_REPORT', 'U_VALUE_CALCULATOR_REPORT', 'OVERWRITING_U_VALUE_DECLARATION_FORM', name='reporttype'), nullable=False),
sa.Column('building_id', sa.Uuid(), nullable=False),
sa.Column('target_table', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
sa.Column('target_id', sa.Uuid(), nullable=False),
sa.ForeignKeyConstraint(['assessor_id'], ['assessorinfo.id'], ),
sa.ForeignKeyConstraint(['building_id'], ['buildings.id'], ),
sa.PrimaryKeyConstraint('id')
)
op.create_table('propertydescription',
sa.Column('id', sa.Uuid(), nullable=False),
sa.Column('built_form', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
sa.Column('detachment_or_position', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
sa.Column('no_of_main_property', sa.Integer(), nullable=False),
sa.Column('no_of_extension_1', sa.Integer(), nullable=True),
sa.Column('no_of_extension_2', sa.Integer(), nullable=True),
sa.Column('no_of_extension_3', sa.Integer(), nullable=True),
sa.Column('no_of_extension_4', sa.Integer(), nullable=True),
sa.Column('no_of_habitable_rooms', sa.Integer(), nullable=False),
sa.Column('no_of_heated_rooms', sa.Integer(), nullable=False),
sa.Column('heated_basement', sa.Boolean(), nullable=False),
sa.Column('conservatory_type', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
sa.Column('percentage_of_draught_proofed', sa.Integer(), nullable=False),
sa.Column('terrain_type', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
sa.Column('conservatory', sa.Boolean(), nullable=False),
sa.Column('main_property_id', sa.Uuid(), nullable=False),
sa.Column('ex1_property_id', sa.Uuid(), nullable=True),
sa.Column('ex2_property_id', sa.Uuid(), nullable=True),
sa.Column('ex3_property_id', sa.Uuid(), nullable=True),
sa.Column('ex4_property_id', sa.Uuid(), nullable=True),
sa.Column('door_id', sa.Uuid(), nullable=True),
sa.Column('ventilation_and_cooling_id', sa.Uuid(), nullable=True),
sa.Column('lighting_id', sa.Uuid(), nullable=True),
sa.Column('water_heating_id', sa.Uuid(), nullable=True),
sa.Column('hot_water_cylinder_id', sa.Uuid(), nullable=True),
sa.Column('solar_water_heating_id', sa.Uuid(), nullable=True),
sa.Column('shower_and_baths_id', sa.Uuid(), nullable=True),
sa.Column('flue_gas_heat_recovery_system_id', sa.Uuid(), nullable=True),
sa.Column('photovoltaic_panel_id', sa.Uuid(), nullable=True),
sa.Column('wind_turbine_id', sa.Uuid(), nullable=True),
sa.Column('other_details_id', sa.Uuid(), nullable=True),
sa.Column('main_heating_id', sa.Uuid(), nullable=True),
sa.Column('main_heating2_id', sa.Uuid(), nullable=True),
sa.Column('secondary_heating_type_id', sa.Uuid(), nullable=True),
sa.ForeignKeyConstraint(['door_id'], ['door.id'], ),
sa.ForeignKeyConstraint(['ex1_property_id'], ['propertydetail.id'], ),
sa.ForeignKeyConstraint(['ex2_property_id'], ['propertydetail.id'], ),
sa.ForeignKeyConstraint(['ex3_property_id'], ['propertydetail.id'], ),
sa.ForeignKeyConstraint(['ex4_property_id'], ['propertydetail.id'], ),
sa.ForeignKeyConstraint(['flue_gas_heat_recovery_system_id'], ['fluegasheatrecoverysystem.id'], ),
sa.ForeignKeyConstraint(['hot_water_cylinder_id'], ['hotwatercylinder.id'], ),
sa.ForeignKeyConstraint(['lighting_id'], ['lighting.id'], ),
sa.ForeignKeyConstraint(['main_heating2_id'], ['heating.id'], ),
sa.ForeignKeyConstraint(['main_heating_id'], ['heating.id'], ),
sa.ForeignKeyConstraint(['main_property_id'], ['propertydetail.id'], ),
sa.ForeignKeyConstraint(['other_details_id'], ['otherdetails.id'], ),
sa.ForeignKeyConstraint(['photovoltaic_panel_id'], ['photovoltaicpanel.id'], ),
sa.ForeignKeyConstraint(['secondary_heating_type_id'], ['heatingtype.id'], ),
sa.ForeignKeyConstraint(['shower_and_baths_id'], ['showerandbaths.id'], ),
sa.ForeignKeyConstraint(['solar_water_heating_id'], ['solarwaterheating.id'], ),
sa.ForeignKeyConstraint(['ventilation_and_cooling_id'], ['ventilationandcooling.id'], ),
sa.ForeignKeyConstraint(['water_heating_id'], ['waterheating.id'], ),
sa.ForeignKeyConstraint(['wind_turbine_id'], ['windturbine.id'], ),
sa.PrimaryKeyConstraint('id')
)
op.create_table('windows',
sa.Column('id', sa.Uuid(), nullable=False),
sa.Column('glazing_type', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
sa.Column('area_m2', sa.Float(), nullable=False),
sa.Column('roof_window', sa.Boolean(), nullable=False),
sa.Column('orientation', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
sa.Column('u_value_w_m2_k', sa.Integer(), nullable=False),
sa.Column('g_value', sa.Integer(), nullable=False),
sa.Column('property_detail_id', sa.Uuid(), nullable=True),
sa.ForeignKeyConstraint(['property_detail_id'], ['propertydetail.id'], ),
sa.PrimaryKeyConstraint('id')
)
op.create_table('presitenote',
sa.Column('id', sa.Uuid(), nullable=False),
sa.Column('summary_info_id', sa.Uuid(), nullable=False),
sa.Column('assessor_id', sa.Uuid(), nullable=False),
sa.Column('pre_site_note_description_id', sa.Uuid(), nullable=True),
sa.ForeignKeyConstraint(['assessor_id'], ['assessorinfo.id'], ),
sa.ForeignKeyConstraint(['pre_site_note_description_id'], ['propertydescription.id'], ),
sa.ForeignKeyConstraint(['summary_info_id'], ['presitenotessummaryinfo.id'], ),
sa.PrimaryKeyConstraint('id')
)
# ### end Alembic commands ###
def downgrade() -> None:
"""Downgrade schema."""
# ### commands auto generated by Alembic - please adjust! ###
op.drop_table('presitenote')
op.drop_table('windows')
op.drop_table('propertydescription')
op.drop_table('documents')
op.drop_table('dimension')
op.drop_table('propertydetail')
op.drop_table('heating')
op.drop_table('assessorinfo')
op.drop_table('windturbine')
op.drop_table('waterheating')
op.drop_table('walls')
op.drop_table('ventilationandcooling')
op.drop_table('solarwaterheating')
op.drop_table('showerandbaths')
op.drop_table('roofs')
op.drop_table('presitenotessummaryinfo')
op.drop_table('photovoltaicpanel')
op.drop_table('otherdetails')
op.drop_table('lighting')
op.drop_table('insulation')
op.drop_table('hotwatercylinder')
op.drop_table('heatingtype')
op.drop_table('heatingsystemcontrols')
op.drop_table('fluegasheatrecoverysystem')
op.drop_table('floors')
op.drop_table('door')
op.drop_table('companyinfo')
op.drop_table('buildings')
# ### end Alembic commands ###

60
alembic/versions/4c67501b7451_added_more_enums.py
View file

@ -1,60 +0,0 @@
"""added more enums
Revision ID: 4c67501b7451
Revises: ac8dba8cef50
Create Date: 2025-09-23 10:22:20.648664
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision: str = "4c67501b7451"
down_revision: Union[str, None] = "ac8dba8cef50"
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
ENUM_NAME = "reporttype"
# Values that were already present BEFORE this migration
OLD_VALUES = (
"QUIDOS_PRESITE_NOTE",
"CHARTED_SURVEYOR_REPORT",
"ENERGY_PERFORMANCE_REPORT",
"U_VALUE_CALCULATOR_REPORT",
"OVERWRITING_U_VALUE_DECLARATION_FORM",
"OSMOSIS_CONDITION_PAS_2035_REPORT",
"DOMNA_CONDITION_PAS_2035_REPORT",
)
# Values we are ADDING in this migration
NEW_VALUES = (
"DECENT_HOMES_RAW_DATA",
"DECENT_HOMES_SUMMARY",
"DECENT_HOMES_PROPERTY_META",
)
def upgrade() -> None:
for v in NEW_VALUES:
op.execute(f"ALTER TYPE {ENUM_NAME} ADD VALUE IF NOT EXISTS '{v}'")
def downgrade() -> None:
# 1) Create a replacement type with ONLY the old values
old_vals = ", ".join(f"'{v}'" for v in OLD_VALUES)
op.execute(f"CREATE TYPE {ENUM_NAME}_old AS ENUM ({old_vals})")
# 2) Move columns to the temporary type
op.execute(
f"ALTER TABLE documents ALTER COLUMN document_type TYPE {ENUM_NAME}_old "
f"USING document_type::text::{ENUM_NAME}_old"
)
op.execute(
f"ALTER TABLE uploaded_files ALTER COLUMN doc_type TYPE {ENUM_NAME}_old "
f"USING doc_type::text::{ENUM_NAME}_old"
)
# 3) Drop original type and rename the temp back
op.execute(f"DROP TYPE {ENUM_NAME}")
op.execute(f"ALTER TYPE {ENUM_NAME}_old RENAME TO {ENUM_NAME}")

46
alembic/versions/57c0dc06cd25_add_company_info.py
View file

@ -1,46 +0,0 @@
"""add company info
Revision ID: 57c0dc06cd25
Revises: 23a4e2cc5467
Create Date: 2025-10-27 20:25:27.686455
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
import sqlmodel
# revision identifiers, used by Alembic.
revision: str = '57c0dc06cd25'
down_revision: Union[str, None] = '23a4e2cc5467'
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
"""Upgrade schema."""
# ### commands auto generated by Alembic - please adjust! ###
op.create_table('hubspot_company_data',
sa.Column('id', sa.Uuid(), nullable=False),
sa.Column('company_id', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
sa.Column('company_name', sqlmodel.sql.sqltypes.AutoString(), nullable=True),
sa.Column('group_id', sqlmodel.sql.sqltypes.AutoString(), nullable=True),
sa.Column(
'created_at',
sa.DateTime(timezone=True),
server_default=sa.text('(CURRENT_TIMESTAMP AT TIME ZONE \'UTC\')'),
nullable=False,
),
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=True),
sa.PrimaryKeyConstraint('id')
)
op.create_index(op.f('ix_hubspot_company_data_company_id'), 'hubspot_company_data', ['company_id'], unique=False)
# ### end Alembic commands ###
def downgrade() -> None:
"""Downgrade schema."""
# ### commands auto generated by Alembic - please adjust! ###
op.drop_index(op.f('ix_hubspot_company_data_company_id'), table_name='hubspot_company_data')
op.drop_table('hubspot_company_data')
# ### end Alembic commands ###

57
alembic/versions/66d2c9c325d6_auto_update.py
View file

@ -1,57 +0,0 @@
"""auto update
Revision ID: 66d2c9c325d6
Revises: 57c0dc06cd25
Create Date: 2025-10-28 11:58:28.356864
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import postgresql
# revision identifiers, used by Alembic.
revision: str = '66d2c9c325d6'
down_revision: Union[str, None] = '57c0dc06cd25'
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
"""Upgrade schema safely."""
# 1️⃣ Fill existing NULLs with current UTC time
op.execute("UPDATE hubspot_company_data SET updated_at = NOW() AT TIME ZONE 'utc' WHERE updated_at IS NULL;")
op.execute("UPDATE hubspot_deal_data SET updated_at = NOW() AT TIME ZONE 'utc' WHERE updated_at IS NULL;")
# 2️⃣ Now alter the column defaults and nullability
op.alter_column(
'hubspot_company_data',
'updated_at',
existing_type=sa.TIMESTAMP(timezone=True),
server_default=sa.text("NOW() AT TIME ZONE 'utc'"),
nullable=False,
)
op.alter_column(
'hubspot_deal_data',
'updated_at',
existing_type=sa.TIMESTAMP(timezone=True),
server_default=sa.text("NOW() AT TIME ZONE 'utc'"),
nullable=False,
)
def downgrade() -> None:
"""Downgrade schema."""
# ### commands auto generated by Alembic - please adjust! ###
op.alter_column('hubspot_deal_data', 'updated_at',
existing_type=postgresql.TIMESTAMP(timezone=True),
server_default=None,
nullable=True)
op.alter_column('hubspot_company_data', 'updated_at',
existing_type=postgresql.TIMESTAMP(timezone=True),
server_default=None,
nullable=True)
# ### end Alembic commands ###

46
alembic/versions/a6e4562797e4_add_new_uploaded_file_table.py
View file

@ -1,46 +0,0 @@
"""add new uploaded file table
Revision ID: a6e4562797e4
Revises:
Create Date: 2025-08-14 14:44:40.992608
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
import sqlmodel
# revision identifiers, used by Alembic.
revision: str = 'a6e4562797e4'
down_revision: Union[str, None] = None
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
"""Upgrade schema."""
# ### commands auto generated by Alembic - please adjust! ###
op.create_table('uploaded_files',
sa.Column('id', sa.Uuid(), nullable=False),
sa.Column('s3_json_uri', sqlmodel.sql.sqltypes.AutoString(), nullable=True),
sa.Column('s3_file_uri', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
sa.Column('doc_type', sa.Enum('QUIDOS_PRESITE_NOTE', 'CHARTED_SURVEYOR_REPORT', 'U_VALUE_CALCULATOR_REPORT', 'OVERWRITING_U_VALUE_DECLARATION_FORM', 'ECO_CONDITION_REPORT', 'WARM_HOMES_CONDITION_REPORT', 'ENERGY_PERFORMANCE_REPORT_WITH_DATA', 'ENERGY_PERFORMANCE_REPORT_SUMMARY_INFORMATION', 'LIG_XML', 'RDSAP_XML', 'FULLSAP_XML', name='reporttype'), nullable=False),
sa.Column('s3_file_upload_timestamp', sa.DateTime(), nullable=False),
sa.Column('s3_json_upload_timestamp', sa.DateTime(), nullable=True),
sa.Column('uprn', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
sa.PrimaryKeyConstraint('id')
)
op.create_index(op.f('ix_uploaded_files_s3_file_uri'), 'uploaded_files', ['s3_file_uri'], unique=False)
op.create_index(op.f('ix_uploaded_files_uprn'), 'uploaded_files', ['uprn'], unique=False)
# ### end Alembic commands ###
def downgrade() -> None:
"""Downgrade schema."""
# ### commands auto generated by Alembic - please adjust! ###
op.drop_index(op.f('ix_uploaded_files_uprn'), table_name='uploaded_files')
op.drop_index(op.f('ix_uploaded_files_s3_file_uri'), table_name='uploaded_files')
op.drop_table('uploaded_files')
# ### end Alembic commands ###

38
alembic/versions/a8cc4a5fccb6_json_uri_is_a_string.py
View file

@ -1,38 +0,0 @@
"""json_uri is a string
Revision ID: a8cc4a5fccb6
Revises: 29113d69989e
Create Date: 2025-08-19 12:35:59.456912
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import postgresql
# revision identifiers, used by Alembic.
revision: str = 'a8cc4a5fccb6'
down_revision: Union[str, None] = '29113d69989e'
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
"""Upgrade schema."""
# ### commands auto generated by Alembic - please adjust! ###
op.alter_column('uploaded_files', 's3_json_uri',
existing_type=postgresql.JSON(astext_type=sa.Text()),
type_=sa.Text(),
existing_nullable=True)
# ### end Alembic commands ###
def downgrade() -> None:
"""Downgrade schema."""
# ### commands auto generated by Alembic - please adjust! ###
op.alter_column('uploaded_files', 's3_json_uri',
existing_type=sa.Text(),
type_=postgresql.JSON(astext_type=sa.Text()),
existing_nullable=True)
# ### end Alembic commands ###

38
alembic/versions/ac8dba8cef50_added_more_report_type.py
View file

@ -1,38 +0,0 @@
"""added more report type
Revision ID: ac8dba8cef50
Revises: a8cc4a5fccb6
Create Date: 2025-09-23 10:14:54.461633
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision: str = 'ac8dba8cef50'
down_revision: Union[str, None] = 'a8cc4a5fccb6'
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
"""Upgrade schema."""
# ### commands auto generated by Alembic - please adjust! ###
op.alter_column('uploaded_files', 'id',
existing_type=sa.UUID(),
server_default=None,
existing_nullable=False)
# ### end Alembic commands ###
def downgrade() -> None:
"""Downgrade schema."""
# ### commands auto generated by Alembic - please adjust! ###
op.alter_column('uploaded_files', 'id',
existing_type=sa.UUID(),
server_default=sa.text('gen_random_uuid()'),
existing_nullable=False)
# ### end Alembic commands ###

33
alembic/versions/c8af22cece92_s3_add.py
View file

@ -1,33 +0,0 @@
"""s3 add
Revision ID: c8af22cece92
Revises: ed6aaa298de4
Create Date: 2025-11-07 15:00:32.917157
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
import sqlmodel
# revision identifiers, used by Alembic.
revision: str = 'c8af22cece92'
down_revision: Union[str, None] = 'ed6aaa298de4'
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
"""Upgrade schema."""
# ### commands auto generated by Alembic - please adjust! ###
op.add_column('hubspot_deal_data', sa.Column('major_condition_issue_evidence_s3_url', sqlmodel.sql.sqltypes.AutoString(), nullable=True))
# ### end Alembic commands ###
def downgrade() -> None:
"""Downgrade schema."""
# ### commands auto generated by Alembic - please adjust! ###
op.drop_column('hubspot_deal_data', 'major_condition_issue_evidence_s3_url')
# ### end Alembic commands ###

34
alembic/versions/e72e15f7e0c3_delete_raw_data.py
View file

@ -1,34 +0,0 @@
"""delete raw_data
Revision ID: e72e15f7e0c3
Revises: 2409147995c5
Create Date: 2025-10-27 15:31:20.870827
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import postgresql
# revision identifiers, used by Alembic.
revision: str = 'e72e15f7e0c3'
down_revision: Union[str, None] = '2409147995c5'
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
"""Upgrade schema."""
# ### commands auto generated by Alembic - please adjust! ###
op.create_index(op.f('ix_hubspot_data_deal_id'), 'hubspot_data', ['deal_id'], unique=False)
op.drop_column('hubspot_data', 'raw_data')
# ### end Alembic commands ###
def downgrade() -> None:
"""Downgrade schema."""
# ### commands auto generated by Alembic - please adjust! ###
op.add_column('hubspot_data', sa.Column('raw_data', postgresql.JSON(astext_type=sa.Text()), autoincrement=False, nullable=True))
op.drop_index(op.f('ix_hubspot_data_deal_id'), table_name='hubspot_data')
# ### end Alembic commands ###

32
alembic/versions/e8507a27795a_auto_generate_id.py
View file

@ -1,32 +0,0 @@
"""auto generate id
Revision ID: e8507a27795a
Revises: 1f1d3d560ccb
Create Date: 2025-08-14 17:20:03.632337
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision: str = 'e8507a27795a'
down_revision: Union[str, None] = '1f1d3d560ccb'
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
"""Upgrade schema."""
# ### commands auto generated by Alembic - please adjust! ###
pass
# ### end Alembic commands ###
def downgrade() -> None:
"""Downgrade schema."""
# ### commands auto generated by Alembic - please adjust! ###
pass
# ### end Alembic commands ###

35
alembic/versions/eccdd5f607c1_add_coorodiantion_and_design_status.py
View file

@ -1,35 +0,0 @@
"""add coorodiantion and design status
Revision ID: eccdd5f607c1
Revises: c8af22cece92
Create Date: 2026-02-19 17:07:14.085232
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
import sqlmodel
# revision identifiers, used by Alembic.
revision: str = 'eccdd5f607c1'
down_revision: Union[str, None] = 'c8af22cece92'
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
"""Upgrade schema."""
# ### commands auto generated by Alembic - please adjust! ###
op.add_column('hubspot_deal_data', sa.Column('coordination_status', sqlmodel.sql.sqltypes.AutoString(), nullable=True))
op.add_column('hubspot_deal_data', sa.Column('design_status', sqlmodel.sql.sqltypes.AutoString(), nullable=True))
# ### end Alembic commands ###
def downgrade() -> None:
"""Downgrade schema."""
# ### commands auto generated by Alembic - please adjust! ###
op.drop_column('hubspot_deal_data', 'design_status')
op.drop_column('hubspot_deal_data', 'coordination_status')
# ### end Alembic commands ###

34
alembic/versions/ed6aaa298de4_added_major_condition_issue_things.py
View file

@ -1,34 +0,0 @@
"""added major condition issue things
Revision ID: ed6aaa298de4
Revises: 66d2c9c325d6
Create Date: 2025-11-05 15:03:11.447367
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
import sqlmodel
# revision identifiers, used by Alembic.
revision: str = 'ed6aaa298de4'
down_revision: Union[str, None] = '66d2c9c325d6'
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
"""Upgrade schema."""
# ### commands auto generated by Alembic - please adjust! ###
op.add_column('hubspot_deal_data', sa.Column('major_condition_issue_description', sqlmodel.sql.sqltypes.AutoString(), nullable=True))
op.add_column('hubspot_deal_data', sa.Column('major_condition_issue_photos', sqlmodel.sql.sqltypes.AutoString(), nullable=True))
# ### end Alembic commands ###
def downgrade() -> None:
"""Downgrade schema."""
# ### commands auto generated by Alembic - please adjust! ###
op.drop_column('hubspot_deal_data', 'major_condition_issue_photos')
op.drop_column('hubspot_deal_data', 'major_condition_issue_description')
# ### end Alembic commands ###

15
deployment/database/provider.tf
View file

@ -1,15 +0,0 @@
terraform {
required_providers {
aws = {
source = "hashicorp/aws"
version = "~> 6.3.0"
}
}
backend "s3" {
= "survey-extractor-tf-state"
region = "eu-west-2"
key = "env:/dev/terraform.tfstate"
}
required_version = ">= 1.2.0"
}

21
deployment/lambda/extractor_and_loader/docker/.dockerignore
View file

@ -1,21 +0,0 @@
# Ignore junk and large files
*.pdf
*.csv
*.xml
*.parquet
*.ipynb
*.mp4
*.mov
*.jpg
*.png
*.zip
*.tar.gz
__pycache__/
*.pyc
*.pyo
*.pyd
build/
dist/
.etl_cache/
tests/
docs/

25
deployment/lambda/extractor_and_loader/docker/Dockerfile
View file

@ -1,25 +0,0 @@
FROM public.ecr.aws/lambda/python:3.12
# Install Poetry (you could pin a version if you like)
RUN curl -sSL https://install.python-poetry.org | python3 -
# Add Poetry to PATH
ENV PATH="/root/.local/bin:$PATH"
# Set working directory
WORKDIR /var/task
# Copy Poetry files first to leverage Docker layer caching
COPY pyproject.toml poetry.lock README.md ./
COPY etl/ etl/
# Install dependencies into /var/task
RUN poetry config virtualenvs.create false \
&& poetry install --only main --no-interaction --no-ansi
# Copy app code
COPY deployment/lambda/extractor_and_loader/docker/app.py ./
# Set Lambda handler
CMD ["app.handler"]

244
deployment/lambda/extractor_and_loader/docker/app.py
View file

@ -1,244 +0,0 @@
import os
import tempfile
import requests
import boto3
from urllib.parse import urlparse
from etl.fileReader.pdfReaderToText import pdfReaderToText
from etl.fileReader.sitenotes import (
SiteNotesExtractor,
WarmHomesConditionReport
)
from uuid import UUID
import json
from typing import Any
from etl.db.db import get_db_session, init_db
from typing import Union
import uuid
from datetime import datetime, timezone
from sqlmodel import select
from sqlalchemy import update
from etl.models.topLevel import uploaded_files
def update_uploaded_file_json_uri_by_query(
db_session,
file_id: Union[str, uuid.UUID],
json_uri: str,
):
"""
Query uploaded_files by id, update s3_json_uri and s3_json_upload_timestamp,
commit, refresh, and return the ORM object. Raises ValueError if not found.
"""
try:
file_id_norm = uuid.UUID(str(file_id))
except (ValueError, AttributeError, TypeError):
file_id_norm = file_id # leave as-is if not a UUID
obj = (
db_session
.query(uploaded_files)
.filter(uploaded_files.id == file_id_norm)
.first()
)
obj.s3_json_uri = json_uri
obj.s3_json_upload_timestamp = datetime.now(timezone.utc)
db_session.add(obj)
db_session.commit()
db_session.refresh(obj)
return obj
def serialize_model(model: Any):
"""Recursively convert Pydantic models/lists into plain dicts."""
if hasattr(model, "dict"):
return {k: serialize_model(v) for k, v in model.dict().items()}
elif isinstance(model, list):
return [serialize_model(item) for item in model]
else:
return model
def make_final_json(rooms_obj, heating_system_obj, occupant, access_and_elevations, bepoke_info):
# Convert to dict recursively
rooms_data = serialize_model(rooms_obj)
heating_data = serialize_model(heating_system_obj)
occupant_data = serialize_model(occupant)
access_and_elevations_data = serialize_model(access_and_elevations)
# Combine into one big JSON-ready dict
final_data = {
"rooms": rooms_data,
"heating_system": heating_data,
"occupant_info": occupant_data,
"access_and_elevations": access_and_elevations_data,
"bespoke_data": bepoke_info
}
# Convert to pretty JSON string
return final_data
def parse_s3_uri(uri: str):
"""
Parse an S3 URI or HTTPS S3 URL into bucket and key.
Supports formats:
- s3://bucket-name/path/to/file
- https://bucket-name.s3.region.amazonaws.com/path/to/file
"""
parsed = urlparse(uri)
if parsed.scheme == "s3":
# s3://bucket/key
bucket = parsed.netloc
key = parsed.path.lstrip("/")
elif parsed.scheme in ("http", "https"):
# https://bucket-name.s3.region.amazonaws.com/key
host_parts = parsed.netloc.split(".")
if len(host_parts) >= 3 and host_parts[1] == "s3":
bucket = host_parts[0]
else:
raise ValueError("Not a valid S3 HTTPS URL format")
key = parsed.path.lstrip("/")
else:
raise ValueError("Unsupported URI scheme")
return bucket, key
def download_private_s3_file(uri) -> str:
bucket_name, key = parse_s3_uri(uri)
"""
Download a private S3 file using hardcoded AWS credentials.
Saves it to /tmp and returns the local file path.
"""
# Hardcoded AWS credentials (quick testing only)
aws_access_key = "AKIAU5A36PPNJMZZ3KRW"
aws_secret_key = "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"
aws_region = "eu-west-2"
# Where to store the file locally
tmp_dir = tempfile.gettempdir()
filename = os.path.basename(key)
file_path = os.path.join(tmp_dir, filename)
# Create S3 client with hardcoded creds
s3 = boto3.client(
"s3",
aws_access_key_id=aws_access_key,
aws_secret_access_key=aws_secret_key,
region_name=aws_region
)
# Download file
s3.download_file(bucket_name, key, file_path)
return file_path
def upload_json_to_s3(json_obj, dest_uri: str) -> str:
"""
Upload a JSON-serializable object to S3 at the given s3:// or https S3 URL.
Returns the public-style HTTPS S3 URL (still private if bucket is private).
"""
bucket, pdf_key = parse_s3_uri(dest_uri)
base_folder = os.path.dirname(pdf_key) # e.g. ".../report"
# Build jsonified folder + timestamp filename
timestamp = datetime.utcnow().strftime("%Y%m%d_%H%M%S")
json_key = f"{base_folder}/jsonified/{timestamp}.json"
# Same region/creds you used for download
aws_access_key = "AKIAU5A36PPNJMZZ3KRW"
aws_secret_key = "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"
aws_region = "eu-west-2"
s3 = boto3.client(
"s3",
aws_access_key_id=aws_access_key,
aws_secret_access_key=aws_secret_key,
region_name=aws_region
)
body = json.dumps(json_obj, ensure_ascii=False, indent=2).encode("utf-8")
s3.put_object(
Bucket=bucket,
Key=json_key,
Body=body,
ContentType="application/json"
# Optional hardening:
# , ServerSideEncryption="AES256"
)
# Return an HTTPS-style S3 URL (matches your input style)
return f"https://{bucket}.s3.{aws_region}.amazonaws.com/{json_key}"
def get_file_uri(id):
with get_db_session() as session:
obj = (
session
.query(uploaded_files)
.filter(uploaded_files.id == id)
.first()
)
if obj is None:
raise RuntimeError(f"Failed to find uploaded_files record with id {id}")
return obj.s3_file_uri
def handler(event, context):
try:
print("trying to connect to db")
init_db()
print("connected to db")
for r in event.get("Records", []):
body = json.loads(r["body"])
id_ = body.get("id")
if not id_: # covers None or empty string
raise ValueError(f"❌ Missing 'id' in SQS body: {body}")
print(f"Retrieving file uri with id {id_}")
file_uri = get_file_uri(id_)
print(f"Retrieved file uri with {file_uri}")
print("Downloading file locally for extraction...")
local_path = download_private_s3_file(file_uri)
# Local development of file, please comment out for prod
# local_path = os.path.join(os.path.join(os.getcwd(), "../../../../../", "home/Downloads/works/67-Aylestone-Road-1 1.pdf"))
# local_path = os.path.join(os.path.join(os.getcwd(), "../../../../../", "home/Downloads/works/2-Wilford-Crescent-West.pdf"))
# local_path = os.path.join(os.path.join(os.getcwd(), "../../../../../", "home/Downloads/works/3-Carlinghow-court.pdf"))
# local_path = os.path.join(os.path.join(os.getcwd(), "../../../../../", "home/Downloads/works/26-Marden-Road.pdf"))
# local_path = os.path.join(os.path.join(os.getcwd(), "../../../../../", "home/Downloads/works/6E-plantagenet-street.pdf"))
print("Extracting file...")
reader = pdfReaderToText(local_path)
# obj2 = WarmHomesConditionReport(reader.text_list, debug=True)
obj = WarmHomesConditionReport(reader.text_list)
json_ = make_final_json(
obj.master_obj[0],
obj.master_obj[1],
obj.master_obj[2],
obj.master_obj[3],
{}
)
print("Extracted completed, made json")
print("uploading json to s3 bucket...")
json_uri = upload_json_to_s3(json_, file_uri)
print("Updating Database with json_uri")
with get_db_session() as session:
update_uploaded_file_json_uri_by_query(
session,
id_,
json_uri,
)
print("job completed successfully")
except Exception as e:
print(f"❌ Error: {e}")
return {
"statusCode": 500,
"body": str(e)
}

62
deployment/lambda/extractor_and_loader/docker/ecr.tf
View file

@ -1,62 +0,0 @@
# ECR repo
resource "aws_ecr_repository" "extractor_and_loader" {
name = "extractor_and_loader"
}
# ECR policy to allow Lambda access
resource "aws_ecr_repository_policy" "extractor_loader_ecr_access" {
repository = aws_ecr_repository.extractor_and_loader.name
policy = jsonencode({
Version = "2008-10-17",
Statement = [{
Sid = "AllowLambdaPull",
Effect = "Allow",
Principal = {
Service = "lambda.amazonaws.com"
},
Action = [
"ecr:GetDownloadUrlForLayer",
"ecr:BatchGetImage",
"ecr:BatchCheckLayerAvailability"
]
}]
})
}
# ECR lifecycle policy to delete tagged images older than 14 days
resource "aws_ecr_lifecycle_policy" "extractor_loader_lifecycle" {
repository = aws_ecr_repository.extractor_and_loader.name
policy = jsonencode({
"rules": [
{
"rulePriority": 2,
"description": "Expire images older than 14 days",
"selection": {
"tagStatus": "untagged",
"countType": "sinceImagePushed",
"countUnit": "days",
"countNumber": 1
},
"action": {
"type": "expire"
}
},
{
"rulePriority": 1,
"description": "Keep last 5 images",
"selection": {
"tagStatus": "tagged",
"tagPrefixList": ["feature"],
"countType": "imageCountMoreThan",
"countNumber": 5
},
"action": {
"type": "expire"
}
}
]
})
}

15
deployment/lambda/extractor_and_loader/docker/provider.tf
View file

@ -1,15 +0,0 @@
terraform {
required_providers {
aws = {
source = "hashicorp/aws"
version = "~> 6.3.0"
}
}
backend "s3" {
bucket = "survey-extractor-tf-state"
region = "eu-west-2"
key = "env:/dev/lambda/ecr/extractor_and_loader.tfstate"
}
required_version = ">= 1.2.0"
}

85
deployment/lambda/extractor_and_loader/extractor_and_loader_lambda.tf
View file

@ -1,85 +0,0 @@
# Reference existing IAM role
data "aws_iam_role" "lambda_exec_role" {
name = "lambda-exec-role"
}
# Reference existing ECR repository
data "aws_ecr_repository" "extractor_and_loader" {
name = "extractor_and_loader"
}
# SQS queue for extractor_and_loader
resource "aws_sqs_queue" "extractor_and_loader_queue" {
name = "extractor-loader-queue"
visibility_timeout_seconds = 1800 # 30 minutes (>= 300s and ~6x Lambda timeout)
}
# Custom IAM policy specific to lambda_example
resource "aws_iam_policy" "extractor_loader_policy" {
name = "extractor_loader_policy"
policy = jsonencode({
Version = "2012-10-17",
Statement = [
{
Effect = "Allow",
Action = [
"sqs:ReceiveMessage",
"sqs:DeleteMessage",
"sqs:GetQueueAttributes",
"sqs:GetQueueUrl",
"sqs:ChangeMessageVisibility"
],
Resource = aws_sqs_queue.extractor_and_loader_queue.arn
},
{
Effect = "Allow",
Action = [
"ecr:GetDownloadUrlForLayer",
"ecr:BatchGetImage",
"ecr:BatchCheckLayerAvailability"
],
Resource = data.aws_ecr_repository.extractor_and_loader.arn
},
{
Effect = "Allow",
Action = ["ecr:GetAuthorizationToken"],
Resource = "*"
}
]
})
}
resource "aws_iam_role_policy_attachment" "extractor_loader_policy_attach" {
role = data.aws_iam_role.lambda_exec_role.name
policy_arn = aws_iam_policy.extractor_loader_policy.arn
}
# Lambda function
resource "aws_lambda_function" "extractor_and_loader" {
function_name = "extractor-and-loader-lambda"
role = data.aws_iam_role.lambda_exec_role.arn
package_type = "Image"
image_uri = "${data.aws_ecr_repository.extractor_and_loader.repository_url}:${var.lambda_image_tag}"
# Increase timeout (max 900 sec / 15 min)
timeout = 300 # e.g. 5 minutes
# Increase memory (default 128 MB)
memory_size = 2048 # try 1024 or 2048 MB to start
environment {
variables = {
DATABASE_URL = "postgresql://postgres:makingwarmhomes@terraform-20250331175522503500000002.cdgzupxvdyp0.eu-west-2.rds.amazonaws.com:5432/surveyDB"
}
}
}
# SQS trigger
resource "aws_lambda_event_source_mapping" "extractor_and_loader_trigger" {
event_source_arn = aws_sqs_queue.extractor_and_loader_queue.arn
function_name = aws_lambda_function.extractor_and_loader.arn
batch_size = 1
}

15
deployment/lambda/extractor_and_loader/provider.tf
View file

@ -1,15 +0,0 @@
terraform {
required_providers {
aws = {
source = "hashicorp/aws"
version = "~> 6.3.0"
}
}
backend "s3" {
bucket = "survey-extractor-tf-state"
region = "eu-west-2"
key = "env:/dev/lambda/eachlambda/extractor_and_loader_lambda.tfstate"
}
required_version = ">= 1.2.0"
}

5
deployment/lambda/extractor_and_loader/vars.tf
View file

@ -1,5 +0,0 @@
variable "lambda_image_tag" {
description = "Docker image tag (e.g. GitHub SHA)"
type = string
default = "local-dev-latest"
}

26
deployment/lambda/lambda_example/docker/Dockerfile
View file

@ -1,26 +0,0 @@
# AWS Lambda python pacakge
FROM public.ecr.aws/lambda/python:3.12
# Install Poetry (you could pin a version if you like)
RUN curl -sSL https://install.python-poetry.org | python3 -
# Add Poetry to PATH
ENV PATH="/root/.local/bin:$PATH"
# Set working directory
WORKDIR /var/task
# Copy Poetry files first to leverage Docker layer caching
COPY pyproject.toml poetry.lock README.md ./
COPY etl/ etl/
# Install dependencies into /var/task
RUN poetry config virtualenvs.create false \
&& poetry install --only main --no-interaction --no-ansi
# Copy app code
COPY deployment/lambda/lambda_example/docker/app.py ./
# Set the CMD to your handler
CMD ["app.handler"]

9
deployment/lambda/lambda_example/docker/app.py
View file

@ -1,9 +0,0 @@
from etl.hubSpotClient.hubspotClient import HubSpotClient
def handler(event, context):
nums = [
]
hubspot = HubSpotClient()
for num in nums:
hubspot.delete_line_item(num)

61
deployment/lambda/lambda_example/docker/ecr.tf
View file

@ -1,61 +0,0 @@
# ECR repo for lambda_example
resource "aws_ecr_repository" "lambda_example" {
name = "lambda_example"
}
# ECR policy to allow Lambda access
resource "aws_ecr_repository_policy" "lambda_example_ecr_access" {
repository = aws_ecr_repository.lambda_example.name
policy = jsonencode({
Version = "2008-10-17",
Statement = [{
Sid = "AllowLambdaPull",
Effect = "Allow",
Principal = {
Service = "lambda.amazonaws.com"
},
Action = [
"ecr:GetDownloadUrlForLayer",
"ecr:BatchGetImage",
"ecr:BatchCheckLayerAvailability"
]
}]
})
}
# ECR lifecycle policy to delete tagged images older than 14 days
resource "aws_ecr_lifecycle_policy" "lambda_example_ecr_lifecycle" {
repository = aws_ecr_repository.lambda_example.name
policy = jsonencode({
"rules": [
{
"rulePriority": 2,
"description": "Expire images older than 14 days",
"selection": {
"tagStatus": "untagged",
"countType": "sinceImagePushed",
"countUnit": "days",
"countNumber": 1
},
"action": {
"type": "expire"
}
},
{
"rulePriority": 1,
"description": "Keep last 5 images",
"selection": {
"tagStatus": "tagged",
"tagPrefixList": ["feature"],
"countType": "imageCountMoreThan",
"countNumber": 5
},
"action": {
"type": "expire"
}
}
]
})
}

0
deployment/lambda/lambda_example/docker/main.tf
View file

3
deployment/lambda/lambda_example/docker/output.tf
View file

@ -1,3 +0,0 @@
output "ecr_repo_url" {
value = aws_ecr_repository.lambda_example.repository_url
}

15
deployment/lambda/lambda_example/docker/provider.tf
View file

@ -1,15 +0,0 @@
terraform {
required_providers {
aws = {
source = "hashicorp/aws"
version = "~> 6.3.0"
}
}
backend "s3" {
bucket = "survey-extractor-tf-state"
region = "eu-west-2"
key = "env:/dev/lambda/ecr/lambda_example_ecr.tfstate"
}
required_version = ">= 1.2.0"
}

73
deployment/lambda/lambda_example/lambda_example_and_config.tf
View file

@ -1,73 +0,0 @@
# Reference existing IAM role
data "aws_iam_role" "lambda_exec_role" {
name = "lambda-exec-role"
}
# Reference existing ECR repository
data "aws_ecr_repository" "lambda_example" {
name = "lambda_example"
}
# SQS queue for lambda_example
resource "aws_sqs_queue" "lambda_example_queue" {
name = "lambda-example-queue"
}
# Custom IAM policy specific to lambda_example
resource "aws_iam_policy" "lambda_example_policy" {
name = "lambda-example-policy"
policy = jsonencode({
Version = "2012-10-17",
Statement = [
{
Effect = "Allow",
Action = [
"sqs:ReceiveMessage",
"sqs:DeleteMessage",
"sqs:GetQueueAttributes",
"sqs:GetQueueUrl",
"sqs:ChangeMessageVisibility"
],
Resource = aws_sqs_queue.lambda_example_queue.arn
},
{
Effect = "Allow",
Action = [
"ecr:GetDownloadUrlForLayer",
"ecr:BatchGetImage",
"ecr:BatchCheckLayerAvailability"
],
Resource = data.aws_ecr_repository.lambda_example.arn
},
{
Effect = "Allow",
Action = ["ecr:GetAuthorizationToken"],
Resource = "*"
}
]
})
}
resource "aws_iam_role_policy_attachment" "lambda_example_policy_attach" {
role = data.aws_iam_role.lambda_exec_role.name
policy_arn = aws_iam_policy.lambda_example_policy.arn
}
# Lambda function
resource "aws_lambda_function" "lambda_example" {
function_name = "lambda-example"
role = data.aws_iam_role.lambda_exec_role.arn
package_type = "Image"
image_uri = "${data.aws_ecr_repository.lambda_example.repository_url}:${var.lambda_image_tag}"
timeout = 10
}
# SQS trigger
resource "aws_lambda_event_source_mapping" "lambda_example_trigger" {
event_source_arn = aws_sqs_queue.lambda_example_queue.arn
function_name = aws_lambda_function.lambda_example.arn
batch_size = 1
}

0
deployment/lambda/lambda_example/main.tf
View file

5
deployment/lambda/lambda_example/vars.tf
View file

@ -1,5 +0,0 @@
variable "lambda_image_tag" {
description = "Docker image tag (e.g. GitHub SHA)"
type = string
default = "local-dev-latest"
}

21
deployment/lambda/lambda_shared/lambda_shared_config.tf
View file

@ -1,21 +0,0 @@
# IAM role for both Lambdas (can be shared)
resource "aws_iam_role" "lambda_exec_role" {
name = "lambda-exec-role"
assume_role_policy = jsonencode({
Version = "2012-10-17",
Statement = [{
Effect = "Allow",
Principal = {
Service = "lambda.amazonaws.com"
},
Action = "sts:AssumeRole"
}]
})
}
resource "aws_iam_role_policy_attachment" "lambda_basic_execution" {
role = aws_iam_role.lambda_exec_role.name
policy_arn = "arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole"
}

0
deployment/lambda/lambda_shared/main.tf
View file

9
deployment/lambda/lambda_shared/output.tf
View file

@ -1,9 +0,0 @@
output "lambda_exec_role_arn" {
description = "The ARN of the IAM role used by the Lambda functions"
value = aws_iam_role.lambda_exec_role.arn
}
output "lambda_exec_role_name" {
description = "The ARN of the IAM role used by the Lambda functions"
value = aws_iam_role.lambda_exec_role.name
}

15
deployment/lambda/lambda_shared/provider.tf
View file

@ -1,15 +0,0 @@
terraform {
required_providers {
aws = {
source = "hashicorp/aws"
version = "~> 6.3.0"
}
}
backend "s3" {
bucket = "survey-extractor-tf-state"
region = "eu-west-2"
key = "env:/dev/lambda/lambda_share_configuration.tfstate"
}
required_version = ">= 1.2.0"
}

21
deployment/lambda/walthamforest_etl/docker/.dockerignore
View file

@ -1,21 +0,0 @@
# Ignore junk and large files
*.pdf
*.csv
*.xml
*.parquet
*.ipynb
*.mp4
*.mov
*.jpg
*.png
*.zip
*.tar.gz
__pycache__/
*.pyc
*.pyo
*.pyd
build/
dist/
.etl_cache/
tests/
docs/

25
deployment/lambda/walthamforest_etl/docker/Dockerfile
View file

@ -1,25 +0,0 @@
FROM public.ecr.aws/lambda/python:3.12
# Install Poetry (you could pin a version if you like)
RUN curl -sSL https://install.python-poetry.org | python3 -
# Add Poetry to PATH
ENV PATH="/root/.local/bin:$PATH"
# Set working directory
WORKDIR /var/task
# Copy Poetry files first to leverage Docker layer caching
COPY pyproject.toml poetry.lock README.md ./
COPY etl/ etl/
# Install dependencies into /var/task
RUN poetry config virtualenvs.create false \
&& poetry install --only main --no-interaction --no-ansi
# Copy app code
COPY deployment/lambda/extractor_and_loader/docker/app.py ./
# Set Lambda handler
CMD ["app.handler"]

380
deployment/lambda/walthamforest_etl/docker/app.py
View file

@ -1,380 +0,0 @@
import pandas as pd
import json
from pprint import pprint
import os
import copy
from collections import defaultdict
from typing import List, Dict, Any, Union, Optional
import boto3
from urllib.parse import urlparse
from decent_homes_pilot import decent_homes_calc
import uuid
from datetime import datetime, timezone, time, date
from decimal import Decimal
from sqlmodel import select
from sqlalchemy import update
from etl.models.topLevel import uploaded_files, ReportType
from etl.db.db import get_db_session, init_db
def process_complex(sheet_name, group_key="ADDRESS"):
df = pd.read_excel("../../../../../home/Downloads/data.xlsx", sheet_name=sheet_name)
element_cols = [
"ELEMENT GROUP", "ELEMENT CODE", "ELEMENT CODE DESCRIPTION",
"ATTRIBUTE CODE", "ATTRIBUTE CODE DESCRIPTION",
"ELEMENT DATE VALUE", "ELEMENT NUMERIC VALUE",
"ELEMENT TEXT VALUE", "QUANTITY",
"INSTALL DATE", "REMAINING LIFE", "ELEMENT COMMENTS"
]
property_cols = [
"PROP REF", "ADDRESS", "OWNERSHIP",
"PROP STATUS", "PROP TYPE", "PROP SUB TYPE"
]
# Prepare output
records = []
# Loop through unique values in group_key (ADDRESS or BLOCK_CODE)
for val in df[group_key].unique():
g = df[df[group_key] == val] # subset
property_info = g[property_cols].drop_duplicates().iloc[0].to_dict()
# build elements dict keyed by ELEMENT CODE DESCRIPTION
elements_dict = {}
for _, row in g[element_cols].drop_duplicates().iterrows():
key = row["ELEMENT CODE DESCRIPTION"] # could also use "ELEMENT CODE"
elements_dict[key] = row.to_dict()
records.append({
group_key: val,
"property_info": property_info,
"elements": elements_dict
})
return records
def process_simple(sheet_name):
df = pd.read_excel("../../../../../home/Downloads/data.xlsx", sheet_name=sheet_name)
records = []
for address in df["Address"].unique():
g = df[df["Address"] == address].drop_duplicates() # subset for that address
row = g.iloc[0] # take first row if multiple
# build dict of all columns except Address
elements_dict = row.drop(labels=["Address"]).to_dict()
records.append({
"ADDRESS": address,
"to_add": elements_dict
})
return records
def combine_records_by_address(
asset_records: List[Dict[str, Any]],
simple_records: List[Dict[str, Any]],
dest_key: str = "to_add",
unique_identifier="Address"
) -> List[Dict[str, Any]]:
"""
Merge process_house_asset_data() and process_simple() results by ADDRESS.
All columns from simple_records['to_add'] will be merged under dest_key.
"""
# Index inputs by ADDRESS
asset_by_addr = {r["ADDRESS"]: r for r in asset_records}
simple_by_addr = {r["ADDRESS"]: r for r in simple_records}
merged: List[Dict[str, Any]] = []
# Use union of addresses from both sources
all_addresses = set(asset_by_addr) | set(simple_by_addr)
for addr in sorted(all_addresses):
base = copy.deepcopy(asset_by_addr.get(addr, {"ADDRESS": addr}))
simple = simple_by_addr.get(addr)
if simple:
base[dest_key] = simple.get("to_add", {})
merged.append(base)
return merged
def combine_records_for_flats(assets: dict, simple: list) -> dict:
"""Attach BLOCK_INFO (from simple[0]) to each asset in assets."""
if not simple or not isinstance(simple[0], dict):
return assets # nothing to add
block_info = simple[0]
for record in assets:
# Make sure record is a dict
# record.update({"BLOCK_INFO": block_info})
for ele_desc in block_info["elements"]:
if ele_desc not in record["elements"]:
record["elements"].update({ele_desc:block_info["elements"][ele_desc]})
return assets
def _json_default(o):
# datetimes → ISO 8601 strings
if isinstance(o, (datetime, date, time)):
return o.isoformat()
# decimals → float (or str if you need exactness)
if isinstance(o, Decimal):
return float(o)
# sets → lists
if isinstance(o, set):
return list(o)
# numpy/pandas types (optional)
try:
import numpy as np
import pandas as pd
if isinstance(o, (np.integer,)):
return int(o)
if isinstance(o, (np.floating,)):
return float(o)
if isinstance(o, (np.ndarray,)):
return o.tolist()
if isinstance(o, (pd.Timestamp,)):
return o.isoformat()
except Exception:
pass
# last resort: string
return str(o)
def uprn_to_address():
df = pd.read_excel("../../../../../home/Downloads/data.xlsx", sheet_name="All Energy Breakdown ")
mapping = df.set_index('Address')['UPRN'].to_dict()
return mapping
def stories_to_address():
df = pd.read_excel("../../../../../home/Downloads/data.xlsx", sheet_name="All Energy Breakdown ")
mapping = df.set_index('Address')['Storeys'].to_dict()
return mapping
def parse_s3_uri(uri: str):
"""
Parse an S3 URI or HTTPS S3 URL into bucket and key.
Supports formats:
- s3://bucket-name/path/to/file
- https://bucket-name.s3.region.amazonaws.com/path/to/file
"""
parsed = urlparse(uri)
if parsed.scheme == "s3":
# s3://bucket/key
bucket = parsed.netloc
key = parsed.path.lstrip("/")
elif parsed.scheme in ("http", "https"):
# https://bucket-name.s3.region.amazonaws.com/key
host_parts = parsed.netloc.split(".")
if len(host_parts) >= 3 and host_parts[1] == "s3":
bucket = host_parts[0]
else:
raise ValueError("Not a valid S3 HTTPS URL format")
key = parsed.path.lstrip("/")
else:
raise ValueError("Unsupported URI scheme")
return bucket, key
def upload_json_to_s3(json_obj, dest_uri: str, location="decent_homes/raw_data") -> str:
"""
Upload a JSON-serializable object to S3 at the given s3:// or https S3 URL.
Returns the public-style HTTPS S3 URL (still private if bucket is private).
"""
bucket, pdf_key = parse_s3_uri(dest_uri)
base_folder = os.path.dirname(pdf_key) # e.g. ".../report"
# Build jsonified folder + timestamp filename
timestamp = datetime.utcnow().strftime("%Y%m%d_%H%M%S")
json_key = f"{base_folder}/{location}/jsonified/{timestamp}.json"
# Same region/creds you used for download
aws_access_key = "AKIAU5A36PPNJMZZ3KRW"
aws_secret_key = "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"
aws_region = "eu-west-2"
s3 = boto3.client(
"s3",
aws_access_key_id=aws_access_key,
aws_secret_access_key=aws_secret_key,
region_name=aws_region
)
body = json.dumps(json_obj, ensure_ascii=False, indent=2, default=_json_default).encode("utf-8")
s3.put_object(
Bucket=bucket,
Key=json_key,
Body=body,
ContentType="application/json"
# Optional hardening:
# , ServerSideEncryption="AES256"
)
# Return an HTTPS-style S3 URL (matches your input style)
return f"https://{bucket}.s3.{aws_region}.amazonaws.com/{json_key}"
def generate_file_uri(UPRN):
timestamp = datetime.utcnow().strftime("%Y%m%d_%H%M%S")
file_uri = f"https://retrofit-energy-assessments-dev.s3.eu-west-2.amazonaws.com/documents/{UPRN}/"
return file_uri
def create_or_update_uploaded_file_entry(
db_session,
uprn: str,
doc_type: ReportType,
json_uri: str,
s3_file_uri: str
):
"""
Create or update an entry in uploaded_files.
- If a record with the same (uprn, doc_type) exists, update it.
- Otherwise, insert a new record.
Commits, refreshes, and returns the ORM object.
"""
existing = (
db_session.query(uploaded_files)
.filter(uploaded_files.uprn == str(uprn), uploaded_files.doc_type == doc_type)
.one_or_none()
)
if existing:
# Update existing record
existing.s3_json_uri = json_uri
existing.s3_json_upload_timestamp = datetime.now(timezone.utc)
existing.s3_file_uri = s3_file_uri
obj = existing
else:
# Insert new record
obj = uploaded_files(
doc_type=doc_type,
s3_json_uri=json_uri,
s3_json_upload_timestamp=datetime.now(timezone.utc),
s3_file_uri=s3_file_uri,
uprn=str(uprn),
)
db_session.add(obj)
db_session.commit()
db_session.refresh(obj)
return obj
def handler(event, context):
uprn_mapping = uprn_to_address()
flats_to_stories = stories_to_address()
# read data for houses only
assets = process_complex("Houses Asset Data")
simple = process_simple("Houses")
houses = combine_records_by_address(assets, simple, dest_key="EPC_DATA")
for house in houses:
pseudo_name = house["ADDRESS"].split(",")[0]
if pseudo_name.lower() in (k.lower() for k in uprn_mapping.keys()):
house.update({"UPRN": uprn_mapping[pseudo_name.upper()]})
#upload to s3
for i,house in enumerate(houses):
uprn = house["UPRN"]
print(uprn)
json_uri = upload_json_to_s3(house, generate_file_uri(house["UPRN"]), location="decent_homes/raw_data")
# Save JSON locally
filename = f"{uprn}.json"
filepath = os.path.join("output", filename) # saves inside an "output" folder
os.makedirs("output", exist_ok=True) # make sure folder exists
with open(filepath, "w", encoding="utf-8") as f:
json.dump(house, f, indent=2, ensure_ascii=False, default=_json_default)
property_decent_home, decent_home_meta = decent_homes_calc(filepath)
json_uri_1 = upload_json_to_s3(property_decent_home, generate_file_uri(uprn), location="decent_homes/property_decent_home")
with get_db_session() as session:
create_or_update_uploaded_file_entry(
db_session=session,
uprn=uprn,
doc_type=ReportType.DECENT_HOMES_SUMMARY,
json_uri=json_uri_1,
s3_file_uri=json_uri,
)
json_uri_1 = upload_json_to_s3(decent_home_meta, generate_file_uri(uprn), location="decent_homes/decent_homes_meta")
with get_db_session() as session:
create_or_update_uploaded_file_entry(
db_session=session,
uprn=uprn,
doc_type=ReportType.DECENT_HOMES_PROPERTY_META,
json_uri=json_uri_1,
s3_file_uri=json_uri,
)
# read data for flats
assets = process_complex("Chingford Rd 236-256 Properties")
simple = process_complex("CHINGFORD ROAD 236-254 Asset Bl", "BLOCK_CODE")
flats = combine_records_for_flats(assets, simple)
for house in flats:
pseudo_name = house["ADDRESS"].split(",")[0]
if pseudo_name.lower() in (k.lower() for k in uprn_mapping.keys()):
print(uprn_mapping[pseudo_name.upper()])
house.update({"UPRN": uprn_mapping[pseudo_name.upper()]})
house["property_info"].update({"FLAT LEVEL": flats_to_stories[pseudo_name.upper()]})
for i,house in enumerate(flats):
uprn = house["UPRN"]
print(uprn)
json_uri = upload_json_to_s3(house, generate_file_uri(house["UPRN"]))
# Save JSON locally
filename = f"{house['UPRN']}.json"
filepath = os.path.join("output", filename) # saves inside an "output" folder
os.makedirs("output", exist_ok=True) # make sure folder exists
with open(filepath, "w", encoding="utf-8") as f:
json.dump(house, f, indent=2, ensure_ascii=False, default=_json_default)
property_decent_home, decent_home_meta = decent_homes_calc(filepath)
json_uri_1 = upload_json_to_s3(property_decent_home, generate_file_uri(uprn), location="decent_homes/property_decent_home")
with get_db_session() as session:
create_or_update_uploaded_file_entry(
db_session=session,
uprn=uprn,
doc_type=ReportType.DECENT_HOMES_SUMMARY,
json_uri=json_uri_1,
s3_file_uri=json_uri,
)
json_uri_1 = upload_json_to_s3(decent_home_meta, generate_file_uri(uprn), location="decent_homes/decent_homes_meta")
with get_db_session() as session:
create_or_update_uploaded_file_entry(
db_session=session,
uprn=uprn,
doc_type=ReportType.DECENT_HOMES_PROPERTY_META,
json_uri=json_uri_1,
s3_file_uri=json_uri,
)
# Keep track of saved file path
# To Do:
# [Jun-te] Spec of quesation that we have for waltham forest
# [Khalim] A document that has our mapping and our understanding of our data

811
deployment/lambda/walthamforest_etl/docker/decent_homes_pilot.py
View file

@ -1,811 +0,0 @@
import json
import os
import pandas as pd
from datetime import datetime
from docutils.nodes import table
def years_between(d1, d2):
# precise year difference (accounts for months/days)
return (d1.year - d2.year) - ((d1.month, d1.day) < (d2.month, d2.day))
def get_element(elements, label):
"""Safely get an element dict by display label (your JSON keys)."""
return elements.get(label)
def append_result(decent_homes_meta, criteria, variable, sub_variable, result, install_date=None, expiry_date=None):
decent_homes_meta.append({
"criteria": criteria,
"variable": variable,
"sub_variable": sub_variable,
"result": result,
"hhsrs_rank": None,
"hhsrs_score": None,
"install_date": install_date,
"expiry_date": expiry_date,
})
def decent_homes_calc(one_property):
# Read in static json, which is transformed by Jun-te's script
folder = "../../../../../home/Downloads/"
fn = one_property
# filenames = ["flat 1.json", "house 1.json"]
houses_waltham_forest_data = pd.read_excel(
os.path.join(folder, "data.xlsx"),
sheet_name="Houses Asset Data"
)
flats_waltham_forest_data = pd.read_excel(
os.path.join(folder, "data.xlsx"),
sheet_name="CHINGFORD ROAD 236-254 Asset Bl"
)
# Standardised variables which will form the enums in the db
HHSRS_VARIABLES = [
"damp_and_mould_growth",
"excess_cold",
"excess_heat",
"asbestos_and_mm_fibres",
"biocides",
"carbon_monoxide_and_fuel_combustion_products",
"lead",
"radiation",
"uncombusted_fuel_gas",
"volatile_organic_compounds",
"crowding_and_space",
"entry_by_intruders",
"lighting",
"noise",
"domestic_hygiene_pests_and_refuse",
"food_safety",
"personal_hygiene_sanitation_and_drainage",
"water_supply",
"falls_associated_with_baths",
"falls_on_level_surfaces",
"falls_on_stairs_and_steps",
"falls_between_levels",
"electrical_hazards",
"fire",
"flames_hot_surfaces_and_materials",
"collision_and_entrapment",
"explosions",
"ergonomics",
"structural_collapse_and_falling_elements"
]
ELEMENT_CODE_TO_DESCRIPTION = {
# One-to-one
"HHSRSDAMP": "damp_and_mould_growth",
"HHSRSCOLD": "excess_cold",
"HHSRSHEAT": "excess_heat",
"HHSRSASB": "asbestos_and_mm_fibres",
"HHSRSBIOC": "biocides",
"HHSRSLEAD": "lead",
"HHSRSRADIA": "radiation",
"HHSRSFUEL": "uncombusted_fuel_gas",
"HHSRSORGAN": "volatile_organic_compounds",
"HHSRSCROWD": "crowding_and_space",
"HHSRSENTRY": "entry_by_intruders",
"HHSRSLIGHT": "lighting",
"HHSRSNOISE": "noise",
"HHSRSDOMES": "domestic_hygiene_pests_and_refuse",
"HHSRSFOOD": "food_safety",
"HHSRSPERS": "personal_hygiene_sanitation_and_drainage",
"HHSRSWATER": "water_supply",
"HHSRSFBATH": "falls_associated_with_baths",
"HHSRSFLEVE": "falls_on_level_surfaces",
"HHSRSFSTAI": "falls_on_stairs_and_steps",
"HHSRSFBETW": "falls_between_levels",
"HHSRSELEC": "electrical_hazards",
"HHSRSFIRE": "fire",
"HHSRSFLAME": "flames_hot_surfaces_and_materials",
"HHSRSEXPLO": "explosions",
"HHSRSPOSI": "ergonomics",
"HHSRSSTRUC": "structural_collapse_and_falling_elements",
# One-to-many expansions
"HHSRSCO": "carbon_monoxide",
"HHSRSSO2": "sulphur_dioxide_and_smoke",
"HHSRSNO2": "nitrogen_dioxide",
"HHSRSENTRP": "collision_and_entrapment",
"HHSRSCLOW": "collision_hazards_and_low_headroom",
}
CRITERION_B_VARIABLES = [
"external_walls_structure", "lintels", "brickwork_spalling", "wall_finish", "roof_structure", "roof_finish",
"chimneys", "windows", "external_doors", "kitchens", "bathrooms", "central_heating_boiler",
"central_heating_distribution_system", "heating_other", "electrical_systems",
]
CRITERION_C_VARIABLES = [
"kitchen_less_than_20_years_old", "kitchen_adequate_space_and_layout", "bathroom_less_than_30_years_old",
"bathroom_wc_appropriately_located", "adequate_external_noise_insulation", "adequate_common_entrance_areas",
]
# Criterion C explicit age limits (different from component lifespans used elsewhere)
CRITERION_C_AGE_LIMITS = {
"kitchen_years_max": 20,
"bathroom_years_max": 30,
}
# Field labels as they appear in your JSON (based on your code)
LABEL_KITCHEN = "Adequacy of Kitchen and Type in Property"
LABEL_BATHROOM = "Adequacy of Bathroom Location in Property"
LABEL_NOISE = "Adequacy of Noise Insulation in Property"
LABEL_COMMON_CIRC = "Circulation Space in Common Area" # flats only
STANDARD_HHSRS_MAPPING = {
"pass": ["TYPRISK"],
"fail": ["MODRISK","SLIGHTRISK"],
"no_data": ["TOBEASSESS"],
}
# Criterion A - mapping of HHSRS variables to Waltham forest element codes
HHSRS_MAPPING = {
"damp_and_mould_growth": {"HHSRSDAMP": STANDARD_HHSRS_MAPPING},
"excess_cold": {"HHSRSCOLD": STANDARD_HHSRS_MAPPING},
"excess_heat": {"HHSRSHEAT": STANDARD_HHSRS_MAPPING},
"asbestos_and_mm_fibres": {"HHSRSASB": STANDARD_HHSRS_MAPPING},
"biocides": {"HHSRSBIOC": STANDARD_HHSRS_MAPPING},
"carbon_monoxide_and_fuel_combustion_products": {
"HHSRSCO": STANDARD_HHSRS_MAPPING,
"HHSRSSO2": STANDARD_HHSRS_MAPPING,
"HHSRSNO2": STANDARD_HHSRS_MAPPING
},
"lead": {"HHSRSLEAD": STANDARD_HHSRS_MAPPING},
"radiation": {"HHSRSRADIA": STANDARD_HHSRS_MAPPING},
"uncombusted_fuel_gas": {"HHSRSFUEL": STANDARD_HHSRS_MAPPING},
"volatile_organic_compounds": {"HHSRSORGAN": STANDARD_HHSRS_MAPPING},
"crowding_and_space": {"HHSRSCROWD": STANDARD_HHSRS_MAPPING},
"entry_by_intruders": {"HHSRSENTRY": STANDARD_HHSRS_MAPPING},
"lighting": {"HHSRSLIGHT": STANDARD_HHSRS_MAPPING},
"noise": {"HHSRSNOISE": STANDARD_HHSRS_MAPPING},
"domestic_hygiene_pests_and_refuse": {"HHSRSDOMES": STANDARD_HHSRS_MAPPING},
"food_safety": {"HHSRSFOOD": STANDARD_HHSRS_MAPPING},
"personal_hygiene_sanitation_and_drainage": {"HHSRSPERS": STANDARD_HHSRS_MAPPING},
"water_supply": {"HHSRSWATER": STANDARD_HHSRS_MAPPING},
"falls_associated_with_baths": {"HHSRSFBATH": STANDARD_HHSRS_MAPPING},
"falls_on_level_surfaces": {"HHSRSFLEVE": STANDARD_HHSRS_MAPPING},
"falls_on_stairs_and_steps": {"HHSRSFSTAI": STANDARD_HHSRS_MAPPING},
"falls_between_levels": {"HHSRSFBETW": STANDARD_HHSRS_MAPPING},
"electrical_hazards": {"HHSRSELEC": STANDARD_HHSRS_MAPPING},
"fire": {"HHSRSFIRE": STANDARD_HHSRS_MAPPING},
"flames_hot_surfaces_and_materials": {"HHSRSFLAME": STANDARD_HHSRS_MAPPING},
"collision_and_entrapment": {"HHSRSENTRP": STANDARD_HHSRS_MAPPING, "HHSRSCLOW": STANDARD_HHSRS_MAPPING},
"explosions": {"HHSRSEXPLO": STANDARD_HHSRS_MAPPING},
"ergonomics": {"HHSRSPOSI": STANDARD_HHSRS_MAPPING},
"structural_collapse_and_falling_elements": {"HHSRSSTRUC": STANDARD_HHSRS_MAPPING}
}
# print(houses_waltham_forest_data[
# houses_waltham_forest_data["ELEMENT CODE"] == "INTBTHADEQ"
# ][["ATTRIBUTE CODE", "ATTRIBUTE CODE DESCRIPTION"]].drop_duplicates())
# print(flats_waltham_forest_data[
# flats_waltham_forest_data["ELEMENT CODE"] == "INTBTHADEQ"
# ][["ATTRIBUTE CODE", "ATTRIBUTE CODE DESCRIPTION"]].drop_duplicates())
# Criterion B
B_COMPONENT_LABELS = {
# Key components
"wall_structure": [
"Wall Structure in External Area",
],
"lintels": [
"Lintels in External Area",
],
"brickwork_spalling": [
"Wall Spalling in External Area",
],
"wall_finish": [
"Wall Finish 1 in External Area",
"Wall Finish 2 in External Area",
"External Decorations in External Area",
"Brickwork Pointing in External Area",
],
"roof_structure": [
"Roof Structure 1 in External Area",
"Roof Structure 2 in External Area",
"Roof Structure 3 in External Area",
"Garage Roof in External Area",
"Garage and Store Roofs in External Area",
"Store Roof in External Area",
"Fascia / Soffit / Bargeboard in External Area",
"Gutters in External Area",
"Downpipes in External Area",
"Internal Downpipes in External Area"
],
"roof_finish": [
"Roof Covering 1 in External Area",
"Roof Covering 2 in External Area",
"Roof Covering 3 in External Area",
],
"chimneys": [
"Chimneys in External Area",
],
"windows": [
"Windows in Property",
"Windows 1 in External Area",
"Windows 2 in External Area",
"Garage and Store Windows in External Area",
"Garage Windows in External Area",
"Store Windows in External Area",
],
"external_doors": [
"Type and Location of Front Door in Property",
"Front Door Fire Rating in Property",
"Patio and French Doors 1 in External Area",
"Back and Side Doors 1 in External Area",
"Back and Side Doors 2 in External Area",
"Garage and Store Doors in External Area",
"Garage Door in External Area",
"Store Door in External Area",
],
"central_heating_boiler": [
# "Heating Improvement Required in Property",
"Boiler Fuel in Property",
"Type of Water Heating in Property",
],
"heating_other": [
# "Heating Distribution System in Property"
"Boiler Fuel in Property",
"Type of Water Heating in Property",
],
"electrical_systems": [
"Electrics Required in Property",
],
# Other components
"kitchen": [
"Adequacy of Kitchen and Type in Property",
],
"bathroom": [
"Adequacy of Bathroom Location in Property",
],
"central_heating_distribution_system": [
"Heating Distribution System in Property",
],
}
KEY_COMPONENTS = {
"wall_structure", "lintels", "brickwork_spalling", "wall_finish",
"roof_structure", "roof_finish", "chimneys", "windows",
"external_doors", "central_heating_boiler", "heating_other",
"electrical_systems",
}
OTHER_COMPONENTS = {
"kitchen", "bathroom", "central_heating_distribution_system",
}
# Criterion C
COMPONENT_LIFESPANS = {
# Key components
"wall_structure": {
"house": 80, "flat_below_6_storeys": 80, "flat_above_6_storeys": 80
},
"lintels": {
"house": 60, "flat_below_6_storeys": 60, "flat_above_6_storeys": 60
},
"brickwork_spalling": {
"house": 30, "flat_below_6_storeys": 30, "flat_above_6_storeys": 30
},
"wall_finish": {
"house": 60, "flat_below_6_storeys": 60, "flat_above_6_storeys": 30
},
"roof_structure": {
"house": 50, "flat_below_6_storeys": 30, "flat_above_6_storeys": 30
},
"roof_finish": {
"house": 50, "flat_below_6_storeys": 30, "flat_above_6_storeys": 30
},
"chimneys": {
"house": 50, "flat_below_6_storeys": 50, "flat_above_6_storeys": None # N/A
},
"windows": {
"house": 40, "flat_below_6_storeys": 30, "flat_above_6_storeys": 30
},
"external_doors": {
"house": 40, "flat_below_6_storeys": 30, "flat_above_6_storeys": 30
},
"central_heating_boiler": {
"house": 15, "flat_below_6_storeys": 15, "flat_above_6_storeys": 15
},
"heating_other": {
"house": 30, "flat_below_6_storeys": 30, "flat_above_6_storeys": 30
},
"electrical_systems": {
"house": 30, "flat_below_6_storeys": 30, "flat_above_6_storeys": 30
},
# Other components
"kitchen": {
"house": 30, "flat_below_6_storeys": 30, "flat_above_6_storeys": 30
},
"bathroom": {
"house": 40, "flat_below_6_storeys": 40, "flat_above_6_storeys": 40
},
"central_heating_distribution_system": {
"house": 40, "flat_below_6_storeys": 40, "flat_above_6_storeys": 40
},
}
# Database design
# creation_date, uprn, variable, result (pass/fail/nodata), hhsrs_score (optional, numeric), hhsrs_rank (A-J),
# install_date (for components which expire, e.g. kitchen), remaining_life (for components which expire, e.g. kitchen),
# TODO: Add the criterion
decent_homes_meta = []
# Use to capture criterion A, B, C and D. Should be:
# {"uprn": int, "creation_date": datetime, "criterion_a": bool, "criterion_b": bool, "criterion_c": bool,
# "criterion_d": bool, "decent_homes": bool"}
property_decent_homes = []
with open(os.path.join(fn), "rb") as f:
data = json.load(f)
today = pd.Timestamp.today().normalize()
property_info = data["property_info"]
if property_info["PROP TYPE"] in ["HOU"]:
property_type = "house"
elif property_info["PROP TYPE"] == "FLA":
if property_info["FLAT LEVEL"] < 6:
property_type = "flat_below_6_storeys"
else:
property_type = "flat_above_6_storeys"
else:
raise NotImplementedError("Unknown property type")
# ---------------- Criterion A ----------------
# Critrion A: pass/fail
# If fail, why?
for hhsrs_variable, mapping in HHSRS_MAPPING.items():
element_code = list(mapping.keys())[0]
# Find the data in the JSON within data["elements"]
check_pass = []
for k, v in data["elements"].items():
if v["ELEMENT CODE"] == element_code:
# We check the attribute code
# Check if pass
if v["ATTRIBUTE CODE"] in mapping[element_code]["pass"]:
result = "pass"
elif v["ATTRIBUTE CODE"] in mapping[element_code]["fail"]:
result = "fail"
elif v["ATTRIBUTE CODE"] in mapping[element_code]["no_data"]:
result = "no_data"
else:
raise ValueError(f"Unknown attribute code: '{v[element_code]}")
check_pass.append(result)
append_result(
decent_homes_meta,
criteria="A",
variable=hhsrs_variable,
sub_variable=ELEMENT_CODE_TO_DESCRIPTION[element_code],
result=result,
install_date=None,
expiry_date=None,
)
# We check if we have a pass, fail or no_data
# if all([x == "pass" for x in check_pass]):
# hhsrs_result = "pass"
# elif any([x == "fail" for x in check_pass]):
# hhsrs_result = "fail"
# elif any([x == "no_data" for x in check_pass]):
# hhsrs_result = "no_data"
# else:
# raise NotImplementedError("Mixed results not implemented")
# ---------------- Criterion B ----------------
# Check each of the components
# ---------------- Criterion B ----------------
property_boiler = get_element(data["elements"], "Boiler Fuel in Property")
for component, labels in B_COMPONENT_LABELS.items():
for label in labels:
label_data = get_element(data["elements"], label)
# Handle no-data or not-applicable
if label_data["ATTRIBUTE CODE"] in ["UNKNOWN", "NONE", "UNKNOWNG", "UNKNOWNS", "UNKNOWNMAT"] and pd.isnull(label_data["INSTALL DATE"]):
continue
# Special skip conditions for heating
no_boiler_condition = (
property_boiler["ATTRIBUTE CODE"] in ["NONENOCH"]
and component == "central_heating_boiler"
)
other_heating_condition = (
label_data["ATTRIBUTE CODE"] in ["NONENOCH"]
and component == "heating_other"
)
if no_boiler_condition or other_heating_condition:
# append_result(
# decent_homes_meta,
# criteria="B",
# variable=component,
# sub_variable=label,
# result="pass",
# install_date=None,
# expiry_date=None,
# )
continue
# Normal case: evaluate install date + lifetime + remaining life
install_date = pd.to_datetime(label_data["INSTALL DATE"])
if pd.isnull(install_date):
append_result(
decent_homes_meta,
criteria="B",
variable=component,
sub_variable=label,
result="no_data",
install_date=str(install_date),
expiry_date=None,
)
continue
component_lifetime = COMPONENT_LIFESPANS[component][property_type]
is_old = years_between(today.to_pydatetime(), install_date.to_pydatetime()) > component_lifetime
if pd.isnull(label_data["REMAINING LIFE"]):
append_result(
decent_homes_meta,
criteria="B",
variable=component,
sub_variable=label,
result="no_data",
install_date=str(install_date),
expiry_date=None,
)
continue
has_failed = label_data["REMAINING LIFE"] < 0
expiry_date = today.to_pydatetime() + pd.DateOffset(years=label_data["REMAINING LIFE"])
component_result = "fail" if is_old and has_failed else "pass"
# Push into decent_homes_meta
append_result(
decent_homes_meta,
criteria="B",
variable=component,
sub_variable=label,
result=component_result,
install_date=str(install_date),
expiry_date=str(expiry_date),
)
# ---------------- Criterion C ----------------
# Guard: property type string already set earlier
is_flat = (property_info["PROP TYPE"] == "FLA")
# 1) Kitchen age ≤ 20 years
kitchen = get_element(data["elements"], LABEL_KITCHEN)
if kitchen:
kit_install_raw = kitchen["INSTALL DATE"]
kit_install = pd.to_datetime(kit_install_raw)
kit_age_years = years_between(today.to_pydatetime(), kit_install.to_pydatetime())
kitchen_age_result = "pass" if kit_age_years <= CRITERION_C_AGE_LIMITS["kitchen_years_max"] else "fail"
# For transparency, store next renewal as install + 20 years (criterion C perspective)
kit_next_due = today.to_pydatetime() + pd.DateOffset(years=kitchen["REMAINING LIFE"])
else:
raise NotImplementedError("Kitchen data missing - pls check")
append_result(
decent_homes_meta,
criteria="C",
variable="kitchen_less_than_20_years_old",
sub_variable="kitchen_less_than_20_years_old",
result=kitchen_age_result,
install_date=str(kit_install),
expiry_date=str(kit_next_due)
)
# 2) Kitchen adequate space/layout
# Prefer explicit codes if you have them, fall back to text in ATTRIBUTE CODE DESCRIPTION
if kitchen:
kit_attr_desc = kitchen["ATTRIBUTE CODE"]
if kit_attr_desc == "STDKITADQ":
kitchen_adequacy_result = "pass"
else:
raise NotImplementedError("No other observed codes yet")
else:
raise NotImplementedError("Kitchen data missing - pls check")
append_result(
decent_homes_meta,
criteria="C",
variable="kitchen_adequate_space_and_layout",
sub_variable="kitchen_adequate_space_and_layout",
result=kitchen_adequacy_result,
)
# 3) Bathroom age ≤ 30 years
bath = get_element(data["elements"], LABEL_BATHROOM)
if bath:
bth_install_raw = bath["INSTALL DATE"]
bth_install = pd.to_datetime(bth_install_raw)
bth_age_years = years_between(today.to_pydatetime(), bth_install.to_pydatetime())
bathroom_age_result = "pass" if bth_age_years <= CRITERION_C_AGE_LIMITS["bathroom_years_max"] else "fail"
bth_next_due = today.to_pydatetime() + pd.DateOffset(years=bath["REMAINING LIFE"])
else:
raise NotImplementedError("Bathroom data missing - pls check")
append_result(
decent_homes_meta,
criteria="C",
variable="bathroom_less_than_30_years_old",
sub_variable="bathroom_less_than_30_years_old",
result=bathroom_age_result,
install_date=str(bth_install),
expiry_date=bth_next_due
)
# 4) Bathroom/WC appropriately located
if bath:
bth_attr_code = bath["ATTRIBUTE CODE"]
if bth_attr_code in {"STDBTHADQ", "ADPBTHADQ"}:
bathroom_location_result = "pass"
elif bth_attr_code in {"STDBTHINAD"}:
bathroom_location_result = "fail"
else:
raise NotImplementedError(f"No other observed codes yet {bth_attr_code}")
else:
raise NotImplementedError("Bathroom data missing - pls check")
append_result(
decent_homes_meta,
criteria="C",
variable="bathroom_wc_appropriately_located",
sub_variable="bathroom_wc_appropriately_located",
result=bathroom_location_result
)
# 5) Adequate external noise insulation
noise = get_element(data["elements"], LABEL_NOISE)
if noise:
noise_code = noise["ATTRIBUTE CODE"]
if noise_code in {"ADEQUATE"}:
noise_result = "pass"
else:
raise NotImplementedError("No other observed codes yet")
else:
raise NotImplementedError("Noise insulation data missing - pls check")
append_result(
decent_homes_meta,
criteria="C",
variable="adequate_external_noise_insulation",
sub_variable="adequate_external_noise_insulation",
result=noise_result
)
# 6) Adequate common entrance areas (flats only)
if is_flat:
common = get_element(data["elements"], LABEL_COMMON_CIRC)
if common:
circ_desc = common["ATTRIBUTE CODE DESCRIPTION"]
if circ_desc in {"Adequate Circulation Space in Common Area"}:
common_areas_result = "pass"
else:
raise NotImplementedError(f"New description on common area {circ_desc}")
else:
common_areas_result = "no_data"
append_result(
decent_homes_meta=decent_homes_meta,
criteria="C",
variable="adequate_common_entrance_areas",
sub_variable="adequate_common_entrance_areas",
result=common_areas_result,
)
# ---------------- Criterion D ----------------
# heating system type
heating = get_element(data["elements"], "Heating Improvement Required in Property")
if heating:
heat_type_code = heating["ATTRIBUTE CODE"]
if heat_type_code in {"NOTAPPLIC"}:
heating_type_result = "pass"
elif heat_type_code in {"WETINSFULL"}:
heating_type_result = "fail"
else:
raise NotImplementedError("No other observed codes yet")
else:
raise NotImplementedError("Heating element missing in dataset")
append_result(
decent_homes_meta,
criteria="D",
variable="efficient_heating_system_type",
sub_variable="efficient_heating_system_type",
result=heating_type_result
)
# heating distribution
heating_dist = get_element(data["elements"], "Heating Distribution System in Property")
if heating_dist:
dist_code = heating_dist["ATTRIBUTE CODE"]
if dist_code == "UNKNOWN":
# For the observed case, there was no heating and wet heating needed to be installed in full so the value
# was unknown
heating_dist_result = "no_data"
elif dist_code in {"RADIATORS", "ELECWARMAR"}:
# Found one with heating distribution - check with Khalim if this is pass
heating_dist_result = "pass"
else:
print(f"heating_dist {heating_dist}")
print(f"dist-code {dist_code}")
raise NotImplementedError("No other observed codes yet")
else:
raise NotImplementedError("Heating distribution element missing in dataset")
append_result(
decent_homes_meta,
criteria="D",
variable="efficient_heating_distribution",
sub_variable="efficient_heating_distribution",
result=heating_dist_result
)
# insulation
loft = get_element(data["elements"], "Size in mm of Loft Insulation Thickness in Property")
wall = get_element(data["elements"], "Wall Insulation Improvement in External Area")
# To determine how much loft insulation is required
# Loft insulation check (example threshold: ≥ 270mm = pass)
if loft:
# We have a specific code, where further loft insulation is needed - It appears the heating type check has
# already been completed in this dataset and so we just need to check the code
loft_code = loft["ATTRIBUTE CODE"]
if loft_code == "LOFTINSRQD":
loft_result = "fail"
elif loft_code.isnumeric():
loft_result = "pass"
elif loft_code == "UNKNOWN":
loft_result = None
else:
raise NotImplementedError(f"Unknown loft insulation code - pls check {loft_code}")
else:
raise NotImplementedError("Loft insulation data missing - pls check")
if loft_result:
append_result(
decent_homes_meta,
criteria="D",
variable="loft_insulation_sufficient",
sub_variable="loft_insulation_sufficient",
result=loft_result
)
# Wall insulation check
if wall:
wall_code = wall["ATTRIBUTE CODE"]
if wall_code in {"NONE"}: # Means no insulation improvement required
wall_result = "pass"
elif wall_code in {"UNKNOWN"}:
wall_result = "no_data"
elif wall_code in {"SOLID"}:
wall_result = "fail"
else:
raise NotImplementedError(f"No other observed codes yet {wall_code}")
else:
raise NotImplementedError("Wall insulation data missing - pls check")
append_result(
decent_homes_meta,
criteria="D",
variable="wall_insulation_sufficient",
sub_variable="wall_insulation_sufficient",
result=wall_result
)
# ---------------- Criterion A overall ----------------
a_vars = set(HHSRS_MAPPING.keys())
latest_a_results = {r["variable"]: r["result"] for r in decent_homes_meta if r["variable"] in a_vars}
if any(v == "fail" for v in latest_a_results.values()):
criterion_a_result = "fail"
elif all(v == "pass" for v in latest_a_results.values()):
criterion_a_result = "pass"
else:
criterion_a_result = "no_data"
# ---------------- Criterion B overall ----------------
component_results = {}
for component in B_COMPONENT_LABELS.keys():
comp_rows = [r for r in decent_homes_meta if
r["criteria"] == "B" and r["variable"] == component and r["sub_variable"] is not None]
comp_sub_results = [r["result"] for r in comp_rows]
if not comp_sub_results: # no rows at all
comp_result = "no_data"
elif any(r == "fail" for r in comp_sub_results):
comp_result = "fail"
elif all(r == "pass" for r in comp_sub_results):
comp_result = "pass"
else:
comp_result = "no_data"
component_results[component] = comp_result
key_fails = [c for c, r in component_results.items() if c in KEY_COMPONENTS and r == "fail"]
other_fails = [c for c, r in component_results.items() if c in OTHER_COMPONENTS and r == "fail"]
if key_fails:
criterion_b_result = "fail"
elif len(other_fails) >= 2:
criterion_b_result = "fail"
elif any(r == "no_data" for r in component_results.values()):
criterion_b_result = "no_data"
else:
criterion_b_result = "pass"
# ---------------- Criterion C overall ----------------
criterion_c_vars = [
"kitchen_less_than_20_years_old",
"kitchen_adequate_space_and_layout",
"bathroom_less_than_30_years_old",
"bathroom_wc_appropriately_located",
"adequate_external_noise_insulation",
]
if is_flat:
criterion_c_vars.append("adequate_common_entrance_areas")
latest_c_results = {r["variable"]: r["result"] for r in decent_homes_meta if r["variable"] in criterion_c_vars}
count_fails = sum(1 for v in latest_c_results.values() if v == "fail")
# optionally count no_data too if you want strict interpretation
criterion_c_result = "fail" if count_fails >= 3 else "pass"
# ---------------- Criterion D overall ----------------
# Needs to have both efficient geating and distribution so all should pass
criterion_d_vars = [
"efficient_heating_system_type",
"efficient_heating_distribution",
"loft_insulation_sufficient",
"wall_insulation_sufficient",
]
latest_d_results = {r["variable"]: r["result"] for r in decent_homes_meta if r["variable"] in criterion_d_vars}
if any(v == "fail" for v in latest_d_results.values()):
criterion_d_result = "fail"
elif all(v == "pass" for v in latest_d_results.values()):
criterion_d_result = "pass"
else:
criterion_d_result = "no_data"
# ---------------- Append to property_decent_homes ----------------
check_pass = [
criterion_a_result,
criterion_b_result,
criterion_c_result,
criterion_d_result
]
decent_homes_result = "no_data"
if all(v == "pass" for v in check_pass):
decent_homes_result = "pass"
elif any(v == "fail" for v in check_pass):
decent_homes_result = "fail"
elif any(v=="no_data" for v in check_pass):
decent_homes_result = "no_data"
property_decent_homes.append({
"uprn": data.get("UPRN"), # TODO: Need UPRN
"creation_date": datetime.now().date().isoformat(),
"criterion_a": criterion_a_result,
"criterion_b": criterion_b_result,
"criterion_c": criterion_c_result,
"criterion_d": criterion_d_result,
"decent_homes": decent_homes_result,
})
return property_decent_homes[0], decent_homes_meta,

63
deployment/lambda/walthamforest_etl/docker/ecr.tf
View file

@ -1,63 +0,0 @@
# ECR repo
resource "aws_ecr_repository" "walthamforest_etl_adhoc_ecr" {
name = "walthamforest_etl_adhoc_ecr"
}
# ECR policy to allow Lambda access
resource "aws_ecr_repository_policy" "walthamforest_etl_adhoc_ecr_access" {
repository = aws_ecr_repository.walthamforest_etl_adhoc_ecr.name
policy = jsonencode({
Version = "2008-10-17",
Statement = [{
Sid = "AllowLambdaPull",
Effect = "Allow",
Principal = {
Service = "lambda.amazonaws.com"
},
Action = [
"ecr:GetDownloadUrlForLayer",
"ecr:BatchGetImage",
"ecr:BatchCheckLayerAvailability"
]
}]
})
}
# ECR lifecycle policy to delete tagged images older than 14 days
resource "aws_ecr_lifecycle_policy" "walthamforest_etl_adhoc_loader_lifecycle" {
repository = aws_ecr_repository.walthamforest_etl_adhoc_ecr.name
policy = jsonencode({
"rules": [
{
"rulePriority": 2,
"description": "Expire images older than 14 days",
"selection": {
"tagStatus": "untagged",
"countType": "sinceImagePushed",
"countUnit": "days",
"countNumber": 1
},
"action": {
"type": "expire"
}
},
{
"rulePriority": 1,
"description": "Keep last 5 images",
"selection": {
"tagStatus": "tagged",
"tagPrefixList": ["feature"],
"countType": "imageCountMoreThan",
"countNumber": 5
},
"action": {
"type": "expire"
}
}
]
})
}

0
deployment/lambda/walthamforest_etl/docker/main.tf
View file

15
deployment/lambda/walthamforest_etl/docker/provider.tf
View file

@ -1,15 +0,0 @@
terraform {
required_providers {
aws = {
source = "hashicorp/aws"
version = "~> 6.3.0"
}
}
backend "s3" {
bucket = "survey-extractor-tf-state"
region = "eu-west-2"
key = "env:/dev/lambda/ecr/walthamforest_etl.tfstate"
}
required_version = ">= 1.2.0"
}

0
deployment/lambda/walthamforest_etl/main.tf
View file

15
deployment/lambda/walthamforest_etl/provider.tf
View file

@ -1,15 +0,0 @@
terraform {
required_providers {
aws = {
source = "hashicorp/aws"
version = "~> 6.3.0"
}
}
backend "s3" {
bucket = "survey-extractor-tf-state"
region = "eu-west-2"
key = "env:/dev/lambda/eachlambda/walthamforest_etl_lambda.tfstate"
}
required_version = ">= 1.2.0"
}

5
deployment/lambda/walthamforest_etl/vars.tf
View file

@ -1,5 +0,0 @@
variable "lambda_image_tag" {
description = "Docker image tag (e.g. GitHub SHA)"
type = string
default = "local-dev-latest"
}

83
deployment/lambda/walthamforest_etl/walthamforest_etl_lambda.tf
View file

@ -1,83 +0,0 @@
# Reference existing IAM role
data "aws_iam_role" "lambda_exec_role" {
name = "lambda-exec-role"
}
# Reference existing ECR repository
data "aws_ecr_repository" "walthamforest_etl_adhoc_ecr" {
name = "walthamforest_etl_adhoc_ecr"
}
# SQS queue
resource "aws_sqs_queue" "walthamforest_etl_adhoc_queue" {
name = "walthamforest_etl_adhoc-queue"
visibility_timeout_seconds = 1800 # 30 minutes (>= 300s and ~6x Lambda timeout)
}
# Custom IAM policy specific to lambda_example
resource "aws_iam_policy" "walthamforest_etl_adhoc_policy" {
name = "walthamforest_adhoc_policy_lambda"
policy = jsonencode({
Version = "2012-10-17",
Statement = [
{
Effect = "Allow",
Action = [
"sqs:ReceiveMessage",
"sqs:DeleteMessage",
"sqs:GetQueueAttributes",
"sqs:GetQueueUrl",
"sqs:ChangeMessageVisibility"
],
Resource = aws_sqs_queue.walthamforest_etl_adhoc_queue.arn
},
{
Effect = "Allow",
Action = [
"ecr:GetDownloadUrlForLayer",
"ecr:BatchGetImage",
"ecr:BatchCheckLayerAvailability"
],
Resource = data.aws_ecr_repository.walthamforest_etl_adhoc_ecr.arn
},
{
Effect = "Allow",
Action = ["ecr:GetAuthorizationToken"],
Resource = "*"
}
]
})
}
resource "aws_iam_role_policy_attachment" "walthamforest_etl_adhoc_policy_attach" {
role = data.aws_iam_role.lambda_exec_role.name
policy_arn = aws_iam_policy.walthamforest_etl_adhoc_policy.arn
}
# Lambda function
resource "aws_lambda_function" "walthamforest_etl_adhoc" {
function_name = "walthamforest_etl_adhoc"
role = data.aws_iam_role.lambda_exec_role.arn
package_type = "Image"
image_uri = "${data.aws_ecr_repository.walthamforest_etl_adhoc_ecr.repository_url}:${var.lambda_image_tag}"
# Increase timeout (max 900 sec / 15 min)
# timeout = 300 # e.g. 5 minutes
# Increase memory (default 128 MB)
memory_size = 2048 # try 1024 or 2048 MB to start
# environment {
# variables = {
# DATABASE_URL = "postgresql://postgres:makingwarmhomes@terraform-20250331175522503500000002.cdgzupxvdyp0.eu-west-2.rds.amazonaws.com:5432/surveyDB"
# }
# }
}
# SQS trigger
resource "aws_lambda_event_source_mapping" "walthamforest_etl_adhoc_trigger" {
event_source_arn = aws_sqs_queue.walthamforest_etl_adhoc_queue.arn
function_name = aws_lambda_function.walthamforest_etl_adhoc.arn
batch_size = 1
}

0
deployment/database/main.tf → deployment/main.tf
View file

0
deployment/database/output.tf → deployment/output.tf
View file

5
deployment/lambda/lambda_example/provider.tf → deployment/provider.tf
View file

@ -2,13 +2,14 @@ terraform {
required_providers {
aws = {
source = "hashicorp/aws"
version = "~> 6.3.0"
version = "~> 4.16"
}
}
backend "s3" {
bucket = "survey-extractor-tf-state"
region = "eu-west-2"
key = "env:/dev/lambda/eachlambda/lambda_example.tfstate"
profile = "domna.dev" # /home/vscode/aws/credentials
key = "terraform.tfstate"
}
required_version = ">= 1.2.0"

2
deployment/database/variables.tf → deployment/variables.tf
View file

@ -14,4 +14,4 @@ variable allocated_storage {
description = "The allocated storage in gigabytes"
type = number
default = 20
}
}

0
deployment/database/vpc.tf → deployment/vpc.tf
View file

102
devcontainer.sh
View file

@ -1,102 +0,0 @@
#!/usr/bin/env bash
#
# devcontainer.sh — devcontainer helper for this repo
#
# Usage:
# ./devcontainer.sh <command>
#
# Commands:
# up build + start the devcontainer (idempotent)
# shell attach a bash shell; auto-ups if not running
# down stop the devcontainer
# rebuild remove + rebuild from scratch, no cache
#
# Examples:
# ./devcontainer.sh shell # one-shot: up if needed, then bash
# ./devcontainer.sh rebuild
set -euo pipefail
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)"
REPO_ROOT="${SCRIPT_DIR}"
CONFIG_PATH="${REPO_ROOT}/.devcontainer/devcontainer.json"
VALID_COMMANDS=(up shell down rebuild)
# --- helpers ---------------------------------------------------------------
usage() {
sed -n '3,15p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//'
exit "${1:-0}"
}
die() {
echo "error: $*" >&2
exit 1
}
in_list() {
local needle="$1"
shift
local item
for item in "$@"; do
[[ "${item}" == "${needle}" ]] && return 0
done
return 1
}
container_id() {
# Find the running container for this repo via devcontainer labels.
docker ps -q \
--filter "label=devcontainer.local_folder=${REPO_ROOT}" \
--filter "label=devcontainer.config_file=${CONFIG_PATH}"
}
# --- argument parsing ------------------------------------------------------
[[ $# -eq 1 ]] || usage 1
COMMAND="$1"
in_list "${COMMAND}" "${VALID_COMMANDS[@]}" \
|| die "invalid command '${COMMAND}' (expected: ${VALID_COMMANDS[*]})"
[[ -f "${CONFIG_PATH}" ]] || die "config not found: ${CONFIG_PATH}"
DC_ARGS=(--workspace-folder "${REPO_ROOT}")
# --- dispatch --------------------------------------------------------------
case "${COMMAND}" in
up)
echo ">> bringing up devcontainer"
devcontainer up "${DC_ARGS[@]}"
;;
shell)
# Auto-up if not already running. `devcontainer up` is idempotent —
# it reuses an existing container, so this is cheap on warm starts.
if [[ -z "$(container_id)" ]]; then
echo ">> devcontainer not running, bringing it up first"
devcontainer up "${DC_ARGS[@]}"
fi
echo ">> attaching shell"
devcontainer exec "${DC_ARGS[@]}" bash 2>/dev/null \
|| devcontainer exec "${DC_ARGS[@]}" sh
;;
down)
cid="$(container_id)"
if [[ -z "${cid}" ]]; then
echo ">> devcontainer not running, nothing to stop"
exit 0
fi
echo ">> stopping devcontainer"
docker stop "${cid}"
;;
rebuild)
echo ">> rebuilding devcontainer from scratch"
devcontainer up "${DC_ARGS[@]}" --remove-existing-container --build-no-cache
;;
esac

50
etl/MonthEndUploader.py
View file

@ -1,50 +0,0 @@
import os
os.environ["SHAREPOINT_CLIENT_ID"] = "0e28c4f9-3e77-4571-8d63-df1857f4266a"
os.environ["SHAREPOINT_CLIENT_SECRET"] = "2s48Q~t8.pI-~rbtQaSCFcYY97Z3LiMYhuo0GaOb"
os.environ["SHAREPOINT_TENANT_ID"] = "6f080c63-8a66-4bbc-9d72-b85d5df30555"
from etl.scraper.scraper import SharePointScraper, SharePointInstaller
from datetime import datetime
def upload_to_month_end_folder(file_name_on_sp, local_file_path, add_to_path):
sharepoint = SharePointScraper(SharePointInstaller.OSMOSIS_ACD)
sharepoint.get_folders_in_path("/")
parent_folder = "General/Junte Kim/month end"
today = datetime.today()
# Format as "Month YYYY"
formatted_date = today.strftime("%B %Y")
sharepoint.create_dir(formatted_date, parent_folder)
sharepoint_path = parent_folder + "/" + formatted_date
# Make day month year folder
formatted_date = today.strftime("%d-%m-%y")
sharepoint.create_dir(formatted_date, sharepoint_path)
sharepoint_path += "/" + formatted_date
# Make company folder
sharepoint.create_dir(add_to_path, sharepoint_path)
sharepoint_path += "/" + add_to_path
print("Uploading to sharepoint...")
sharepoint.upload_file(local_file_path, sharepoint_path, file_name_on_sp)
print(f"Finished upload of {local_file_path} to sharepoint. It's found under {sharepoint_path}/{file_name_on_sp}")
def upload_to_nick_folder(file_name_on_sp, local_file_path, add_to_path=None):
sharepoint = SharePointScraper(SharePointInstaller.OSMOSIS_ACD)
parent_folder = "General/Junte Kim/For Nick"
today = datetime.today()
formatted_date = today.strftime("%Y-%m-%d%H-%M-S")
sharepoint.create_dir(formatted_date, parent_folder)
sharepoint_path = parent_folder + "/" + formatted_date
if add_to_path:
sharepoint.create_dir(add_to_path, sharepoint_path)
sharepoint_path += "/" + add_to_path
print("Uploading to sharepoint...")
sharepoint.upload_file(local_file_path, sharepoint_path, file_name_on_sp)
print(f"Finished upload of {local_file_path} to sharepoint. It's found under {sharepoint_path}/{file_name_on_sp}")

14
etl/condition_report_etl.py
View file

@ -1,14 +0,0 @@
from pprint import pprint
from etl.db.db import get_db_session, init_db
from etl.surveyedData.surveryedData import surveyedDataProcessor
condition_report_file_path = "/workspaces/survey-extractor/etl/files/osmosis_condition_report.pdf"
sdp = surveyedDataProcessor("123 Fake Street", [condition_report_file_path])
pprint(sdp.condition_report.master_obj)
init_db()
with get_db_session() as db_session:
sdp.load_condition_report(db_session)
# TODO: add the ability to add document type, and sharepoint or s3 link so we can process access it again
# Terraform lambda set up to start this job from a s3 link

2
etl/daily_script.py
View file

@ -1,5 +1,5 @@
import os
from fileReader.pdfReaderToText import pdfReaderToText
from pdfReader.pdfReaderToText import pdfReaderToText
from etl.scraper.scraper import SharePointScraper, SharePointInstaller, WEEK_COMMENCING
from pprint import pprint, pformat
import logging

18
etl/db/db.py
View file

@ -1,7 +1,6 @@
from sqlmodel import SQLModel, create_engine, Session
from pydantic_settings import BaseSettings
from typing import Optional, List
from sqlalchemy.pool import QueuePool
class Settings(BaseSettings):
DATABASE_URL: Optional[str] = None # Default to None if not set
@ -10,20 +9,8 @@ class Settings(BaseSettings):
env_file = ".env" # Load from an optional .env file
settings = Settings()
# engine to the dabatase, currently set up to connect via settings. database
engine = (
create_engine(
settings.DATABASE_URL,
poolclass=QueuePool, # use standard connection pool
pool_pre_ping=True, # test connection before use
pool_recycle=300, # reconnect every 5 minutes
pool_size=5, # limit pool size for CI/CD or serverless
max_overflow=2, # allow brief overuse
connect_args={"sslmode": "require"}, # enforce SSL for cloud DBs
)
if settings.DATABASE_URL
else None
)
engine = create_engine(settings.DATABASE_URL) if settings.DATABASE_URL else None
def get_db_session():
if engine is None:
@ -32,5 +19,4 @@ def get_db_session():
def init_db():
if engine:
# Links SQLModel and metadata defined in sqlmodel instance
SQLModel.metadata.create_all(engine)

407
etl/db/hubSpotLoad.py
View file

@ -1,305 +1,136 @@
from etl.hubSpotClient.hubspot import HubSpotClient, DealStage
from etl.surveyPrice.surveyPrice import SurveyPrice
from etl.surveyedData.surveryedData import surveyedDataProcessor
from etl.scraper.scraper import SharePointScraper, SharePointInstaller
from etl.db.db import get_db_session, init_db
from etl.models.topLevel import HubspotDealData, HubspotCommpanyData
from sqlmodel import select
from etl.s3.s3_uploader import S3Uploader
import hashlib
import os
import pandas as pd
from etl.db.db import get_db_session, init_db
from etl.utils.utils import get_sharepoint_path
class HubspotTodb:
class HubspotTodb():
def __init__(self):
init_db()
self.s3 = S3Uploader()
def new_record_company(self, company_data):
"""Adds a new record to the hubspot_company_data table."""
with get_db_session() as session:
new_record = HubspotCommpanyData(
company_id=company_data.get("hs_object_id"),
company_name=company_data.get("name"),
)
session.add(new_record)
session.commit()
session.refresh(new_record)
return new_record
def new_record_to_hubspot_data(self, deal_data, company, listing, hubspot_client):
print("⚠️ Deprecated — use the new interface instead.")
return self.upsert_hubspot_deal(deal_data, company, listing, hubspot_client)
def find_all_deals_with_company_id(self, company_id):
"""Returns a list of deals for a given company_id."""
with get_db_session() as session:
return (
session.query(HubspotDealData)
.filter(HubspotDealData.company_id == company_id)
.all()
)
def find_deal_with_deal_id(self, deal_id):
with get_db_session() as session:
return (
session.query(HubspotDealData)
.filter(HubspotDealData.deal_id == deal_id)
.one()
)
def _sha256(self, file_path: str) -> str:
"""Compute SHA-256 checksum of a file."""
sha256 = hashlib.sha256()
with open(file_path, "rb") as f:
for chunk in iter(lambda: f.read(8192), b""):
sha256.update(chunk)
return sha256.hexdigest()
def update_deal(self, deal_in_db, hubspot_client):
"""
Checks if a deal needs updating and syncs it with HubSpot.
Also handles major_condition_issue_photos file upload to S3 with integrity check.
"""
def soft_assert(condition, message="Assertion Failed"):
if not condition:
print(f"⚠️ Soft Assert Failed: {message}")
return False
return True
print(f"🔍 Checking if deal needs updating (deal_id={deal_in_db.deal_id})")
hs_deal, hs_company_id, hs_listing = hubspot_client.get_deal_info_for_db(
deal_in_db.deal_id
)
# Soft compare key fields
checks = [
soft_assert(
deal_in_db.deal_id == hs_deal.get("hs_object_id"), "deal_id mismatch"
),
soft_assert(deal_in_db.company_id == hs_company_id, "company_id mismatch"),
soft_assert(
deal_in_db.landlord_property_id == hs_listing.get("owner_property_id"),
"landlord_property_id mismatch",
),
soft_assert(
deal_in_db.outcome == hs_deal.get("outcome"), "outcome mismatch"
),
soft_assert(
deal_in_db.dealstage == hs_deal.get("dealstage"), "dealstage mismatch"
),
soft_assert(
deal_in_db.dealname == hs_deal.get("dealname"), "dealname mismatch"
),
soft_assert(
deal_in_db.project_code == hs_deal.get("project_code"),
"project_code mismatch",
),
soft_assert(
deal_in_db.uprn == hs_listing.get("national_uprn"), "uprn mismatch"
),
soft_assert(
deal_in_db.outcome_notes == hs_deal.get("outcome_notes"),
"outcome_notes mismatch",
),
soft_assert(
deal_in_db.major_condition_issue_description
== hs_deal.get("major_condition_issue_description"),
"major condition description mismatch",
),
soft_assert(
deal_in_db.major_condition_issue_photos
== hs_deal.get("major_condition_issue_photos"),
"major condition issue photos mismatch",
),
soft_assert(
deal_in_db.coordination_status
== hs_deal.get("coordination_status__stage_1_"),
"coordination stage 1 status mismatch",
),
soft_assert(
deal_in_db.design_status == hs_deal.get("retrofit_design_status"),
"retrofit design mismatch",
),
]
# If discrepancies found, update from HubSpot
if not all(checks):
print(
f"❗ Discrepancies found for deal_id {deal_in_db.deal_id} — syncing with HubSpot."
)
self.upsert_hubspot_deal(hs_deal, hs_company_id, hs_listing, hubspot_client)
return False
# Handle photo upload if it exists but S3 URL is missing
if (
deal_in_db.major_condition_issue_photos
and not deal_in_db.major_condition_issue_evidence_s3_url
):
print(
f"🖼️ Found photo for deal_id {deal_in_db.deal_id} — uploading to S3..."
)
photo_url = hs_deal.get("major_condition_issue_photos")
if photo_url:
try:
# Download from HubSpot using fresh URL from hs_deal (not stale DB URL)
local_file = hubspot_client.download_file_from_url(photo_url)
# Upload to S3
bucket = "retrofit-data-dev"
s3_url = self.s3.upload_file(
local_file, bucket, prefix="hubspot/awaabs_law_evidence/"
)
# Download again to verify integrity
downloaded = self.s3.download_from_url(s3_url)
if self._sha256(local_file) == self._sha256(downloaded):
print("✅ SHA256 match verified — upload successful.")
else:
print("❌ SHA256 mismatch — integrity check failed.")
raise ValueError("File integrity check failed after S3 upload.")
# Update DB record with S3 URL
with get_db_session() as session:
db_record = session.get(HubspotDealData, deal_in_db.id)
db_record.major_condition_issue_evidence_s3_url = s3_url
session.add(db_record)
session.commit()
print(
f"✅ Updated DB with S3 URL for deal_id={deal_in_db.deal_id}"
)
return False
except Exception as e:
print(
f"⚠️ Failed to download/upload photo for deal_id {deal_in_db.deal_id}: {e}"
)
# Continue without the file — don't crash the entire update
else:
print(f"⚠️ Photo URL missing for deal_id {deal_in_db.deal_id}")
self.hubspot = HubSpotClient()
self.deals_in_hubspot = None
self.data_in_sharepoint = []
self.sp = SurveyPrice()
def get_all_deals(self):
self.deals_in_hubspot = self.sp.get_all_surveys_from_hubspot()
return self.deals_in_hubspot
def get_sharepoint_scraper(self, installer):
sp = None
if installer.upper() == "J & J CRUMP":
sp = SharePointScraper(SharePointInstaller.JJC)
elif installer.upper() == "SCIS":
sp = SharePointScraper(SharePointInstaller.SOUTH_COAST_INSULATION)
elif installer.upper() == "SGEC":
sp = SharePointScraper(SharePointInstaller.SGEC)
else:
print(f"✅ No update or upload required for deal_id {deal_in_db.deal_id}.")
sp = None
return True
return sp
def create_files_locally(self, sp, path, address):
address_paths = {}
file_names_to_download = {}
avoid = [".jpg",".mov", ".JPG", ".heic", ".HEIC", ".png", ".PNG", ".jpeg", ".JPEG", ".mov", ".MOV", ".mp4", ".MP4"]
def upsert_hubspot_deal(self, deal_data, company, listing, hubspot_client):
"""
Inserts or updates a deal record.
Also uploads photos if present and adds S3 URL.
"""
microsoft_graph_data = sp.get_folders_in_path(path)
for file in microsoft_graph_data['value']:
if 'file' in file:
if any(file["name"].endswith(ext) for ext in avoid):
continue
file_names_to_download.update({file["name"]: file['@microsoft.graph.downloadUrl']})
each_file = []
for file_name, url in file_names_to_download.items():
content = sp.get_file_content(url)
file_path = sp.create_temp_file(content, f"{address}/{file_name}")
each_file.append(file_path)
address_paths.update({address: each_file})
return address_paths
def string_to_installer(self, installer):
if installer.upper() == "J & J CRUMP":
return SharePointInstaller.JJC
elif installer.upper() == "SCIS":
return SharePointInstaller.SOUTH_COAST_INSULATION
elif installer.upper() == "SGEC":
return SharePointInstaller.SGEC
else:
return None
def work_out_invoice(self, row):
survey = self.gather_data_from_sharepoint_url(row)
installer = self.string_to_installer(row["HUBSPOT_INSTALLER"])
survey_pd = pd.DataFrame([self.sp.survey_to_pandas_format(surveyInfo=survey, installer=installer)])
hubspot_data = pd.DataFrame([row])
merged_df = self.sp.merge_hub_spot_and_survey_information_from_sharepoint_url(hubspot_data, survey_pd)
return self.sp.calculate_all_price(merged_df)
# self.sp.calculate_one_price_with_sharepoint_url(row, )
def gather_data_from_sharepoint_url(self, row):
sp = self.get_sharepoint_scraper(row["HUBSPOT_INSTALLER"])
path = get_sharepoint_path(row["HUBSPOT_SHAREPOINT_PATH"])
data_loc = self.create_files_locally(sp, path, row["HUBSPOT_DEAL_ADDRESS"])
for add, file_loc in data_loc.items():
sdp = surveyedDataProcessor(add, file_loc)
sdp.hubspot_deal_id = row["HUBSPOT_DEAL_ID"]
with get_db_session() as session:
self.load_one_pre_site_note(session, sdp, row)
return sdp
def gather_data_from_each_sharepoint(self):
self.get_all_deals()
for _, row in self.deals_in_hubspot.iterrows():
sp = self.get_sharepoint_scraper(row["HUBSPOT_INSTALLER"])
path = self.get_sharepoint_path(row["HUBSPOT_SHAREPOINT_PATH"])
data_loc = self.create_files_locally(sp, path, row["HUBSPOT_DEAL_ADDRESS"])
for add, file_loc in data_loc.items():
sdp = surveyedDataProcessor(add, file_loc)
sdp.hubspot_deal_id = row["HUBSPOT_DEAL_ID"]
self.data_in_sharepoint.append(sdp)
def load_all(self, fast=False):
if fast is False:
self.gather_data_from_each_sharepoint()
with get_db_session() as session:
print(deal_data)
deal_id = deal_data.get("hs_object_id")
self.load_all_pre_site_note(session)
session.commit()
statement = select(HubspotDealData).where(
HubspotDealData.deal_id == deal_id
)
existing = session.exec(statement).first()
def load_one_pre_site_note(self, db_session, surveyedData, hubspot_data):
df = hubspot_data
assessor = surveyedData.load_assessor_table(db_session)
if existing:
print(f"🔄 Updating existing deal (deal_id={deal_id})")
# Loads the pre site summary information
summary_info = surveyedData.load_pre_site_notes_summary_table(db_session)
for attr, value in {
"dealname": deal_data.get("dealname"),
"dealstage": deal_data.get("dealstage"),
"landlord_property_id": listing.get("owner_property_id"),
"uprn": listing.get("national_uprn"),
"outcome": deal_data.get("outcome"),
"outcome_notes": deal_data.get("outcome_notes"),
"project_code": deal_data.get("project_code"),
"company_id": company,
"major_condition_issue_description": deal_data.get(
"major_condition_issue_description"
),
"major_condition_issue_photos": deal_data.get(
"major_condition_issue_photos"
),
"major_condition_issue_description": deal_data.get(
"major_condition_issue_description"
),
"major_condition_issue_photos": deal_data.get(
"major_condition_issue_photos"
),
"coordination_status": deal_data.get(
"coordination_status__stage_1_"
),
"design_status": deal_data.get("retrofit_design_status"),
}.items():
setattr(existing, attr, value or getattr(existing, attr))
property_description = surveyedData.load_property_description(db_session)
# Upload if photo exists but S3 link missing
if (
existing.major_condition_issue_photos
and not existing.major_condition_issue_evidence_s3_url
):
# Fetch fresh URL from HubSpot instead of using potentially expired stored URL
fresh_deal = hubspot_client.from_deal_get_info(existing.deal_id)
photo_url = fresh_deal.get("major_condition_issue_photos")
# Creates the a final pre site note table that links all information
presitenote = surveyedData.create_pre_site_note_table(db_session, assessor, summary_info, property_description)
if photo_url:
try:
local_file = hubspot_client.download_file_from_url(
photo_url
)
s3_url = self.s3.upload_file(
local_file,
"retrofit-data-dev",
prefix="hubspot/awaabs_law_evidence/",
)
existing.major_condition_issue_evidence_s3_url = s3_url
except Exception as e:
print(
f"⚠️ Failed to download photo for deal_id {existing.deal_id}: {e}"
)
# Continue without the file — don't crash the update
else:
print(f"⚠️ Photo URL missing for deal_id {existing.deal_id}")
building_table = surveyedData.create_buildings_table(
db_session,
df["HUBSPOT_LANDLORD_ID"],
df["HUBSPOT_DOMNA_ID"],
)
documents = surveyedData.create_document_table_via_pre_site_note(db_session, presitenote, assessor, building_table)
session.add(existing)
session.commit()
session.refresh(existing)
return existing
def load_all_pre_site_note(self, db_session):
# Loads all pre
for surveyedData in self.data_in_sharepoint:
self.load_one_pre_site_note(surveyedData=surveyedData, db_session=db_session)
else:
print(f"🆕 Inserting new deal (deal_id={deal_id})")
new_record = HubspotDealData(
deal_id=deal_id,
dealname=deal_data.get("dealname"),
dealstage=deal_data.get("dealstage"),
landlord_property_id=listing.get("owner_property_id"),
uprn=listing.get("national_uprn"),
outcome=deal_data.get("outcome"),
outcome_notes=deal_data.get("outcome_notes"),
project_code=deal_data.get("project_code"),
company_id=company,
major_condition_issue_description=deal_data.get(
"major_condition_issue_description"
),
major_condition_issue_photos=deal_data.get(
"major_condition_issue_photos"
),
coordination_status=deal_data.get("coordination_status__stage_1_"),
design_status=deal_data.get("retrofit_design_status"),
)
# Handle upload at insert time
if new_record.major_condition_issue_photos:
try:
local_file = hubspot_client.download_file_from_url(
new_record.major_condition_issue_photos
)
s3_url = self.s3.upload_file(
local_file,
"retrofit-data-dev",
prefix="hubspot/awaabs_law_evidence/",
)
new_record.major_condition_issue_evidence_s3_url = s3_url
except Exception as e:
print(
f"⚠️ Failed to download photo for deal_id {new_record.deal_id}: {e}"
)
# Continue without the file — don't crash the insert
session.add(new_record)
session.commit()
session.refresh(new_record)
return new_record

2
etl/development.py
View file

@ -1,6 +1,6 @@
from etl.scraper.scraper import SharePointScraper, SharePointInstaller
from pprint import pformat
from etl.fileReader.pdfReaderToText import pdfReaderToText
from etl.pdfReader.pdfReaderToText import pdfReaderToText
from etl.surveyedData.surveryedData import surveyedDataProcessor
import pandas as pd

21
etl/dimitra_hubspot_notes_gather.py
View file

@ -6,7 +6,7 @@ from bs4 import BeautifulSoup
from openpyxl import Workbook
from openpyxl.styles import Font
from etl.scraper.scraper import SharePointScraper, SharePointInstaller, previous_monday
from etl.hubSpotClient.hubspotClient import HubSpotClient, DealStage
from etl.hubSpotClient.hubspot import HubSpotClient, DealStage
from collections import defaultdict
import time
# Auth credentials
@ -63,7 +63,6 @@ for pipeline in pipelines.results:
for stage in pipeline.stages:
if stage.label.upper().strip() not in [s.upper() for s in exclude_stage.get(pipeline_name, [])]:
for deals in hubspot.get_all_deals_from_stage_id(stage.id):
print(f"Scraping deal {deals['deal_id']}")
time.sleep(1)
deal_notes_by_week = {"Week 1": [], "Week 2": [], "Week 3": []}
notes = hubspot.get_notes_from_deals_id(deals["deal_id"])
@ -72,12 +71,7 @@ for pipeline in pipelines.results:
week_label = get_week_label(note["created_at"])
if not week_label:
continue
html_body = note.get('note')
if not html_body:
print(f"Skipping note with missing 'note' field: {note}")
continue
print(f"Debugging purposes html_body looks like {html_body}")
html_body = note['note']
soup = BeautifulSoup(html_body, "html.parser")
plain_text = soup.get_text(separator="\n")
deal_notes_by_week[week_label].append(plain_text)
@ -91,12 +85,11 @@ for pipeline in pipelines.results:
except:
owner_name = "Couldn't find owner information"
# Unique identifier to Domna Homes' hubspot
portal_id = 145275138
notes_data[pipeline_name].append({
"Deal Name": deal_name.upper(),
"Deal URL": f"https://app-eu1.hubspot.com/contacts/{portal_id}/record/0-3/{deals['deal_id']}/",
"Deal URL": f"https://app-eu1.hubspot.com/contacts/{portal_id}/record/0-3/{deals["deal_id"]}/",
"Deal Owner": owner_name,
"Deal Stage": stage.label.upper(),
"Value": deals["value"],
@ -157,7 +150,10 @@ for pipeline, deals in notes_data.items():
# Generate file name with next Monday’s date
formatted = previous_monday()
days_ahead = (7 - today.weekday()) % 7
days_ahead = 7 if days_ahead == 0 else days_ahead
next_monday = today + timedelta(days=days_ahead)
formatted = next_monday.strftime("%d-%m-%Y Monday")
file_name = f"{formatted} DEAL_NOTES_FROM_HUBSPOT.xlsx"
output_path = os.path.abspath(file_name)
wb.save(output_path)
@ -169,6 +165,3 @@ sharepoint_client.upload_file(
f"02. Sales and Marketing/02. Deal Notes from Hubspot/{formatted}",
file_name
)
print("hello world")

9
etl/epr_etl_example.py
View file

@ -1,9 +0,0 @@
from etl.surveyedData.surveryedData import surveyedDataProcessor
files = [
# "/tmp/sharepoint/Sandwell/SANDWELL-001/26 Willow close B64 6EG/Content (13).pdf",
"/tmp/sharepoint/Livewest/Livewest-001/12 Birch End/Summary Information 12 Birch End.pdf"
]
from sqlalchemy.dialects.postgresql import UUID
sdp = surveyedDataProcessor("fake address", files)

16
etl/fileReader/reportType.py
View file

@ -1,16 +0,0 @@
from enum import Enum
class ReportType(Enum):
QUIDOS_PRESITE_NOTE = "quidos_presite_note"
CHARTED_SURVEYOR_REPORT = "charted_surveyor_report"
U_VALUE_CALCULATOR_REPORT = "u_value_calculator_report"
OVERWRITING_U_VALUE_DECLARATION_FORM = "overwriting_u_value_declaration_form"
ECO_CONDITION_REPORT = "osmosis_condition_pas_2035_report"
WARM_HOMES_CONDITION_REPORT = "warm_homes_condition_pas_2035_report"
SMART_EPC_SITE_NOTE = "smart_epc_site_note"
ENERGY_PERFORMANCE_REPORT_WITH_DATA = "energy_performance_report_with_data"
ENERGY_PERFORMANCE_REPORT_SUMMARY_INFORMATION = "energy_performance_report_summary_information"
LIG_XML = "lodgement_xml_needed_for_lodgement_to_like_trademark"
RDSAP_XML = "reduce_xml_needed_to_generate_full_sap_xml"
FULLSAP_XML = "full_xml_needed_for_co_ordination"

1965
etl/fileReader/sitenotes.py
View file

File diff suppressed because it is too large Load diff

43
etl/fileReader/xmlReader.py
View file

@ -1,43 +0,0 @@
from etl.utils.logger import Logger
import logging
from xml.dom.minidom import parse
import os
from etl.fileReader.reportType import ReportType
class xmlReader():
def __init__(self, file_path):
self.source_path = file_path
self.logger = Logger(name='xmlReader', level=logging.INFO).get_logger()
self.xml_obj = None
self.type = None
self.get_xml_obj()
def get_xml_obj(self):
try:
if not os.path.exists(self.source_path):
self.logger.error(f"File not found: {self.source_path}")
return None
with open(self.source_path, 'r', encoding='utf-8') as file:
self.xml_obj = parse(file)
self.get_type()
return self.xml_obj
except Exception as e:
self.logger.error(f"Failed to parse XML file {self.source_path}: {e}")
self.xml_obj = None
return self.xml_obj
def get_type(self):
xmlHeaderName = self.xml_obj.documentElement.tagName
xmlHeaderName = xmlHeaderName.lower()
if xmlHeaderName == 'RdSap-Report'.lower():
self.type = ReportType.LIG_XML
elif xmlHeaderName == "SurveyRec".lower():
self.type = ReportType.RDSAP_XML
elif xmlHeaderName == "ImportExportRecord".lower():
self.type = ReportType.FULLSAP_XML
else:
pass
return self.type

BIN
etl/files/osmosis_condition_report.pdf
View file

Binary file not shown.

0
etl/fileReader/__init__.py → etl/hubSpotClient/__init__.py
View file

252
etl/hubSpotClient/hubspot.py Normal file
View file

@ -0,0 +1,252 @@
import hubspot
from enum import Enum
from hubspot.crm.deals import PublicObjectSearchRequest
from hubspot.crm.deals.models import SimplePublicObjectInput
from etl.hubSpotClient.types import SubmissionInfoFromDeal
import time
from pydantic import ValidationError
from etl.utils.logger import Logger
import logging
class DealStage(Enum):
SURVEYED_COMPLETE_NEEDS_SIGN_OFF = "1617223914"
SURVEYED_NO_ACCESS_NEED_SIGN_OFF = "1617223915"
CUSTOMER_CONTACTED = "888730834"
SURVEYED_COMPLETED_SIGNED_OFF = "1617223916"
NEEDS_ADDITIONAL_INFORMATION_FROM_ASSESSOR = "1887736000"
class HubSpotClient():
def __init__(self):
self.access_token = "pat-eu1-064f7f5c-a7d8-4d93-a9b2-b604da6164a6"
self.client = hubspot.Client.create(access_token=self.access_token)
self.logger = Logger(name='HubSpotClient', level=logging.INFO).get_logger()
def get_all_deals(self):
return self.client.crm.deals.get_all()
def get_owner_name_from_id(self, owner_id):
owner = self.client.crm.owners.owners_api.get_by_id(owner_id)
time.sleep(1)
first_name = owner.first_name or ""
last_name = owner.last_name or ""
return f"{first_name} {last_name}".strip()
def get_deal_name_by_id(self, deal_id):
try:
deal = self.client.crm.deals.basic_api.get_by_id(deal_id)
time.sleep(1)
return deal.properties.get("dealname", "No deal name")
except Exception as e:
return "Unknown Deal" # Fallback if the deal name is not found
def get_listings_from_deals_id(self, deals_id):
from hubspot.crm.objects import PublicObjectSearchRequest
found_notes = []
after = None
while True:
# Correct filter for notes associated with the given deal ID
search_request = PublicObjectSearchRequest(
filter_groups=[{
"filters": [{
"propertyName": "associations.deal", # Filter by association to the deal
"operator": "EQ",
"value": deals_id,
}]
}],
properties=["domna_property_id", "owner_property_id", 'national_uprn'], # Properties of the note you need
limit=200,
after=after,
)
# Call the search API
response = self.client.crm.objects.search_api.do_search(object_type="0-420", public_object_search_request=search_request)
time.sleep(1)
# Add the results to the found_notes list
found_notes.extend(response.results)
# Handle pagination if more results are available
if not response.paging or not response.paging.next:
break
after = response.paging.next.after
if found_notes:
return found_notes[0]
return None
def get_domna_and_landlord_id(self, deals_id):
data = self.get_listings_from_deals_id(deals_id)
return data.properties['domna_property_id'], data.properties['owner_property_id'], data.properties['national_uprn']
def get_notes_from_deals_id(self, deals_id):
from hubspot.crm.objects import PublicObjectSearchRequest
found_notes = []
after = None
while True:
# Correct filter for notes associated with the given deal ID
search_request = PublicObjectSearchRequest(
filter_groups=[{
"filters": [{
"propertyName": "associations.deal", # Filter by association to the deal
"operator": "EQ",
"value": deals_id,
}]
}],
properties=["hs_note_body", "hubspot_owner_id"], # Properties of the note you need
limit=200,
after=after,
)
# Call the search API
response = self.client.crm.objects.search_api.do_search(object_type="notes", public_object_search_request=search_request)
time.sleep(1)
# Add the results to the found_notes list
found_notes.extend(response.results)
# Handle pagination if more results are available
if not response.paging or not response.paging.next:
break
after = response.paging.next.after
all_notes = []
for note in found_notes:
# Extract note content and author information
note_body = note.properties.get("hs_note_body", "No content")
# Collect note details in a dictionary
all_notes.append({
"note_id": note.id,
"note": note_body,
"created_at": note.created_at.strftime("%Y-%m-%d %H:%M:%S"),
})
return all_notes
def get_all_deals_from_stage_id(self, stage_id):
found_deals = []
after = None
while True:
search_request = PublicObjectSearchRequest(
filter_groups=[{
"filters": [{
"propertyName": "dealstage",
"operator": "EQ",
"value": stage_id,
}]
}],
properties=[
"dealname",
"amount",
"hubspot_owner_id",
],
limit=200,
after=after,
)
response = self.client.crm.deals.search_api.do_search(search_request)
time.sleep(1)
found_deals.extend(response.results)
if not response.paging or not response.paging.next:
break
after = response.paging.next.after
all_deals = []
for deal in found_deals:
all_deals.append({
"deal_id": deal.id,
"value": deal.properties["amount"],
"deal_owner": deal.properties.get("hubspot_owner_id"),
})
return all_deals
def get_associations_for_deal(self, deal_id, to_object_type):
"""
Returns a list of associated object IDs of type `to_object_type`
(e.g. "contacts", "companies", "notes", etc.)
"""
assoc_resp = self.client.crm.deals.associations_api.get_all(
deal_id=deal_id,
to_object_type=to_object_type
)
return [assoc.id for assoc in assoc_resp.results]
def get_deals_from_deal_stage(self, deal_stage: DealStage):
found_deals = []
after = None
while True:
search_request = PublicObjectSearchRequest(
filter_groups=[{
"filters": [{
"propertyName": "dealstage",
"operator": "EQ",
"value": deal_stage.value,
}]
}],
properties=[
"dealname",
"number_of_wet_rooms_needing_ventilation",
"work_type",
"property_needs_trickle_vents",
"domna_survey_post_sap",
"existing_wall_insulation",
"installer",
"submission_folder",
],
limit=200,
after=after,
)
response = self.client.crm.deals.search_api.do_search(search_request)
found_deals.extend(response.results)
if not response.paging or not response.paging.next:
break
after = response.paging.next.after
all_deals = []
for deal in found_deals:
domna_id, landlord_id, uprn = self.get_domna_and_landlord_id(deal.id)
try:
all_deals.append(SubmissionInfoFromDeal(
deal_id= deal.properties["hs_object_id"],
deal_name=deal.properties["dealname"],
work_type=deal.properties["work_type"],
needs_trickle_ventilation=True if deal.properties.get("property_needs_trickle_vents", "NO").upper() == "YES" else False,
post_sap_score=int(deal.properties["domna_survey_post_sap"]),
existing_wall_insulation=deal.properties.get("existing_wall_insulation") if deal.properties.get("existing_wall_insulation") else "None",
no_of_wet_rooms=int(deal.properties["number_of_wet_rooms_needing_ventilation"]),
installer=deal.properties["installer"],
submission_folder_path = deal.properties["submission_folder"],
landlord_id = landlord_id,
domna_id = domna_id,
uprn = uprn,
))
except Exception as e:
deal_id = deal.properties['hs_object_id']
self.logger.info(f"Deal <{deal_id}> not valid")
self.move_deals_to_different_stage([deal_id], DealStage.NEEDS_ADDITIONAL_INFORMATION_FROM_ASSESSOR.value)
return all_deals
def print_all_pipeline_ids(self):
pipelines = self.client.crm.pipelines.pipelines_api.get_all(object_type="deals")
for pipeline in pipelines.results:
print(f"Pipeline: {pipeline.label}")
for stage in pipeline.stages:
print(f" - Label: {stage.label}")
print(f" ID: {stage.id}")
def move_deals_to_different_stage(self, list_of_deals_id, to_stage_id):
deal_properties = SimplePublicObjectInput(
properties={
"dealstage": to_stage_id
}
)
for deal_id in list_of_deals_id:
self.client.crm.deals.basic_api.update(
deal_id,
simple_public_object_input=deal_properties
)
self.logger.info(f"Deal {deal_id} moved to stage with ID {to_stage_id}.")

447
etl/hubSpotClient/hubspotClient.py
View file

@ -1,447 +0,0 @@
import hubspot
from enum import Enum
from etl.utils.logger import Logger
import logging
from hubspot.crm.associations import ApiException
import os
import requests
from hubspot.crm.objects import SimplePublicObjectInput
from hubspot.crm.associations.v4 import AssociationSpec
from hubspot.crm.associations import ApiException
class Companies(Enum):
ABRI = "237615001799"
SOUTHERN_HOUSING_GROUP = "109343619305"
LIVEWEST = "86205872354"
SURESERVE = "301745289413"
HOMEGROUP = "94946071794"
APPLE = "184769046716"
THE_GUINESS_PARTNERSHIP = "86970043613"
class DealStage(Enum):
SURVEYED_COMPLETE_NEEDS_SIGN_OFF = "1617223914"
SURVEYED_NO_ACCESS_NEED_SIGN_OFF = "1617223915"
CUSTOMER_CONTACTED = "888730834"
SURVEYED_COMPLETED_SIGNED_OFF = "1617223916"
FILES_MISSING_FROM_ASSESSOR = "1887736000"
class Pipeline(Enum):
OPERATIONS_SOCIAL_HOUSING = "1167582403"
class HubSpotClient:
def __init__(self):
self.access_token = "pat-eu1-064f7f5c-a7d8-4d93-a9b2-b604da6164a6"
self.client = hubspot.Client.create(access_token=self.access_token)
self.logger = Logger(name="HubSpotClient", level=logging.INFO).get_logger()
self.all_deals = None
def get_all_deals(self):
self.all_deals = self.client.crm.deals.get_all()
return self.all_deals
def get_deal_ids_by_pipeline(self, pipeline_id):
"""
Get all deal IDs associated with a given pipeline.
"""
if self.all_deals is None:
self.get_all_deals()
# Filter deals where properties['pipeline'] matches the given pipeline_id
filtered_deals = [
deal
for deal in self.all_deals
if deal.properties["pipeline"] == str(pipeline_id)
]
# Extract and return only the deal IDs
deal_ids = [deal.id for deal in filtered_deals]
return deal_ids
def get_deals_from_company(self, company_id: str) -> list[str]:
associations_api = self.client.crm.associations.v4.basic_api
deal_ids = []
after = None
while True:
response = associations_api.get_page(
object_type="companies",
object_id=company_id,
to_object_type="deals",
limit=100,
after=after,
)
deal_ids.extend(assoc.to_object_id for assoc in response.results)
if not response.paging or not response.paging.next:
break
after = response.paging.next.after
return deal_ids
def from_deal_get_associated_listing(self, deal_id: str):
"""
Get the associated listing information for a given deal.
Returns a dictionary of listing properties, or None if not found.
"""
associations_api = self.client.crm.associations.v4.basic_api
listings_api = (
self.client.crm.objects.basic_api
) # works for custom objects like "listing"
# Fetch associated listing(s)
response = associations_api.get_page(
object_type="deals",
object_id=deal_id,
to_object_type="0-420", # <-- use your exact custom object name slug here
limit=1,
)
if not response.results:
self.logger.info(f"No listing association found for deal {deal_id}")
return None
listing_id = response.results[0].to_object_id
self.logger.info(f"Associated listing ID for deal {deal_id}: {listing_id}")
# Fetch listing details (the "listing information")
listing = listings_api.get_by_id(
object_type="0-420", # again, must match your HubSpot object name
object_id=listing_id,
properties=[
"national_uprn",
"domna_property_id",
"owner_property_id",
],
)
listing_info = listing.properties
self.logger.info(f"Listing info for deal {deal_id}: {listing_info}")
return listing_info
def from_deal_get_info(self, deal_id):
deal = self.client.crm.deals.basic_api.get_by_id(
deal_id,
properties=[
"dealname",
"dealstage",
"pipeline",
"outcome", # outcome,
"outcome_notes", # outcome notes
"project_code",
"major_condition_issue_description",
"major_condition_issue_photos",
"coordination_status__stage_1_", # Coordiantion Status (Stage 1),
"retrofit_design_status", # Retrofit Design Status
],
)
return deal.properties
def from_deal_get_associated_company_id(self, deal_id: str):
"""
Get the associated company ID from a given deal ID.
Returns the associated company ID, or None if not found.
"""
try:
associations_api = self.client.crm.associations.v4.basic_api
# Fetch associations for this specific deal only
response = associations_api.get_page(
object_type="deals",
object_id=deal_id,
to_object_type="companies",
limit=1 # Expect only one associated company
)
if not response.results:
self.logger.info(f"No company association found for deal {deal_id}")
return None
company_id = response.results[0].to_object_id
self.logger.info(f"Associated company ID for deal {deal_id}: {company_id}")
return company_id
except ApiException as e:
self.logger.error(f"Error fetching associated company for deal {deal_id}: {e}")
return None
def get_deal_info_for_db(self, deal_id):
deal = self.from_deal_get_info(deal_id)
company = self.from_deal_get_associated_company_id(deal_id)
listing = self.from_deal_get_associated_listing(deal_id)
return deal, company, listing
def get_company_information(self, company_id):
company = self.client.crm.companies.basic_api.get_by_id(
company_id,
properties=[
"name",
],
)
company_info = company.properties
return company_info
def get_all_pipelines(self):
"""
Retrieve all pipelines for deals, returning a list of dicts with pipeline names and IDs.
"""
try:
pipelines_api = self.client.crm.pipelines.pipelines_api
response = pipelines_api.get_all(object_type="deals")
pipelines = [
{"name": pipeline.label, "id": pipeline.id}
for pipeline in response.results
]
self.logger.info(f"Retrieved {len(pipelines)} pipelines.")
return pipelines
except Exception as e:
self.logger.error(f"Error retrieving pipelines: {e}")
return []
def get_deal_stages(self, pipeline_id=None):
"""
Retrieve all deal stages for a given pipeline.
If no pipeline_id is provided, retrieves all stages for all pipelines.
Returns a list of dicts with pipeline name, stage name, and stage ID.
"""
try:
pipelines_api = self.client.crm.pipelines.pipelines_api
response = pipelines_api.get_all(object_type="deals")
all_stages = []
for pipeline in response.results:
# Skip other pipelines if a specific one is requested
if pipeline_id and pipeline.id != str(pipeline_id):
continue
stages = [
{
"pipeline_name": pipeline.label,
"pipeline_id": pipeline.id,
"stage_name": stage.label,
"stage_id": stage.id,
}
for stage in pipeline.stages
]
all_stages.extend(stages)
if not all_stages:
self.logger.info(
f"No deal stages found for pipeline {pipeline_id if pipeline_id else 'ALL'}"
)
else:
self.logger.info(f"Retrieved {len(all_stages)} deal stages.")
return all_stages
except Exception as e:
self.logger.error(f"Error retrieving deal stages: {e}")
return []
def download_file_from_url(self, download_url: str, save_path: str = None) -> str:
"""
Download a file from a HubSpot file URL (public or private), keeping its original file type.
Includes retry logic for transient failures and normalization of URL-encoded special characters.
"""
import mimetypes
import requests
import os
import time
import re
from urllib.parse import urlparse, urlunparse, unquote, quote
# Strip signature and expiration from CDN URLs to get a fresh one with response-content-disposition
# This is how HubSpot UI generates longer-lived URLs
if "cdnp1.hubspotusercontent" in download_url:
from urllib.parse import urlparse, urlunparse
parsed = urlparse(download_url)
# Keep only the path, strip all query params to force CDN to generate fresh signature
download_url = f"{parsed.scheme}://{parsed.netloc}{parsed.path}?response-content-disposition=attachment"
self.logger.info(
f"Requesting fresh CDN signature with response-content-disposition: {download_url}"
)
# Normalize URL-encoded special characters in the path (not query string)
# to avoid signature validation issues with special characters like %2c, %20
parsed = urlparse(download_url)
clean_path = quote(unquote(parsed.path), safe="/@:")
download_url = urlunparse(parsed._replace(path=clean_path))
max_attempts = 3
retry_delays = [1, 2, 4] # exponential backoff in seconds
last_exception = None
for attempt in range(max_attempts):
try:
headers = {}
# Add auth token for API endpoints (not direct CDN URLs)
if "hubspot.com/form-integrations" in download_url or "api-eu1.hubspot.com" in download_url:
headers["Authorization"] = f"Bearer {self.access_token}"
self.logger.info(
f"Downloading HubSpot file (attempt {attempt + 1}/{max_attempts}): {download_url}"
)
response = requests.get(
download_url,
headers=headers,
stream=True,
allow_redirects=True,
timeout=(10, 30),
)
response.raise_for_status()
# Try to infer filename from Content-Disposition header
content_disposition = response.headers.get("content-disposition")
if content_disposition and "filename=" in content_disposition:
filename = content_disposition.split("filename=")[1].strip('"')
else:
# fallback: extract from URL or content-type
filename = (
os.path.basename(download_url.split("?")[0]) or "hubspot_download"
)
if "." not in filename:
content_type = response.headers.get("content-type")
ext = (
mimetypes.guess_extension(content_type.split(";")[0])
if content_type
else None
)
if ext:
filename += ext
# Make sure save_path is valid
if save_path is None:
save_path = os.path.abspath(filename)
elif os.path.isdir(save_path):
save_path = os.path.join(save_path, filename)
else:
# if user passes a file path directly, leave it
save_path = os.path.abspath(save_path)
with open(save_path, "wb") as f:
for chunk in response.iter_content(chunk_size=8192):
f.write(chunk)
self.logger.info(f"File downloaded successfully → {save_path}")
return save_path
except requests.exceptions.HTTPError as e:
# Don't retry on 404 — file genuinely doesn't exist
if e.response is not None and e.response.status_code == 404:
self.logger.error(f"Failed to download file from HubSpot: {e}")
raise
last_exception = e
if attempt < max_attempts - 1:
delay = retry_delays[attempt]
self.logger.warning(
f"HTTP error (attempt {attempt + 1}/{max_attempts}): {e} — retrying in {delay}s"
)
time.sleep(delay)
except (
requests.exceptions.ConnectionError,
requests.exceptions.Timeout,
) as e:
last_exception = e
if attempt < max_attempts - 1:
delay = retry_delays[attempt]
self.logger.warning(
f"Connection/timeout error (attempt {attempt + 1}/{max_attempts}): {e} — retrying in {delay}s"
)
time.sleep(delay)
except requests.exceptions.RequestException as e:
# Other request errors (e.g., invalid URL) — don't retry
self.logger.error(f"Failed to download file from HubSpot: {e}")
raise
# If we got here, all retries failed
self.logger.error(
f"Failed to download file after {max_attempts} attempts: {last_exception}"
)
raise last_exception
def create_line_item_from_product(self, product_id: str, quantity: int = 1):
# Fetch product mapping
product = self.client.crm.products.basic_api.get_by_id(
product_id, properties=["name", "price", "hs_price"]
)
name = product.properties.get("name")
price = (
product.properties.get("price") or product.properties.get("hs_price") or "0"
)
# Build line item payload
line_item_input = SimplePublicObjectInput(
properties={
"hs_product_id": product_id,
"name": name,
"quantity": str(quantity),
"price": price,
"amount": str(float(price) * quantity),
"invoiced": "Outstanding",
}
)
# Create line item
line_item = self.client.crm.line_items.basic_api.create(line_item_input)
return line_item.id
def associate_line_item_to_deal(self, line_item_id: str, deal_id: str):
self.logger.info(f"Associating line item {line_item_id} → deal {deal_id}")
association_api = self.client.crm.associations.v4.basic_api
association_api.create(
"0-3", # to object type
deal_id, # to object id
"line_items", # from object type
line_item_id, # from object id
[
AssociationSpec(
association_category="HUBSPOT_DEFINED",
association_type_id=19, # line_item → deal
)
],
)
def add_product_line_item_to_deal(
self, deal_id: str, product_id: str, quantity: int = 1
):
# Step 1: Create the line item from product mapping
line_item_id = self.create_line_item_from_product(product_id, quantity)
# Step 2: Associate the created line item to the deal
self.associate_line_item_to_deal(line_item_id, deal_id)
return line_item_id
def delete_line_item(self, line_item_id: str):
"""
Delete (archive) a line item in HubSpot by its ID.
"""
try:
self.logger.info(f"Deleting line item {line_item_id}...")
self.client.crm.line_items.basic_api.archive(line_item_id)
self.logger.info(f"Line item {line_item_id} deleted successfully.")
return True
except ApiException as e:
self.logger.error(f"Failed to delete line item {line_item_id}: {e}")
return False

15
etl/hubSpotClient/hubspot_types.py
View file

@ -1,15 +0,0 @@
# from sqlmodel import Field, SQLModel
# from sqlalchemy import Column
# from sqlalchemy.dialects.postgresql import UUID
# import uuid
# from pydantic import Field
# class BaseModel(SQLModel):
# id: uuid.UUID = Field(
# default_factory=uuid.uuid4,
# sa_column=Column(UUID(as_uuid=True), primary_key=True)
# )

325
etl/hubSpotClient/scripts/README.md
View file

@ -1,325 +0,0 @@
# HubSpot Client Scripts - Onboarding Guide
## Overview
The scripts in this directory form a **3-stage ETL pipeline** for syncing HubSpot data (companies and deals) into the local database:
1. **Stage 0 (Seed)**: `hubspot_company.py` — Load company master records
2. **Stage 1 (Bulk Load)**: `hubspot_gather_all_deals.py` — Initial load of all deals for a company
3. **Stage 2 (Sync/Update)**: `hubspot_update_script.py` — Ongoing synchronization (runs on a schedule)
These scripts work together to keep your local database in sync with HubSpot while handling photo uploads to S3 and maintaining data integrity.
---
## Onboarding a New Client
Follow these steps in order when adding a new company/client:
### Step 1: Add the Company to the `Companies` Enum
Edit `../hubspotClient.py` and add your new company to the `Companies` enum class:
```python
class Companies(Enum):
ABRI = "237615001799"
SOUTHERN_HOUSING_GROUP = "109343619305"
LIVEWEST = "86205872354"
SURESERVE = "301745289413"
HOMEGROUP = "94946071794"
APPLE = "184769046716"
THE_GUINESS_PARTNERSHIP = "86970043613"
YOUR_NEW_COMPANY = "YOUR_HUBSPOT_COMPANY_ID" # ← Add here
```
**How to find your HubSpot Company ID:**
- Log into HubSpot
- Navigate to **Contacts** → **Companies**
- Click on the company name
- The URL will be: `https://app.hubspot.com/crm/xxx/objects/companies/COMPANY_ID` — copy that ID
### Step 2: Update Each Script to Include Your Company
After adding the enum, update the company lists in all three scripts:
#### `hubspot_company.py` (line ~6)
```python
companies = [
Companies.THE_GUINESS_PARTNERSHIP,
Companies.YOUR_NEW_COMPANY # ← Add here
]
```
#### `hubspot_gather_all_deals.py` (line ~7)
```python
valuable_companies = [
Companies.THE_GUINESS_PARTNERSHIP.value,
Companies.YOUR_NEW_COMPANY.value # ← Add here
]
```
#### `hubspot_update_script.py` (line ~12)
```python
companies = [
Companies.THE_GUINESS_PARTNERSHIP,
Companies.YOUR_NEW_COMPANY # ← Add here
]
```
### Step 3: Run `hubspot_company.py` (One-time setup)
This script seeds the company record into the `hubspot_company_data` table. Run it once:
```bash
python etl/hubSpotClient/scripts/hubspot_company.py
```
**What it does:**
- Connects to HubSpot and fetches company information (name, ID)
- Inserts the company record into the local database
**Output:** You'll see the company added to `hubspot_company_data` table.
### Step 3.5: Update Group ID in Database (Manual)
After the company record is created, you need to manually update the **group ID** for the new company. This is done via DBeaver or pgAdmin:
**Steps:**
1. Open DBeaver or pgAdmin and connect to the database
2. Navigate to the `hubspot_company_data` table
3. Find the row with your new company (search by `company_name` or `company_id`)
4. Edit the **`group_id`** column to the portfolio/group ID you want to track for this company
5. Save the changes
**Example Query** (if you prefer SQL):
```sql
UPDATE hubspot_company_data
SET group_id = 'YOUR_GROUP_ID'
WHERE company_id = 'YOUR_COMPANY_ID';
```
**What is Group ID?**
- The group ID identifies which portfolio/group in your system this company belongs to
- Each company can be associated with one group ID for tracking and organization
- This field is used for tracking and reporting across your survey data
### Step 4: Run `hubspot_gather_all_deals.py` (One-time bulk load)
This script performs the initial load of all deals for your company, filtered by the `OPERATIONS_SOCIAL_HOUSING` pipeline. Run it once per company:
```bash
python etl/hubSpotClient/scripts/hubspot_gather_all_deals.py
```
**What it does:**
- Fetches all deal IDs associated with your company from HubSpot
- For each deal, retrieves detailed properties:
- `dealname`, `dealstage`, `pipeline`, `outcome`, `outcome_notes`, `project_code`
- `major_condition_issue_description`, `major_condition_issue_photos`
- `coordination_status__stage_1_`, `retrofit_design_status`
- Filters to only deals in the `OPERATIONS_SOCIAL_HOUSING` pipeline
- Fetches the associated listing (UPRN, property IDs)
- Inserts each deal into the `hubspot_data` table
- **Downloads photo evidence files** and uploads them to S3 (bucket: `retrofit-data-dev`)
**⚠️ Note:** This script can take a long time if your company has many deals. It processes deals serially with progress reporting via `tqdm`.
**Output:** Deals appear in `hubspot_data` table; photos appear in S3 at `s3://retrofit-data-dev/hubspot/awaabs_law_evidence/`.
### Step 5: `hubspot_update_script.py` (Automatic scheduling)
After the initial setup, **no manual action is needed**. This script runs automatically every 15 minutes during working hours as a scheduled job.
**What it does:**
- Queries the local database for all stored deals for each company
- Compares each deal's stored fields against the live HubSpot data (13 fields checked)
- Updates the database if any values have changed in HubSpot
- **Uploads newly available photos** to S3 (with SHA-256 integrity verification)
- Prints a summary report of changes, updates, and any failures
---
## Script Reference
### `hubspot_company.py`
**Stage:** Seed (one-time setup)
**Frequency:** Run once per new company
**Speed:** Fast
**Purpose:** Load company master data into the database.
**Database Output:**
- Table: `hubspot_company_data`
- Fields: `company_id`, `company_name`
**Code Flow:**
```
For each company in config:
1. Call HubSpot API: get_company_information(company_id)
2. Insert record into hubspot_company_data table
```
---
### `hubspot_gather_all_deals.py`
**Stage:** Bulk Load (one-time per company)
**Frequency:** Run once per company (manually triggered)
**Speed:** Slow (serial processing of all deals)
**Purpose:** Perform initial load of all deals for target companies.
**Database Output:**
- Table: `hubspot_data`
- Fields: `deal_id`, `deal_name`, `company_id`, `stage`, `outcome`, `photos_s3_url`, and others
**S3 Output:**
- Bucket: `retrofit-data-dev`
- Path: `hubspot/awaabs_law_evidence/{filename}`
**Code Flow:**
```
For each company in config:
1. Fetch all deal IDs from HubSpot
2. For each deal:
a. Get deal properties from HubSpot
b. Filter by OPERATIONS_SOCIAL_HOUSING pipeline
c. Fetch associated listing data (UPRN, property IDs)
d. Insert deal into hubspot_data table
e. If photos exist: download from HubSpot URL, upload to S3, save S3 URL to DB
f. Print progress: "Uploaded deal_id {id} to db"
```
**Error Handling:** None — script will abort on first error. Re-run to retry.
---
### `hubspot_update_script.py`
**Stage:** Sync/Update (ongoing maintenance)
**Frequency:** Every 15 minutes during working hours (automated schedule)
**Speed:** Fast (only processes stored deals, compares, updates deltas)
**Purpose:** Keep database synchronized with live HubSpot data; handle new/updated photos.
**Database Operations:**
- Reads: All deals from `hubspot_data` for each company
- Writes: Updates only when fields differ from HubSpot
- S3 Uploads: New or previously missing photos
**Summary Report:**
After completion, prints a table of per-company statistics:
```
Company | Checked | Updated | Up-to-date | Failed
```
Plus detailed error messages for any failed updates.
**Code Flow:**
```
1. Initialize HubSpot client (warm-up: get_deal_stages)
2. For each company:
a. Query DB for all deals with company_id
b. For each deal:
- Fetch live deal data from HubSpot
- Compare 13 fields: deal_id, company_id, landlord_property_id, outcome,
dealstage, dealname, project_code, uprn,
outcome_notes, major_condition_issue_description,
major_condition_issue_photos, coordination_status,
design_status
- If any field differs: call upsert_hubspot_deal() to update DB
- If photos exist in HubSpot but not yet in S3:
* Download file from HubSpot URL
* Upload to S3
* Verify SHA-256 hash integrity
* Save S3 URL back to DB
- Collect success/failure counts
c. Print per-company summary
3. Print all failures (if any) with error messages
```
**Error Handling:** Wrapped in try/except per deal. Failures are logged, and the script continues to the next deal.
---
## Common Tasks
### I added a new company but deals aren't showing up
**Checklist:**
- [ ] Company added to `Companies` enum in `hubspotClient.py`
- [ ] Company added to the `companies` list in **all three scripts**
- [ ] Ran `hubspot_company.py` successfully
- [ ] Ran `hubspot_gather_all_deals.py` and watched for "Uploaded deal_id" messages
- [ ] Check database: `SELECT COUNT(*) FROM hubspot_data WHERE company_id = 'YOUR_ID'`
- [ ] Check HubSpot: Does the company have any deals in the OPERATIONS_SOCIAL_HOUSING pipeline?
### Deals exist in HubSpot but aren't syncing
The `hubspot_gather_all_deals.py` script only loads deals in the `OPERATIONS_SOCIAL_HOUSING` pipeline. If deals are in a different pipeline, they won't be loaded. Check the deal's pipeline in HubSpot.
### Photos aren't uploading
- First run of `hubspot_gather_all_deals.py` should upload photos at import time
- Subsequent runs of `hubspot_update_script.py` will upload newly available photos
- Check S3 bucket `retrofit-data-dev` under `hubspot/awaabs_law_evidence/`
- Check DB field `major_condition_issue_photos` (photo S3 URL is stored here)
### I need to re-sync everything for a company
1. Clear the deals from the database:
```sql
DELETE FROM hubspot_data WHERE company_id = 'YOUR_COMPANY_ID';
```
2. Clear the company:
```sql
DELETE FROM hubspot_company_data WHERE company_id = 'YOUR_COMPANY_ID';
```
3. Re-run from **Step 3** above (run `hubspot_company.py`, then `hubspot_gather_all_deals.py`)
---
## Dependencies
All scripts depend on:
- `HubSpotClient` from `../hubspotClient.py` — Handles HubSpot API calls
- `HubspotTodb` from `../../db/hubSpotLoad.py` — Handles database operations (insert/upsert/query)
- `tqdm` — Progress bars
- Python `requests` — HTTP downloads for photo files
Environment Requirements:
- Valid HubSpot API token (configured in `HubSpotClient.__init__()`)
- Database connection (configured in `HubspotTodb`)
- S3 credentials (for photo uploads)
- Network access to HubSpot API and S3
---
## Notes & Tips
1. **Idempotency:** `hubspot_gather_all_deals.py` and `hubspot_update_script.py` use upsert logic, so they can be run multiple times without creating duplicates.
2. **Large Portfolios:** If a company has thousands of deals, `hubspot_gather_all_deals.py` will take a while. Use `tqdm` progress indicators to monitor.
3. **Error Handling:** `hubspot_update_script.py` has error handling per deal. `hubspot_company.py` and `hubspot_gather_all_deals.py` do not — any failure aborts the script. If interrupted, simply re-run.
4. **Schedule:** `hubspot_update_script.py` is scheduled to run every 15 minutes during business hours (typically configured as a cron job or similar scheduler).
5. **Photo Integrity:** The `hubspot_update_script.py` verifies downloaded photos using SHA-256 hashing before committing the S3 URL to the database.
6. **Unused Fields:** The scripts populate `deals_to_add` and `deal_to_companies` dicts in `hubspot_gather_all_deals.py` but don't use them downstream. This is harmless but could be cleaned up in future refactors.
---
## Troubleshooting
| Issue | Likely Cause | Solution |
|-------|--------------|----------|
| "Company not found" error | Company enum not added or typo in name | Double-check `Companies` enum in `hubspotClient.py` |
| Deal count mismatch | Company wasn't added to the script's companies list | Ensure company is in `valuable_companies` / `companies` in all 3 scripts |
| Slow script execution | Large portfolio or network latency | Normal for first run; `hubspot_update_script.py` is faster on subsequent runs |
| Photos not uploading | Deal doesn't have `major_condition_issue_photos` property | Photos only upload if HubSpot deal has photos attached |
| S3 upload fails | Credentials or bucket issues | Check IAM permissions and bucket name (`retrofit-data-dev`) |
| Update script reports failures | Stale data or missing DB fields | Check error messages in summary report; may need to re-sync company |

19
etl/hubSpotClient/scripts/debug_one_address.py
View file

@ -1,19 +0,0 @@
from etl.hubSpotClient.hubspotClient import HubSpotClient
from etl.db.hubSpotLoad import HubspotTodb
working_deal_id = "319174821072"
deal_id = "484368267483"
hubspot = HubSpotClient()
db = HubspotTodb()
deal = db.find_deal_with_deal_id(deal_id)
deal2 = db.find_deal_with_deal_id(working_deal_id)
db.update_deal(deal2, hubspot)
db.update_deal(deal, hubspot)

16
etl/hubSpotClient/scripts/hubspot_company.py
View file

@ -1,16 +0,0 @@
from etl.hubSpotClient.hubspotClient import HubSpotClient, Companies, Pipeline
from tqdm import tqdm
from etl.db.hubSpotLoad import HubspotTodb
hubspot = HubSpotClient()
companies = [
Companies.THE_GUINESS_PARTNERSHIP,
Companies.SOUTHERN_HOUSING_GROUP,
]
# All deals from a pipeline_id via filter
for company in companies:
new_company_info = hubspot.get_company_information(company.value)
loader = HubspotTodb()
loader.new_record_company(new_company_info)

42
etl/hubSpotClient/scripts/hubspot_gather_all_deals.py
View file

@ -1,42 +0,0 @@
from etl.hubSpotClient.hubspotClient import HubSpotClient, Companies, Pipeline
from tqdm import tqdm
from etl.db.hubSpotLoad import HubspotTodb
hubspot = HubSpotClient()
loader = HubspotTodb()
PIPELINE_ID = Pipeline.OPERATIONS_SOCIAL_HOUSING.value
valuable_companies = [
Companies.THE_GUINESS_PARTNERSHIP.value,
Companies.SOUTHERN_HOUSING_GROUP.value,
]
deals_to_add = []
deal_to_companies = {}
for company_id in valuable_companies:
# 🔥 Cheap: company → deals
deal_ids = hubspot.get_deals_from_company(company_id)
for deal_id in tqdm(deal_ids, desc=f"Company {company_id}"):
# Fetch minimal deal info once
deal_data = hubspot.from_deal_get_info(deal_id)
print(f"working on deal {deal_id}")
# Filter by pipeline (small local filter)
if deal_data.get("pipeline") != PIPELINE_ID:
continue
deals_to_add.append(deal_id)
deal_to_companies[deal_id] = company_id
listing_data = hubspot.from_deal_get_associated_listing(deal_id)
loader.new_record_to_hubspot_data(
deal_data,
company_id,
listing_data,
hubspot
)
print(f"Uploaded deal_id {deal_id} to db")

86
etl/hubSpotClient/scripts/hubspot_update_script.py
View file

@ -1,86 +0,0 @@
from etl.hubSpotClient.hubspotClient import HubSpotClient, Companies, Pipeline
from tqdm import tqdm
from etl.db.hubSpotLoad import HubspotTodb
hubspot = HubSpotClient()
hubspot.get_deal_stages()
db = HubspotTodb()
companies = [
Companies.THE_GUINESS_PARTNERSHIP,
Companies.SOUTHERN_HOUSING_GROUP,
]
# Global trackers
all_failed_deals = []
summary_report = {}
print("\n🚀 Starting HubSpot deal consistency check...\n")
for company in companies:
print(f"\n🏢 Processing company: {company.name}")
records = db.find_all_deals_with_company_id(company.value)
updated_count = 0
checked_count = 0
failed_deals = []
for deal in tqdm(records, desc=f"Checking HubSpot deals for {company.name}"):
checked_count += 1
try:
print(f"🔍 Working on deal {deal}")
was_up_to_date = db.update_deal(deal, hubspot)
if not was_up_to_date:
updated_count += 1
print(f"🧩 Deal {deal} was updated.")
else:
print(f"📈 Deal {deal} already up to date.")
except Exception as e:
failed_info = {
"company": company.name,
"deal_id": deal,
"error": str(e)
}
failed_deals.append(failed_info)
all_failed_deals.append(failed_info)
print(f"❌ Failed to update deal {deal}: {e}")
# Store per-company summary (don’t print yet)
summary_report[company.name] = {
"checked": checked_count,
"updated": updated_count,
"failed": len(failed_deals),
"up_to_date": checked_count - updated_count - len(failed_deals),
}
# Company-level quick summary
print(f"\n✅ Finished checking {checked_count} deals for company {company.name}.")
print(f" 🧩 {updated_count} deal(s) were updated.")
print(f" 📈 {summary_report[company.name]['up_to_date']} deal(s) were already up to date.")
print(f" ⚠️ {len(failed_deals)} deal(s) failed.\n")
# ---- Final Summary Report ----
print("\n" + "=" * 100)
print("📊 FINAL SUMMARY REPORT (ALL COMPANIES)")
print("=" * 100)
for company_name, stats in summary_report.items():
print(f"\n🏢 {company_name}")
print(f" - Total deals checked: {stats['checked']}")
print(f" - Updated deals: {stats['updated']}")
print(f" - Up-to-date deals: {stats['up_to_date']}")
print(f" - Failed deals: {stats['failed']}")
# ---- Global Failed Deals ----
if all_failed_deals:
print("\n" + "=" * 100)
print("⚠️ FAILED DEALS DETAILS")
print("=" * 100)
for f in all_failed_deals:
print(f" - Company: {f['company']:<25} | Deal ID: {f['deal_id']} | Error: {f['error']}")
else:
print("\n🎉 No failed deals across any company!")
print("\n🏁 HubSpot deal consistency check complete!\n")

58
etl/hubSpotClient/types.py Normal file
View file

@ -0,0 +1,58 @@
from sqlmodel import Field, SQLModel
from sqlalchemy import Column
from sqlalchemy.dialects.postgresql import UUID
import uuid
from pydantic import Field, field_validator, ValidationError, model_validator
from etl.utils.utils import get_sharepoint_path
from etl.scraper.scraper import SharePointScraper, SharePointInstaller
def string_to_installer(installer):
if installer.upper() == "J & J CRUMP":
return SharePointInstaller.JJC
elif installer.upper() == "SCIS":
return SharePointInstaller.SOUTH_COAST_INSULATION
elif installer.upper() == "SGEC":
return SharePointInstaller.SGEC
else:
return None
class BaseModel(SQLModel):
id: uuid.UUID = Field(
default_factory=uuid.uuid4,
sa_column=Column(UUID(as_uuid=True), primary_key=True)
)
class SubmissionInfoFromDeal(BaseModel):
deal_id: str = Field(..., min_length=1)
deal_name: str = Field(..., min_length=1)
work_type: str = Field(..., min_length=1)
needs_trickle_ventilation: bool
post_sap_score: int
existing_wall_insulation: str = Field(..., min_length=1)
no_of_wet_rooms: int
installer: str = Field(..., min_length=1)
submission_folder_path: str = Field(..., min_length=1)
landlord_id: str = Field(..., min_length=1)
domna_id: str = Field(..., min_length=1)
uprn: str = Field(..., min_length=1)
@field_validator('post_sap_score', 'no_of_wet_rooms')
@classmethod
def must_be_non_negative(cls, v):
if v < 0:
raise ValidationError("Must be non-negative for Post Sap Score")
return v
@model_validator(mode="after")
def check_submission_folder_path(self):
path = get_sharepoint_path(self.submission_folder_path)
installer = string_to_installer(self.installer)
sp = SharePointScraper(installer)
files = sp.get_folders_in_path(path)
if "value" in files:
if len(files["value"]) > 0:
return self
raise RuntimeError("Sharepoint URL invalid")

Some files were not shown because too many files have changed in this diff Show more