From bdb2c1c827657df42e3a14be2bda845c99487945 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Thu, 29 Jan 2026 17:28:45 +0000 Subject: [PATCH 01/86] added code to run sal --- .devcontainer/asset_list/docker-compose.yml | 6 +- backend/address2UPRN/tests/test_data.csv | 195 +++++++++++++++++++- 2 files changed, 197 insertions(+), 4 deletions(-) diff --git a/.devcontainer/asset_list/docker-compose.yml b/.devcontainer/asset_list/docker-compose.yml index 67b27444..06e4124d 100644 --- a/.devcontainer/asset_list/docker-compose.yml +++ b/.devcontainer/asset_list/docker-compose.yml @@ -4,11 +4,11 @@ services: model-sal: user: "${UID}:${GID}" build: - context: .. - dockerfile: .devcontainer/Dockerfile + context: ../.. + dockerfile: .devcontainer/asset_list/Dockerfile command: sleep infinity volumes: - - ..:/workspaces/model + - ../../:/workspaces/model networks: - model-net diff --git a/backend/address2UPRN/tests/test_data.csv b/backend/address2UPRN/tests/test_data.csv index f3d9b64c..91bc49e1 100644 --- a/backend/address2UPRN/tests/test_data.csv +++ b/backend/address2UPRN/tests/test_data.csv @@ -164,4 +164,197 @@ FLAT 8 599 HARROW ROAD,W10 4RA,None 24b Honley Road,SE6 2HZ,None FLAT B 158 LEAHURST ROAD,SE13 5NL,100021976974 2 COLLEGE HOUSE,CM7 1JS,100091449870 -3 COLLEGE HOUSE,CM7 1JS,100091449871 \ No newline at end of file +3 COLLEGE HOUSE,CM7 1JS,100091449871 +1 Anita Street,M4 5DU,None +2 Anita Street,M4 5DU,77123061 +5 Anita Street,M4 5DU,77123081 +6 Anita Street,M4 5DU,77123082 +8 Anita Street,M4 5DU,None +9 Anita Street,M4 5DU,None +10 Anita Street,M4 5DU,77123051 +12 Anita Street,M4 5DU,77123053 +19 Anita Street,M4 5DU,None +22 Anita Street,M4 5DU,None +26 Anita Street,M4 5DU,77123068 +28 Anita Street,M4 5DU,None +30 Anita Street,M4 5DU,None +32 Anita Street,M4 5DU,None +33 Anita Street,M4 5DU,77123076 +34 Anita Street,M4 5DU,None +35 Anita Street,M4 5DU,77123078 +36 Anita Street,M4 5DU,77123079 +23 George Leigh Street,M4 5DR,77123171 +25 George Leigh Street,M4 5DR,None +35 George Leigh Street,M4 5DR,77123177 +39 George Leigh Street,M4 5DR,77123179 +41 George Leigh Street,M4 5DR,None +43 George Leigh Street,M4 5DR,None +49 George Leigh Street,M4 5DR,None +51 George Leigh Street,M4 5DR,77123185 +55 George Leigh Street,M4 5DR,None +57 George Leigh Street,M4 5DR,None +"1a, Victoria Square",M4 5DX,77211153 +2a Victoria Square ,M4 5DX,None +"4a, Victoria Square",M4 5DX,77211155 +5a Victoria Square,M4 5DX,77211156 + 6a Victoria Square,M4 5DX,77211157 +7a Victoria Square,M4 5DX,77211158 +8a Victoria Square,M4 5DX,77211159 +9a Victoria Square,M4 5DX,77211160 +10a Victoria Square,M4 5DX,77211161 +11a Victoria Square,M4 5DX,77211162 +12a Victoria Square,M4 5DX,77211163 +13a Victoria Square,M4 5DX,77211164 +14a Victoria Square,M4 5DX,77211165 +15a Victoria Square,M4 5DX,77211166 +16a Victoria Square,M4 5DX,77211167 +17a Victoria Square,M4 5DX,77211168 +18a Victoria Square,M4 5DX,77211169 +19a Victoria Square,M4 5DX,77211170 +20a Victoria Square,M4 5DX,77211171 +21a Victoria Square,M4 5DY,77211172 +22a Victoria Square,M4 5DY,None +23a Victoria Square,M4 5DY,77211174 +24a Victoria Square,M4 5DY,77211175 +25a Victoria Square,M4 5DY,77211176 +26a Victoria Square,M4 5DY,77211177 +27a Victoria Square,M4 5DY,77211178 +28a Victoria Square,M4 5DY,None +29a Victoria Square,M4 5DY,77211180 +30a Victoria Square,M4 5DY,77211181 +31a Victoria Square,M4 5DY,77211182 +32a Victoria Square,M4 5DY,77211183 +33a Victoria Square,M4 5DY,77211184 +34a Victoria Square,M4 5DY,77211185 +35a Victoria Square,M4 5DY,None +36a Victoria Square,M4 5DY,77211187 +37a Victoria Square,M4 5DY,77211188 +38a Victoria Square,M4 5DY,77211189 +39a Victoria Square,M4 5DY,77211190 +40a Victoria Square,M4 5DY,None +41a Victoria Square,M4 5DY,77211192 +42a Victoria Square,M4 5DY,77211193 +43a Victoria Square,M4 5DY,77211194 +44a Victoria Square,M4 5DY,77211195 +45a Victoria Square,M4 5DY,77211196 +46a Victoria Square,M4 5DY,77211197 +47a Victoria Square,M4 5DY,77211198 +48a Victoria Square,M4 5DY,77211199 +49a Victoria Square,M4 5DY,77211200 +50a Victoria Square,M4 5DY,77211201 +51a Victoria Square,M4 5DY,77211202 +52a Victoria Square,M4 5DY,77211203 +53a Victoria Square,M4 5DY,77211204 +54a Victoria Square,M4 5DY,77211205 +55a Victoria Square,M4 5DY,77211206 +56a Victoria Square,M4 5DZ,77211207 +57a Victoria Square,M4 5DZ,None +58a Victoria Square,M4 5DZ,77211209 +59a Victoria Square,M4 5DZ,77211210 +60a Victoria Square,M4 5DZ,77211211 +61a Victoria Square,M4 5DZ,77211212 +62a Victoria Square,M4 5DZ,77211213 +63a Victoria Square,M4 5DZ,None +64a Victoria Square,M4 5DZ,77211215 +65a Victoria Square,M4 5DZ,77211216 +66a Victoria Square,M4 5DZ,None +67a Victoria Square,M4 5DZ,None +68a Victoria Square,M4 5DZ,77211219 +69a Victoria Square,M4 5DZ,77211220 +70a Victoria Square,M4 5DZ,77211221 +71a Victoria Square,M4 5DZ,77211222 +72a Victoria Square,M4 5DZ,77211223 +73a Victoria Square,M4 5DZ,77211224 +74a Victoria Square,M4 5DZ,None +75a Victoria Square,M4 5DZ,77211226 +76a Victoria Square,M4 5DZ,77211227 +77a Victoria Square,M4 5DZ,None +78a Victoria Square,M4 5DZ,77211229 +79a Victoria Square,M4 5DZ,77211230 +80a Victoria Square,M4 5DZ,77211231 +81a Victoria Square,M4 5DZ,77211232 +82 Victoria Square,M4 5DZ,None +83a Victoria Square,M4 5DZ,77211234 +84a Victoria Square,M4 5DZ,None +85a Victoria Square,M4 5DZ,77211236 +86a Victoria Square,M4 5DZ,77211237 +87a Victoria Square,M4 5DZ,77211238 +88a Victoria Square,M4 5DZ,None +89a Victoria Square,M4 5DZ,77211240 +90a Victoria Square,M4 5DZ,77211241 +91a Victoria Square,M4 5DZ,77211242 +92a Victoria Square,M4 5DZ,77211243 +93a Victoria Square,M4 5EA,77211244 +94a Victoria Square,M4 5EA,None +95a Victoria Square,M4 5EA,77211246 +96a Victoria Square,M4 5EA,77211247 +97a Victoria Square,M4 5EA,77211248 +98a Victoria Square,M4 5EA,77211249 +99a Victoria Square,M4 5EA,77211250 +100a Victoria Square,M4 5EA,77211251 +101a Victoria Square,M4 5EA,None +102a Victoria Square,M4 5EA,None +103a Victoria Square,M4 5EA,77211254 +104a Victoria Square,M4 5EA,77211255 +105a Victoria Square,M4 5EA,None +106a Victoria Square,M4 5EA,77211257 +107a Victoria Square,M4 5EA,77211258 +108a Victoria Square,M4 5EA,77211259 +109a Victoria Square,M4 5EA,77211260 +110a Victoria Square,M4 5EA,77211261 +111a Victoria Square,M4 5EA,77211262 +112a Victoria Square,M4 5EA,None +113a Victoria Square,M4 5EA,77211264 +114a Victoria Square,M4 5EA,77211265 +115a Victoria Square,M4 5EA,77211266 +116a Victoria Square,M4 5EA,77211267 +117a Victoria Square,M4 5EA,None +118a Victoria Square,M4 5EA,None +119a Victoria Square,M4 5EA,77211270 +120a Victoria Square,M4 5EA,77211271 +121a Victoria Square,M4 5EA,77211272 +122a Victoria Square,M4 5EA,77211273 +123a Victoria Square,M4 5EA,77211274 +124a Victoria Square,M4 5EA,None +125a Victoria Square,M4 5EA,77211276 +126a Victoria Square,M4 5EA,77211277 +127a Victoria Square,M4 5EA,77211278 +128a Victoria Square,M4 5EA,77211279 +129a Victoria Square,M4 5EA,77211280 +130a Victoria Square,M4 5FA,77211281 +131a Victoria Square,M4 5FA,77211282 +132a Victoria Square,M4 5FA,77211283 +133a Victoria Square,M4 5FA,None +134a Victoria Square,M4 5FA,77211285 +135a Victoria Square,M4 5FA,77211286 +136a Victoria Square,M4 5FA,77211287 +137a Victoria Square,M4 5FA,77211288 +138a Victoria Square,M4 5FA,77211289 +139a Victoria Square,M4 5FA,77211290 +140a Victoria Square,M4 5FA,77211291 +141a Victoria Square,M4 5FA,None +142a Victoria Square,M4 5FA,77211293 +143a Victoria Square,M4 5FA,77211294 +144a Victoria Square,M4 5FA,77211295 +145a Victoria Square,M4 5FA,None +146a Victoria Square,M4 5FA,77211297 +147a Victoria Square,M4 5FA,77211298 +148a Victoria Square,M4 5FA,77211299 +149a Victoria Square,M4 5FA,77211300 +150a Victoria Square,M4 5FA,77211301 +151a Victoria Square,M4 5FA,None +152a Victoria Square,M4 5FA,77211303 +153a Victoria Square,M4 5FA,None +154a Victoria Square,M4 5FA,77211305 +155a Victoria Square,M4 5FA,None +156a Victoria Square,M4 5FA,77211307 +157a Victoria Square,M4 5FA,77211308 +158a Victoria Square,M4 5FA,77211309 +159a Victoria Square,M4 5FA,None +160a Victoria Square,M4 5FA,77211311 +161a Victoria Square,M4 5FA,None +162a Victoria Square,M4 5FA,None +163a Victoria Square,M4 5FA,77211314 +164a Victoria Square,M4 5FA,77211315 +165a Victoria Square,M4 5FA,77211316 +166a Victoria Square,M4 5FA,None \ No newline at end of file From 0fccc0cc10f2c96e768cc0fc3cf51cc7999ecdd4 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Fri, 30 Jan 2026 13:48:07 +0000 Subject: [PATCH 02/86] just test if plan works --- .devcontainer/asset_list/requirements.txt | 1 + .github/workflows/deploy_terraform.yml | 35 ++-- asset_list/app.py | 220 ++++++++++++---------- 3 files changed, 135 insertions(+), 121 deletions(-) diff --git a/.devcontainer/asset_list/requirements.txt b/.devcontainer/asset_list/requirements.txt index cfab95ec..0640f2c9 100644 --- a/.devcontainer/asset_list/requirements.txt +++ b/.devcontainer/asset_list/requirements.txt @@ -21,3 +21,4 @@ pydantic>=1.10.7,<2 sqlmodel # Formatting black==26.1.0 +dotenv diff --git a/.github/workflows/deploy_terraform.yml b/.github/workflows/deploy_terraform.yml index a7aef225..05a667bb 100644 --- a/.github/workflows/deploy_terraform.yml +++ b/.github/workflows/deploy_terraform.yml @@ -3,8 +3,7 @@ name: Deploy terraform stack on: push: branches: - - dev - - prod + - "*" jobs: deploy: @@ -47,22 +46,22 @@ jobs: - name: Terraform Init run: cd infrastructure/terraform && terraform init - - name: Terraform Workspace - run: | - BRANCH_NAME=$(echo "${{ github.ref }}" | sed -e "s/^refs\/heads\///") - cd infrastructure/terraform - terraform workspace select ${BRANCH_NAME} || terraform workspace new ${BRANCH_NAME} + # - name: Terraform Workspace + # run: | + # BRANCH_NAME=$(echo "${{ github.ref }}" | sed -e "s/^refs\/heads\///") + # cd infrastructure/terraform + # terraform workspace select ${BRANCH_NAME} || terraform workspace new ${BRANCH_NAME} - name: Terraform Plan run: | BRANCH_NAME=$(echo "${{ github.ref }}" | sed -e "s/^refs\/heads\///") - cd infrastructure/terraform && terraform plan -var-file=${BRANCH_NAME}.tfvars + cd infrastructure/terraform && terraform plan -var-file=dev.tfvars - - name: Deploy to Dev - if: github.ref == 'refs/heads/dev' - run: cd infrastructure/terraform && terraform apply -var-file=dev.tfvars -auto-approve - env: - name: dev + # - name: Deploy to Dev + # if: github.ref == 'refs/heads/dev' + # run: cd infrastructure/terraform && terraform apply -var-file=dev.tfvars -auto-approve + # env: + # name: dev - name: Configure AWS credentials (ProdAdmin) uses: aws-actions/configure-aws-credentials@v1 @@ -73,8 +72,8 @@ jobs: env: AWS_PROFILE: "ProdAdmin" - - name: Deploy to Prod - if: github.ref == 'refs/heads/prod' - run: cd infrastructure/terraform && terraform apply -var-file=prod.tfvars -auto-approve - env: - name: prod + # - name: Deploy to Prod + # if: github.ref == 'refs/heads/prod' + # run: cd infrastructure/terraform && terraform apply -var-file=prod.tfvars -auto-approve + # env: + # name: prod diff --git a/asset_list/app.py b/asset_list/app.py index e8ce408e..1c7200fd 100644 --- a/asset_list/app.py +++ b/asset_list/app.py @@ -12,23 +12,35 @@ from asset_list.utils import get_data from dotenv import load_dotenv from backend.SearchEpc import SearchEpc + load_dotenv(dotenv_path="backend/.env") -EPC_AUTH_TOKEN = os.getenv("EPC_AUTH_TOKEN", "a2Nvbm5rb3dsZXNzYXJAZ21haWwuY29tOjY5MGJiMWM0NmIyOGI5ZDUxYzAxMzQzYzNiZGNlZGJjZDNmODQwMzA=") +EPC_AUTH_TOKEN = os.getenv( + "EPC_AUTH_TOKEN", + "a2Nvbm5rb3dsZXNzYXJAZ21haWwuY29tOjY5MGJiMWM0NmIyOGI5ZDUxYzAxMzQzYzNiZGNlZGJjZDNmODQwMzA=", +) -def extract_address1(asset_list, full_address_col, postcode_col, method="first_two_words"): +def extract_address1( + asset_list, full_address_col, postcode_col, method="first_two_words" +): if method == "first_two_words": - asset_list["address1_extracted"] = asset_list[full_address_col].str.split(" ").str[:2].str.join(" ") + asset_list["address1_extracted"] = ( + asset_list[full_address_col].str.split(" ").str[:2].str.join(" ") + ) return asset_list if method == "first_word": - asset_list["address1_extracted"] = asset_list[full_address_col].str.split(" ").str[0] + asset_list["address1_extracted"] = ( + asset_list[full_address_col].str.split(" ").str[0] + ) return asset_list if method == "house_number_extraction": asset_list["address1_extracted"] = asset_list.apply( - lambda x: SearchEpc.get_house_number(address=x[full_address_col], postcode=x[postcode_col]), - axis=1 + lambda x: SearchEpc.get_house_number( + address=x[full_address_col], postcode=x[postcode_col] + ), + axis=1, ) return asset_list @@ -57,58 +69,17 @@ def app(): EPC recommendations Property UPRN """ -<<<<<<< HEAD - data_folder = ("/workspaces/model/asset_list") + data_folder = "/workspaces/model/asset_list" data_filename = "assets.xlsx" -======= - - data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Hackney" - data_filename = "Domna SHF Wave 3.xlsx" - sheet_name = "Domna Wave 3" - postcode_column = 'Postcode' - address1_column = "Address 1" - address1_method = None - fulladdress_column = None - address_cols_to_concat = ["Address 1"] - missing_postcodes_method = None - landlord_year_built = None - landlord_os_uprn = "UPRN" - landlord_property_type = None - landlord_built_form = None - landlord_wall_construction = None - landlord_roof_construction = None - landlord_heating_system = None - landlord_existing_pv = None - landlord_property_id = "Row ID" - landlord_sap = None - outcomes_filename = None - outcomes_sheetname = None - outcomes_postcode = None - outcomes_houseno = None - outcomes_id = None - outcomes_address = None - master_filepaths = [] - master_id_colnames = [] - master_to_asset_list_filepath = None - phase = False - ecosurv_landlords = None - asset_list_header = 0 - landlord_block_reference = None - - # Peabody data for cleaning - data_folder = ("/Users/khalimconn-kowlessar/Documents/hestia/Customers/Peabody/Nov 2025 Consulting " - "Project/data_validation") - data_filename = "to_standardise_uprns.xlsx" ->>>>>>> 3874da6177cbcc37f7a488bec0a06e387906653c sheet_name = "Sheet1" - postcode_column = 'Postcode' + postcode_column = "Post Code" address1_column = None - address1_method = 'house_number_extraction' - fulladdress_column = 'Address' + address1_method = "house_number_extraction" + fulladdress_column = "User Input" address_cols_to_concat = None missing_postcodes_method = None landlord_year_built = None - landlord_os_uprn = None + landlord_os_uprn = "UPRN" landlord_property_type = None landlord_built_form = None landlord_wall_construction = None @@ -155,49 +126,62 @@ def app(): landlord_existing_pv=landlord_existing_pv, landlord_sap=landlord_sap, landlord_block_reference=landlord_block_reference, - phase=phase + phase=phase, ) asset_list.init_standardise() # We produce the new maps, which can be saved for future useage new_property_type_map = { - k: v for k, v in ( - asset_list.variable_mappings[asset_list.landlord_property_type] if - asset_list.landlord_property_type else {} + k: v + for k, v in ( + asset_list.variable_mappings[asset_list.landlord_property_type] + if asset_list.landlord_property_type + else {} ).items() if k not in PROPERTY_MAPPING } new_built_form_map = { - k: v for k, v in ( - asset_list.variable_mappings[asset_list.landlord_built_form] if - asset_list.landlord_built_form else {} + k: v + for k, v in ( + asset_list.variable_mappings[asset_list.landlord_built_form] + if asset_list.landlord_built_form + else {} ).items() if k not in BUILT_FORM_MAPPINGS } new_wall_map = { - k: v for k, v in ( - asset_list.variable_mappings[asset_list.landlord_wall_construction] if - asset_list.landlord_wall_construction else {} + k: v + for k, v in ( + asset_list.variable_mappings[asset_list.landlord_wall_construction] + if asset_list.landlord_wall_construction + else {} ).items() if k not in WALL_CONSTRUCTION_MAPPINGS } new_heating_map = { - k: v for k, v in ( - asset_list.variable_mappings[asset_list.landlord_heating_system] if - asset_list.landlord_heating_system else {} + k: v + for k, v in ( + asset_list.variable_mappings[asset_list.landlord_heating_system] + if asset_list.landlord_heating_system + else {} ).items() if k not in HEATING_MAPPINGS } new_existing_pv_map = { - k: v for k, v in ( - asset_list.variable_mappings[asset_list.landlord_existing_pv] if asset_list.landlord_existing_pv else {} + k: v + for k, v in ( + asset_list.variable_mappings[asset_list.landlord_existing_pv] + if asset_list.landlord_existing_pv + else {} ).items() if k not in EXISTING_PV_MAPPINGS } new_roof_construction_map = { - k: v for k, v in ( - asset_list.variable_mappings[asset_list.landlord_roof_construction] if - asset_list.landlord_roof_construction else {} + k: v + for k, v in ( + asset_list.variable_mappings[asset_list.landlord_roof_construction] + if asset_list.landlord_roof_construction + else {} ).items() if k not in ROOF_CONSTRUCTION_MAPPINGS } @@ -211,7 +195,7 @@ def app(): outcomes_address=outcomes_address, outcomes_postcode=outcomes_postcode, outcomes_houseno=outcomes_houseno, - outcomes_id=outcomes_id + outcomes_id=outcomes_id, ) asset_list.flag_survey_master( @@ -245,14 +229,16 @@ def app(): skip = max(chunk_indexes) if any(x in folder_contents for x in downloaded_files): - skip = max([i for i in chunk_indexes if filename.format(i=i) in folder_contents]) + skip = max( + [i for i in chunk_indexes if filename.format(i=i) in folder_contents] + ) for i in range(0, len(asset_list.standardised_asset_list), chunk_size): print(f"Processing chunk {i} to {i + chunk_size}") if skip is not None and not force_retrieve_data: if i <= skip: continue - chunk = asset_list.standardised_asset_list[i:i + chunk_size] + chunk = asset_list.standardised_asset_list[i : i + chunk_size] epc_data_chunk, errors_chunk, no_epc_chunk = get_data( df=chunk, row_id_name=asset_list.DOMNA_PROPERTY_ID, @@ -264,7 +250,7 @@ def app(): built_form_column=AssetList.STANDARD_BUILT_FORM, manual_uprn_map=manual_uprn_map, epc_api_only=epc_api_only, - epc_auth_token=EPC_AUTH_TOKEN + epc_auth_token=EPC_AUTH_TOKEN, ) # We now retrieve any failed properties @@ -287,7 +273,9 @@ def app(): # Append the failed data to the main data # Store the chunk locally as a csv - pd.DataFrame(epc_data_chunk).to_csv(os.path.join(data_folder, f"Chunks/Chunk {i}.csv"), index=False) + pd.DataFrame(epc_data_chunk).to_csv( + os.path.join(data_folder, f"Chunks/Chunk {i}.csv"), index=False + ) # Store the errors and no-data locally with open(os.path.join(data_folder, f"Chunks/Chunk {i} errors.json"), "w") as f: json.dump(errors_chunk, f) @@ -318,7 +306,9 @@ def app(): unique_recommendations = set() for _, row in recommendations_df.iterrows(): - unique_recommendations.update([rec["improvement-summary-text"] for rec in row["recommendations"]]) + unique_recommendations.update( + [rec["improvement-summary-text"] for rec in row["recommendations"]] + ) columns = [asset_list.DOMNA_PROPERTY_ID] + list(unique_recommendations) transformed_data = [] @@ -338,20 +328,24 @@ def app(): transformed_df = pd.DataFrame(transformed_data) for col in [ "Floor insulation (solid floor)", - "Floor insulation", "Floor insulation (suspended floor)" + "Floor insulation", + "Floor insulation (suspended floor)", ]: if col not in transformed_df.columns: transformed_df[col] = False transformed_df = transformed_df[ [ - asset_list.DOMNA_PROPERTY_ID, "Floor insulation (solid floor)", - "Floor insulation", "Floor insulation (suspended floor)" + asset_list.DOMNA_PROPERTY_ID, + "Floor insulation (solid floor)", + "Floor insulation", + "Floor insulation (suspended floor)", ] ] transformed_df["epc_has_floor_recommendation"] = ( - transformed_df["Floor insulation (solid floor)"] | transformed_df["Floor insulation"] | - transformed_df["Floor insulation (suspended floor)"] + transformed_df["Floor insulation (solid floor)"] + | transformed_df["Floor insulation"] + | transformed_df["Floor insulation (suspended floor)"] ) # Get the find my epc data @@ -364,21 +358,20 @@ def app(): find_my_epc_data.append( { asset_list.DOMNA_PROPERTY_ID: x[asset_list.DOMNA_PROPERTY_ID], - **x["find_my_epc_data"] + **x["find_my_epc_data"], } ) else: find_my_epc_data.append( - { - asset_list.DOMNA_PROPERTY_ID: x[asset_list.DOMNA_PROPERTY_ID] - } + {asset_list.DOMNA_PROPERTY_ID: x[asset_list.DOMNA_PROPERTY_ID]} ) find_my_epc_data = pd.DataFrame(find_my_epc_data) find_my_epc_data = find_my_epc_data.merge( transformed_df[[asset_list.DOMNA_PROPERTY_ID, "epc_has_floor_recommendation"]], - how="left", on=asset_list.DOMNA_PROPERTY_ID + how="left", + on=asset_list.DOMNA_PROPERTY_ID, ) # We check if we get the solar pv column: @@ -388,24 +381,26 @@ def app(): # Retrieve just the data we need epc_df = epc_df[ [asset_list.DOMNA_PROPERTY_ID] + list(asset_list.EPC_API_DATA_NAMES.keys()) - ].rename( - columns=asset_list.EPC_API_DATA_NAMES - ) + ].rename(columns=asset_list.EPC_API_DATA_NAMES) # Look for columns not in the find my EPC data, which will have happened if we didn't # retrieve it in the first place - missed_find_epc_cols = [c for c in list(asset_list.FIND_EPC_DATA_NAMES.keys()) if c not in find_my_epc_data.columns] + missed_find_epc_cols = [ + c + for c in list(asset_list.FIND_EPC_DATA_NAMES.keys()) + if c not in find_my_epc_data.columns + ] if missed_find_epc_cols: for c in missed_find_epc_cols: find_my_epc_data[c] = None epc_df = epc_df.merge( find_my_epc_data[ - [asset_list.DOMNA_PROPERTY_ID, "epc_has_floor_recommendation"] + list(asset_list.FIND_EPC_DATA_NAMES.keys()) - ] - .rename(columns=asset_list.FIND_EPC_DATA_NAMES), + [asset_list.DOMNA_PROPERTY_ID, "epc_has_floor_recommendation"] + + list(asset_list.FIND_EPC_DATA_NAMES.keys()) + ].rename(columns=asset_list.FIND_EPC_DATA_NAMES), how="left", - on=asset_list.DOMNA_PROPERTY_ID + on=asset_list.DOMNA_PROPERTY_ID, ) asset_list.merge_data(epc_df) @@ -422,7 +417,10 @@ def app(): asset_list.get_work_figures() # Store as an excel - filename = os.path.join(data_folder, ".".join(data_filename.split(".")[:-1])) + " - Standardised.xlsx" + filename = ( + os.path.join(data_folder, ".".join(data_filename.split(".")[:-1])) + + " - Standardised.xlsx" + ) # Store the data in two tabs. One for the asset list with the EPC data and the second with the flat data # Determine inspections priority @@ -446,26 +444,42 @@ def app(): # ) with pd.ExcelWriter(filename) as writer: - asset_list.standardised_asset_list.to_excel(writer, sheet_name="Standardised Asset List", index=False) + asset_list.standardised_asset_list.to_excel( + writer, sheet_name="Standardised Asset List", index=False + ) if asset_list.block_analysis_df is not None: - asset_list.block_analysis_df.to_excel(writer, sheet_name="Block Analysis", index=False) + asset_list.block_analysis_df.to_excel( + writer, sheet_name="Block Analysis", index=False + ) # If we have outcomes, we add a tab with the outcomes if not asset_list.outcomes_for_output.empty: - asset_list.outcomes_for_output.to_excel(writer, sheet_name="Outcomes", index=False) + asset_list.outcomes_for_output.to_excel( + writer, sheet_name="Outcomes", index=False + ) if not asset_list.unmatched_submissions.empty: - asset_list.unmatched_submissions.to_excel(writer, sheet_name="Unmatched Submissions", index=False) + asset_list.unmatched_submissions.to_excel( + writer, sheet_name="Unmatched Submissions", index=False + ) if not asset_list.outcomes_no_match.empty: - asset_list.outcomes_no_match.to_excel(writer, sheet_name="Unmatched Outcomes", index=False) + asset_list.outcomes_no_match.to_excel( + writer, sheet_name="Unmatched Outcomes", index=False + ) if not asset_list.ecosurv_no_match.empty: - asset_list.ecosurv_no_match.to_excel(writer, sheet_name="Unmatched Ecosurv", index=False) + asset_list.ecosurv_no_match.to_excel( + writer, sheet_name="Unmatched Ecosurv", index=False + ) if not asset_list.geographical_areas.empty: - asset_list.geographical_areas.to_excel(writer, sheet_name="Geographical Areas", index=False) + asset_list.geographical_areas.to_excel( + writer, sheet_name="Geographical Areas", index=False + ) # Store dupes if asset_list.duplicated_addresses is not None: if not asset_list.duplicated_addresses.empty: - asset_list.duplicated_addresses.to_excel(writer, sheet_name="Duplicate Properties", index=False) + asset_list.duplicated_addresses.to_excel( + writer, sheet_name="Duplicate Properties", index=False + ) From 6594c3fa1e5cb93a082dc0e3223c0aaa53f51041 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Fri, 30 Jan 2026 13:50:02 +0000 Subject: [PATCH 03/86] just test if plan works --- .github/workflows/deploy_terraform.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy_terraform.yml b/.github/workflows/deploy_terraform.yml index 05a667bb..5bc02440 100644 --- a/.github/workflows/deploy_terraform.yml +++ b/.github/workflows/deploy_terraform.yml @@ -3,7 +3,7 @@ name: Deploy terraform stack on: push: branches: - - "*" + - "**" jobs: deploy: From afd19734992e2e272302848ef382bb36c8486c2e Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Fri, 30 Jan 2026 14:16:19 +0000 Subject: [PATCH 04/86] add tf workspace --- .github/workflows/deploy_terraform.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/deploy_terraform.yml b/.github/workflows/deploy_terraform.yml index 5bc02440..16ca1068 100644 --- a/.github/workflows/deploy_terraform.yml +++ b/.github/workflows/deploy_terraform.yml @@ -46,16 +46,16 @@ jobs: - name: Terraform Init run: cd infrastructure/terraform && terraform init - # - name: Terraform Workspace - # run: | - # BRANCH_NAME=$(echo "${{ github.ref }}" | sed -e "s/^refs\/heads\///") - # cd infrastructure/terraform - # terraform workspace select ${BRANCH_NAME} || terraform workspace new ${BRANCH_NAME} + - name: Terraform Workspace + run: | + BRANCH_NAME=$(echo "${{ github.ref }}" | sed -e "s/^refs\/heads\///") + cd infrastructure/terraform + terraform workspace select ${BRANCH_NAME} || terraform workspace new ${BRANCH_NAME} - name: Terraform Plan run: | BRANCH_NAME=$(echo "${{ github.ref }}" | sed -e "s/^refs\/heads\///") - cd infrastructure/terraform && terraform plan -var-file=dev.tfvars + cd infrastructure/terraform && terraform plan -var-file=${BRANCH_NAME}.tfvars # - name: Deploy to Dev # if: github.ref == 'refs/heads/dev' From 2b01ac9f6c0b13a5afe071af7b8f857dbe48fe94 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Fri, 30 Jan 2026 14:35:56 +0000 Subject: [PATCH 05/86] safely check deployment and plan --- .github/workflows/deploy_terraform.yml | 35 ++++++++++++++------------ 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/.github/workflows/deploy_terraform.yml b/.github/workflows/deploy_terraform.yml index 16ca1068..7210a8e2 100644 --- a/.github/workflows/deploy_terraform.yml +++ b/.github/workflows/deploy_terraform.yml @@ -48,29 +48,32 @@ jobs: - name: Terraform Workspace run: | - BRANCH_NAME=$(echo "${{ github.ref }}" | sed -e "s/^refs\/heads\///") + # BRANCH_NAME=$(echo "${{ github.ref }}" | sed -e "s/^refs\/heads\///") cd infrastructure/terraform - terraform workspace select ${BRANCH_NAME} || terraform workspace new ${BRANCH_NAME} + # terraform workspace select ${BRANCH_NAME} || terraform workspace new ${BRANCH_NAME} + # Until Khalim makes a different environment for us + terraform workspace select dev - name: Terraform Plan run: | BRANCH_NAME=$(echo "${{ github.ref }}" | sed -e "s/^refs\/heads\///") - cd infrastructure/terraform && terraform plan -var-file=${BRANCH_NAME}.tfvars + cd infrastructure/terraform && terraform plan -var-file=dev.tfvars - # - name: Deploy to Dev - # if: github.ref == 'refs/heads/dev' - # run: cd infrastructure/terraform && terraform apply -var-file=dev.tfvars -auto-approve - # env: - # name: dev - - - name: Configure AWS credentials (ProdAdmin) - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.PROD_AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.PROD_AWS_SECRET_ACCESS_KEY }} - aws-region: eu-west-2 + - name: Deploy to Dev + if: github.ref == 'refs/heads/dev' + run: echo "hello world" + # run: cd infrastructure/terraform && terraform apply -var-file=dev.tfvars -auto-approve env: - AWS_PROFILE: "ProdAdmin" + name: dev + + # - name: Configure AWS credentials (ProdAdmin) + # uses: aws-actions/configure-aws-credentials@v1 + # with: + # aws-access-key-id: ${{ secrets.PROD_AWS_ACCESS_KEY_ID }} + # aws-secret-access-key: ${{ secrets.PROD_AWS_SECRET_ACCESS_KEY }} + # aws-region: eu-west-2 + # env: + # AWS_PROFILE: "ProdAdmin" # - name: Deploy to Prod # if: github.ref == 'refs/heads/prod' From 6c05b0d6a4a7918641cfa6bcf776a9a61b51ffae Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Mon, 2 Feb 2026 16:38:17 +0000 Subject: [PATCH 06/86] added terraform files and test plan --- .../actions/actions/lambda-deploy/action.yml | 86 ++++ .../actions/terraform-deploy/action.yml | 55 +++ .github/workflows/deploy_terraform.yml | 38 +- asset_list/app.py | 8 +- backend/address2UPRN/Dockerfile | 7 + backend/address2UPRN/main.py | 47 ++ backend/address2UPRN/tests/test_data.csv | 20 +- infrastructure/terraform/lamdas/backend.tf | 0 infrastructure/terraform/lamdas/dev.tfvars | 0 infrastructure/terraform/lamdas/main.tf | 0 infrastructure/terraform/lamdas/variables.tf | 0 .../terraform/modules/lambda_with_sqs/main.tf | 23 + .../modules/lambda_with_sqs/outputs.tf | 15 + .../modules/lambda_with_sqs/variables.tf | 32 ++ .../terraform/{ => shared}/dev.tfvars | 0 infrastructure/terraform/{ => shared}/main.tf | 0 .../terraform/{ => shared}/secrets.tf | 0 .../terraform/{ => shared}/variables.tf | 0 sfr/principal_pitch/2_export_data.py | 405 ++++++------------ 19 files changed, 417 insertions(+), 319 deletions(-) create mode 100644 .github/workflows/actions/actions/lambda-deploy/action.yml create mode 100644 .github/workflows/actions/actions/terraform-deploy/action.yml create mode 100644 backend/address2UPRN/Dockerfile create mode 100644 infrastructure/terraform/lamdas/backend.tf create mode 100644 infrastructure/terraform/lamdas/dev.tfvars create mode 100644 infrastructure/terraform/lamdas/main.tf create mode 100644 infrastructure/terraform/lamdas/variables.tf create mode 100644 infrastructure/terraform/modules/lambda_with_sqs/main.tf create mode 100644 infrastructure/terraform/modules/lambda_with_sqs/outputs.tf create mode 100644 infrastructure/terraform/modules/lambda_with_sqs/variables.tf rename infrastructure/terraform/{ => shared}/dev.tfvars (100%) rename infrastructure/terraform/{ => shared}/main.tf (100%) rename infrastructure/terraform/{ => shared}/secrets.tf (100%) rename infrastructure/terraform/{ => shared}/variables.tf (100%) diff --git a/.github/workflows/actions/actions/lambda-deploy/action.yml b/.github/workflows/actions/actions/lambda-deploy/action.yml new file mode 100644 index 00000000..3ca0fc8d --- /dev/null +++ b/.github/workflows/actions/actions/lambda-deploy/action.yml @@ -0,0 +1,86 @@ +name: "Build and Push Lambda Image to ECR" +description: "Reusable action for building and pushing lambda Docker image to ECR" + +inputs: + ecr_name: + description: "Lambda name / ECR repo name" + required: true + dockerfile_path: + description: "Path to Dockerfile" + required: true + ecr_tf_dir: + description: "Path to ECR terraform directory" + required: true + lambda_tf_dir: + description: "Path to Lambda terraform directory" + required: true + aws-access-key-id: + description: "AWS access key" + required: true + aws-secret-access-key: + description: "AWS secret key" + required: true + aws-region: + description: "AWS region" + required: true + git-sha: + description: "Git commit SHA" + required: true + git-ref: + description: "Git ref name" + required: true + +runs: + using: "composite" + steps: + - uses: actions/checkout@v4 + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-access-key-id: ${{ inputs.aws-access-key-id }} + aws-secret-access-key: ${{ inputs.aws-secret-access-key }} + aws-region: ${{ inputs.aws-region }} + + - name: Log in to Amazon ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@v2 + + - name: Deploy ECR + uses: ./.github/workflows/actions/terraform-deploy + with: + working_directory: ${{ inputs.ecr_tf_dir }} + aws-access-key-id: ${{ inputs.aws-access-key-id }} + aws-secret-access-key: ${{ inputs.aws-secret-access-key }} + aws-region: ${{ inputs.aws-region }} + - name: Set Docker image tag + id: set_tag + shell: bash + run: | + SHORT_SHA=$(echo "${{ inputs.git-sha }}" | cut -c1-7) + BRANCH=$(echo "${{ inputs.git-ref }}" | tr '/' '-') + TAG="${BRANCH}-${SHORT_SHA}" + echo "IMAGE_TAG=${TAG}" >> $GITHUB_ENV + echo "tag=$TAG" >> $GITHUB_OUTPUT + + - name: Build and push Docker image + shell: bash + run: | + IMAGE_URI=${{ steps.login-ecr.outputs.registry }}/${{ inputs.ecr_name }}:${{ steps.set_tag.outputs.tag }} + echo "Building Docker image for ${{ inputs.ecr_name }}..." + docker build -t $IMAGE_URI -f ${{ inputs.dockerfile_path }} . + + echo "Pushing to ECR..." + docker push $IMAGE_URI + + - name: Deploy Lambda + uses: ./.github/workflows/actions/terraform-deploy + with: + working_directory: ${{ inputs.lambda_tf_dir }} + aws-access-key-id: ${{ inputs.aws-access-key-id }} + aws-secret-access-key: ${{ inputs.aws-secret-access-key }} + aws-region: ${{ inputs.aws-region }} + lambda-image-tag: ${{ steps.set_tag.outputs.tag }} + + + diff --git a/.github/workflows/actions/actions/terraform-deploy/action.yml b/.github/workflows/actions/actions/terraform-deploy/action.yml new file mode 100644 index 00000000..56133299 --- /dev/null +++ b/.github/workflows/actions/actions/terraform-deploy/action.yml @@ -0,0 +1,55 @@ +name: "Terraform Plan Shared Config" +description: "Plans shared Terraform config for Lambdas" + +inputs: + working_directory: + description: "Directory containing Terraform config" + required: true + aws-access-key-id: + description: "AWS access key" + required: true + aws-secret-access-key: + description: "AWS secret key" + required: true + aws-region: + description: "AWS region" + required: true + lambda-image-tag: + description: "Tag of the Lambda image (e.g., GitHub SHA)" + required: false + +runs: + using: "composite" + steps: + - uses: actions/checkout@v4 + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-access-key-id: ${{ inputs.aws-access-key-id }} + aws-secret-access-key: ${{ inputs.aws-secret-access-key }} + aws-region: ${{ inputs.aws-region }} + + - name: Setup Terraform + uses: hashicorp/setup-terraform@v3 + + - name: Terraform Init + working-directory: ${{ inputs.working_directory }} + shell: bash + run: terraform init -reconfigure + + - name: Terraform Plan + working-directory: ${{ inputs.working_directory }} + shell: bash + run: | + if [ -n "${{ inputs.lambda-image-tag }}" ]; then + terraform plan -out=tfplan -var="lambda_image_tag=${{ inputs.lambda-image-tag }}" + else + terraform plan -out=tfplan + fi + + - name: Terraform Apply + working-directory: ${{ inputs.working_directory }} + shell: bash + run: terraform apply -auto-approve tfplan + diff --git a/.github/workflows/deploy_terraform.yml b/.github/workflows/deploy_terraform.yml index 7210a8e2..fe52a1e2 100644 --- a/.github/workflows/deploy_terraform.yml +++ b/.github/workflows/deploy_terraform.yml @@ -43,40 +43,16 @@ jobs: env: AWS_PROFILE: "DevAdmin" + # Deploy shared terrform things - name: Terraform Init - run: cd infrastructure/terraform && terraform init + run: cd infrastructure/terraform/shared && terraform init - name: Terraform Workspace run: | - # BRANCH_NAME=$(echo "${{ github.ref }}" | sed -e "s/^refs\/heads\///") - cd infrastructure/terraform - # terraform workspace select ${BRANCH_NAME} || terraform workspace new ${BRANCH_NAME} - # Until Khalim makes a different environment for us - terraform workspace select dev + cd infrastructure/terraform/shared + terraform workspace select dev || terraform workspace new dev - - name: Terraform Plan + - name: Terraform Plan (shared) run: | - BRANCH_NAME=$(echo "${{ github.ref }}" | sed -e "s/^refs\/heads\///") - cd infrastructure/terraform && terraform plan -var-file=dev.tfvars - - - name: Deploy to Dev - if: github.ref == 'refs/heads/dev' - run: echo "hello world" - # run: cd infrastructure/terraform && terraform apply -var-file=dev.tfvars -auto-approve - env: - name: dev - - # - name: Configure AWS credentials (ProdAdmin) - # uses: aws-actions/configure-aws-credentials@v1 - # with: - # aws-access-key-id: ${{ secrets.PROD_AWS_ACCESS_KEY_ID }} - # aws-secret-access-key: ${{ secrets.PROD_AWS_SECRET_ACCESS_KEY }} - # aws-region: eu-west-2 - # env: - # AWS_PROFILE: "ProdAdmin" - - # - name: Deploy to Prod - # if: github.ref == 'refs/heads/prod' - # run: cd infrastructure/terraform && terraform apply -var-file=prod.tfvars -auto-approve - # env: - # name: prod + cd infrastructure/terraform/shared + terraform plan -var-file=dev.tfvars \ No newline at end of file diff --git a/asset_list/app.py b/asset_list/app.py index 1c7200fd..8b3abefb 100644 --- a/asset_list/app.py +++ b/asset_list/app.py @@ -72,21 +72,21 @@ def app(): data_folder = "/workspaces/model/asset_list" data_filename = "assets.xlsx" sheet_name = "Sheet1" - postcode_column = "Post Code" + postcode_column = "POSTCODE" address1_column = None address1_method = "house_number_extraction" - fulladdress_column = "User Input" + fulladdress_column = "ADDRESS" address_cols_to_concat = None missing_postcodes_method = None landlord_year_built = None landlord_os_uprn = "UPRN" landlord_property_type = None - landlord_built_form = None + landlord_built_form = "BUILD FORM" landlord_wall_construction = None landlord_roof_construction = None landlord_heating_system = None landlord_existing_pv = None - landlord_property_id = "LLUPRN" + landlord_property_id = "UPRN" landlord_sap = None outcomes_filename = None outcomes_sheetname = None diff --git a/backend/address2UPRN/Dockerfile b/backend/address2UPRN/Dockerfile new file mode 100644 index 00000000..d7485a3f --- /dev/null +++ b/backend/address2UPRN/Dockerfile @@ -0,0 +1,7 @@ +FROM public.ecr.aws/lambda/python:3.10 + +# Copy function code +COPY app.py ${LAMBDA_TASK_ROOT} + +# Set the handler +CMD ["main.handler"] diff --git a/backend/address2UPRN/main.py b/backend/address2UPRN/main.py index 58b25d74..9d27a5ce 100644 --- a/backend/address2UPRN/main.py +++ b/backend/address2UPRN/main.py @@ -14,6 +14,9 @@ EPC_AUTH_TOKEN = os.getenv( "EPC_AUTH_TOKEN", ) +if EPC_AUTH_TOKEN is None: + raise RuntimeError("EPC_AUTH_TOKEN not defined in env") + import re from difflib import SequenceMatcher from typing import Set @@ -38,6 +41,34 @@ def levenshtein(a: str, b: str) -> float: def tokenise(s: str) -> Set[str]: return set(s.split()) + def extract_building_number(s: str) -> str | None: + """ + Extract the main building number (NOT flat/unit). + Assumes formats like: + - '42 moreton road' + - 'flat 3 42 moreton road' + """ + tokens = s.split() + + # remove flat/unit context + cleaned = [] + skip_next = False + for t in tokens: + if t in ("flat", "apt", "apartment", "unit"): + skip_next = True + continue + if skip_next: + skip_next = False + continue + cleaned.append(t) + + # first remaining number is building number + for t in cleaned: + if re.fullmatch(r"\d+[a-z]?", t): + return t + + return None + a_norm = normalise_address(a) b_norm = normalise_address(b) @@ -52,6 +83,13 @@ def levenshtein(a: str, b: str) -> float: if nums_a and nums_b and nums_a.isdisjoint(nums_b): return 0.0 + # 🔒 HARD GUARD: building number must match + bld_a = extract_building_number(a_norm) + bld_b = extract_building_number(b_norm) + + if bld_a and bld_b and bld_a != bld_b: + return 0.0 + # --- order-sensitive flat/building guard --- seq_a = extract_number_sequence(a_norm) seq_b = extract_number_sequence(b_norm) @@ -418,6 +456,10 @@ def run_all_test(): get_uprn("46 Oswald Street", "E5 0BT"), False ) # this one return "flat 1, in 1 semley gate" get_uprn_candidates(get_epc_data_with_postcode("e5 0bt"), "48 Oswald Street") + get_uprn_candidates( + get_epc_data_with_postcode("Cr2 7dl"), + "FLAT 3; 42 MORETON ROAD, SOUTH CROYDON, SURREY", + ) if __name__ == "__main__": @@ -511,6 +553,11 @@ if __name__ == "__main__": ) +def handler(event, context): + print("hello world") + return {"statusCode": 200, "body": "hello world"} + + # TO do function dispatcher, # get_uprn_candidates(get_epc_data_with_postcode("E9 5NH"),"Flat 1, 5 Semley Gate" and Flat 5, 1 Semley Gate) diff --git a/backend/address2UPRN/tests/test_data.csv b/backend/address2UPRN/tests/test_data.csv index 91bc49e1..ee23813b 100644 --- a/backend/address2UPRN/tests/test_data.csv +++ b/backend/address2UPRN/tests/test_data.csv @@ -115,11 +115,16 @@ FLAT 43 Goodstone Court,HA1 4FL,10070269095 8 Genteel House Samara Drive,UB1 1FJ,12189842 9 Genteel House Samara Drive,UB1 1FJ,12189843 10 Genteel House Samara Drive,UB1 1FJ,12189844 -1 ASH TREE HOUSE,SE5 0TE,10009803979 -3 ASH TREE HOUSE,SE5 0TE,10009803981 -5 ASH TREE HOUSE,SE5 0TE,10009803983 -8 ASH TREE HOUSE,SE5 0TE,10009803986 -12 ASH TREE HOUSE,SE5 0TE,10009803990 +1 ASH TREE HOUSE,SE5 0TE,None +"Flat 1 Ash Tree House, 2, Thompson Avenue",SE5 0TE,10009803979 +3 ASH TREE HOUSE,SE5 0TE,None +Flat 3 ASH TREE HOUSE,SE5 0TE,10009803981 +5 ASH TREE HOUSE,SE5 0TE,None +Flat 5 ASH TREE HOUSE,SE5 0TE,10009803983 +Flat 8 ASH TREE HOUSE,SE5 0TE,10009803986 +8 ASH TREE HOUSE,SE5 0TE,None +Flat 12 ASH TREE HOUSE,SE5 0TE,10009803990 +12 ASH TREE HOUSE,SE5 0TE,None FLAT 1 599 HARROW ROAD,W10 4RA,217113930 FLAT 2 599 HARROW ROAD,W10 4RA,217113931 FLAT 3 599 HARROW ROAD,W10 4RA,None @@ -332,7 +337,7 @@ FLAT B 158 LEAHURST ROAD,SE13 5NL,100021976974 138a Victoria Square,M4 5FA,77211289 139a Victoria Square,M4 5FA,77211290 140a Victoria Square,M4 5FA,77211291 -141a Victoria Square,M4 5FA,None +141a Victoria Square,M4 5FA,77211292 142a Victoria Square,M4 5FA,77211293 143a Victoria Square,M4 5FA,77211294 144a Victoria Square,M4 5FA,77211295 @@ -357,4 +362,5 @@ FLAT B 158 LEAHURST ROAD,SE13 5NL,100021976974 163a Victoria Square,M4 5FA,77211314 164a Victoria Square,M4 5FA,77211315 165a Victoria Square,M4 5FA,77211316 -166a Victoria Square,M4 5FA,None \ No newline at end of file +166a Victoria Square,M4 5FA,None +"FLAT 3; 42 MORETON ROAD, SOUTH CROYDON, SURREY",CR2 7DL,None \ No newline at end of file diff --git a/infrastructure/terraform/lamdas/backend.tf b/infrastructure/terraform/lamdas/backend.tf new file mode 100644 index 00000000..e69de29b diff --git a/infrastructure/terraform/lamdas/dev.tfvars b/infrastructure/terraform/lamdas/dev.tfvars new file mode 100644 index 00000000..e69de29b diff --git a/infrastructure/terraform/lamdas/main.tf b/infrastructure/terraform/lamdas/main.tf new file mode 100644 index 00000000..e69de29b diff --git a/infrastructure/terraform/lamdas/variables.tf b/infrastructure/terraform/lamdas/variables.tf new file mode 100644 index 00000000..e69de29b diff --git a/infrastructure/terraform/modules/lambda_with_sqs/main.tf b/infrastructure/terraform/modules/lambda_with_sqs/main.tf new file mode 100644 index 00000000..1b4e1847 --- /dev/null +++ b/infrastructure/terraform/modules/lambda_with_sqs/main.tf @@ -0,0 +1,23 @@ +resource "aws_sqs_queue" "this" { + name = "${var.name}-queue" + tags = var.tags +} + +resource "aws_lambda_function" "this" { + function_name = var.name + role = var.lambda_role_arn + + package_type = "Image" + image_uri = var.image_uri + + timeout = var.timeout + + tags = var.tags +} + +resource "aws_lambda_event_source_mapping" "this" { + event_source_arn = aws_sqs_queue.this.arn + function_name = aws_lambda_function.this.arn + + batch_size = var.sqs_batch_size +} diff --git a/infrastructure/terraform/modules/lambda_with_sqs/outputs.tf b/infrastructure/terraform/modules/lambda_with_sqs/outputs.tf new file mode 100644 index 00000000..dc755850 --- /dev/null +++ b/infrastructure/terraform/modules/lambda_with_sqs/outputs.tf @@ -0,0 +1,15 @@ +output "lambda_name" { + value = aws_lambda_function.this.function_name +} + +output "lambda_arn" { + value = aws_lambda_function.this.arn +} + +output "sqs_queue_url" { + value = aws_sqs_queue.this.url +} + +output "sqs_queue_arn" { + value = aws_sqs_queue.this.arn +} diff --git a/infrastructure/terraform/modules/lambda_with_sqs/variables.tf b/infrastructure/terraform/modules/lambda_with_sqs/variables.tf new file mode 100644 index 00000000..8ac24942 --- /dev/null +++ b/infrastructure/terraform/modules/lambda_with_sqs/variables.tf @@ -0,0 +1,32 @@ +variable "name" { + description = "Base name for lambda and related resources" + type = string +} + +variable "image_uri" { + description = "ECR image URI with tag" + type = string +} + +variable "lambda_role_arn" { + description = "IAM role ARN for Lambda execution" + type = string +} + +variable "timeout" { + description = "Lambda timeout in seconds" + type = number + default = 10 +} + +variable "sqs_batch_size" { + description = "Number of SQS messages per batch" + type = number + default = 1 +} + +variable "tags" { + description = "Tags to apply to resources" + type = map(string) + default = {} +} diff --git a/infrastructure/terraform/dev.tfvars b/infrastructure/terraform/shared/dev.tfvars similarity index 100% rename from infrastructure/terraform/dev.tfvars rename to infrastructure/terraform/shared/dev.tfvars diff --git a/infrastructure/terraform/main.tf b/infrastructure/terraform/shared/main.tf similarity index 100% rename from infrastructure/terraform/main.tf rename to infrastructure/terraform/shared/main.tf diff --git a/infrastructure/terraform/secrets.tf b/infrastructure/terraform/shared/secrets.tf similarity index 100% rename from infrastructure/terraform/secrets.tf rename to infrastructure/terraform/shared/secrets.tf diff --git a/infrastructure/terraform/variables.tf b/infrastructure/terraform/shared/variables.tf similarity index 100% rename from infrastructure/terraform/variables.tf rename to infrastructure/terraform/shared/variables.tf diff --git a/sfr/principal_pitch/2_export_data.py b/sfr/principal_pitch/2_export_data.py index f0fc5cd1..ae807654 100644 --- a/sfr/principal_pitch/2_export_data.py +++ b/sfr/principal_pitch/2_export_data.py @@ -7,20 +7,29 @@ import numpy as np from backend.app.utils import sap_to_epc from sqlalchemy.orm import sessionmaker from backend.app.db.connection import db_engine, db_read_session -from backend.app.db.models.recommendations import Recommendation, Plan, PlanRecommendations, RecommendationMaterials -from backend.app.db.models.portfolio import PropertyModel, PropertyDetailsEpcModel, PropertyDetailsSpatial +from backend.app.db.models.recommendations import ( + Recommendation, + Plan, + PlanRecommendations, + RecommendationMaterials, +) +from backend.app.db.models.portfolio import ( + PropertyModel, + PropertyDetailsEpcModel, + PropertyDetailsSpatial, +) from backend.app.db.functions.materials_functions import get_materials from collections import defaultdict from sqlalchemy import func # PORTFOLIO_ID = 206 # SCENARIOS = [389] -PORTFOLIO_ID = 485 # Peabody +PORTFOLIO_ID = 502 # Peabody SCENARIOS = [ - 970, + 986, ] scenario_names = { - 970: "EPC C - No solid floor, EQI, IWI", + 986: "EPC C", } @@ -31,22 +40,26 @@ def get_data(portfolio_id, scenario_ids): # -------------------- # Properties # -------------------- - properties_query = session.query( - PropertyModel, - PropertyDetailsEpcModel - ).join( - PropertyDetailsEpcModel, - PropertyModel.id == PropertyDetailsEpcModel.property_id - ).filter( - PropertyModel.portfolio_id == portfolio_id - ).all() + properties_query = ( + session.query(PropertyModel, PropertyDetailsEpcModel) + .join( + PropertyDetailsEpcModel, + PropertyModel.id == PropertyDetailsEpcModel.property_id, + ) + .filter(PropertyModel.portfolio_id == portfolio_id) + .all() + ) properties_data = [ { - **{col.name: getattr(p.PropertyModel, col.name) - for col in PropertyModel.__table__.columns}, - **{col.name: getattr(p.PropertyDetailsEpcModel, col.name) - for col in PropertyDetailsEpcModel.__table__.columns}, + **{ + col.name: getattr(p.PropertyModel, col.name) + for col in PropertyModel.__table__.columns + }, + **{ + col.name: getattr(p.PropertyDetailsEpcModel, col.name) + for col in PropertyDetailsEpcModel.__table__.columns + }, } for p in properties_query ] @@ -58,13 +71,10 @@ def get_data(portfolio_id, scenario_ids): session.query( Plan.scenario_id, Plan.property_id, - func.max(Plan.created_at).label("latest_created_at") + func.max(Plan.created_at).label("latest_created_at"), ) .filter(Plan.scenario_id.in_(scenario_ids)) - .group_by( - Plan.scenario_id, - Plan.property_id - ) + .group_by(Plan.scenario_id, Plan.property_id) .subquery() ) @@ -76,9 +86,9 @@ def get_data(portfolio_id, scenario_ids): session.query(Plan) .join( latest_plans_subq, - (Plan.scenario_id == latest_plans_subq.c.scenario_id) & - (Plan.property_id == latest_plans_subq.c.property_id) & - (Plan.created_at == latest_plans_subq.c.latest_created_at) + (Plan.scenario_id == latest_plans_subq.c.scenario_id) + & (Plan.property_id == latest_plans_subq.c.property_id) + & (Plan.created_at == latest_plans_subq.c.latest_created_at), ) .all() ) @@ -103,28 +113,29 @@ def get_data(portfolio_id, scenario_ids): # -------------------- # Recommendations (NO materials yet) # -------------------- - recommendations_query = session.query( - Recommendation, - Plan.scenario_id, - PlanRecommendations.plan_id - ).join( - PlanRecommendations, - Recommendation.id == PlanRecommendations.recommendation_id - ).join( - Plan, - Plan.id == PlanRecommendations.plan_id - ).filter( - PlanRecommendations.plan_id.in_(plan_ids), - Recommendation.default.is_(True), - Recommendation.already_installed.is_(False) - ).all() + recommendations_query = ( + session.query(Recommendation, Plan.scenario_id, PlanRecommendations.plan_id) + .join( + PlanRecommendations, + Recommendation.id == PlanRecommendations.recommendation_id, + ) + .join(Plan, Plan.id == PlanRecommendations.plan_id) + .filter( + PlanRecommendations.plan_id.in_(plan_ids), + Recommendation.default.is_(True), + Recommendation.already_installed.is_(False), + ) + .all() + ) recommendations_data = [ { - **{col.name: getattr(r.Recommendation, col.name) - for col in Recommendation.__table__.columns}, + **{ + col.name: getattr(r.Recommendation, col.name) + for col in Recommendation.__table__.columns + }, "scenario_id": r.scenario_id, - "materials": [] # placeholder + "materials": [], # placeholder } for r in recommendations_query ] @@ -134,23 +145,25 @@ def get_data(portfolio_id, scenario_ids): # -------------------- # Recommendation materials (SEPARATE QUERY) # -------------------- - materials_query = session.query( - RecommendationMaterials - ).filter( - RecommendationMaterials.recommendation_id.in_(recommendation_ids) - ).all() + materials_query = ( + session.query(RecommendationMaterials) + .filter(RecommendationMaterials.recommendation_id.in_(recommendation_ids)) + .all() + ) # Group materials by recommendation_id materials_by_recommendation = defaultdict(list) for m in materials_query: - materials_by_recommendation[m.recommendation_id].append({ - "material_id": m.material_id, - "depth": m.depth, - "quantity": m.quantity, - "quantity_unit": m.quantity_unit, - "estimated_cost": m.estimated_cost, - }) + materials_by_recommendation[m.recommendation_id].append( + { + "material_id": m.material_id, + "depth": m.depth, + "quantity": m.quantity, + "quantity_unit": m.quantity_unit, + "estimated_cost": m.estimated_cost, + } + ) # Attach materials safely (no filtering side effects) for r in recommendations_data: @@ -161,7 +174,9 @@ def get_data(portfolio_id, scenario_ids): return properties_data, plans_data, recommendations_data -properties_data, plans_data, recommendations_data = get_data(portfolio_id=PORTFOLIO_ID, scenario_ids=SCENARIOS) +properties_data, plans_data, recommendations_data = get_data( + portfolio_id=PORTFOLIO_ID, scenario_ids=SCENARIOS +) properties_df = pd.DataFrame(properties_data) plans_df = pd.DataFrame(plans_data) @@ -172,10 +187,8 @@ with db_read_session() as session: materials = pd.DataFrame(materials) -material_lookup = ( - materials - .set_index("id")[["type", "includes_battery"]] - .to_dict("index") +material_lookup = materials.set_index("id")[["type", "includes_battery"]].to_dict( + "index" ) @@ -189,14 +202,14 @@ def has_solar_with_battery(materials_list): return False -recommendations_df["has_solar_with_battery"] = ( - recommendations_df["materials"].apply(has_solar_with_battery) +recommendations_df["has_solar_with_battery"] = recommendations_df["materials"].apply( + has_solar_with_battery ) recommendations_df["measure_type"] = np.where( recommendations_df["has_solar_with_battery"] == True, recommendations_df["measure_type"] + "_with_battery", - recommendations_df["measure_type"] + recommendations_df["measure_type"], ) # Adjust material type to indicate if there is a battery included @@ -211,50 +224,67 @@ from utils.s3 import read_csv_from_s3, read_excel_from_s3 for scenario_id in SCENARIOS: # Get recs for this scenario - recommended_measures_df = recommendations_df[recommendations_df["scenario_id"] == scenario_id][ - ["property_id", "measure_type", "estimated_cost", "default"] + recommended_measures_df = recommendations_df[ + recommendations_df["scenario_id"] == scenario_id + ][["property_id", "measure_type", "estimated_cost", "default"]] + recommended_measures_df = recommended_measures_df[ + recommended_measures_df["default"] ] - recommended_measures_df = recommended_measures_df[recommended_measures_df["default"]] recommended_measures_df = recommended_measures_df.drop(columns=["default"]) - post_install_sap = recommendations_df[recommendations_df["scenario_id"] == scenario_id][ - ["property_id", "default", "sap_points"]] + post_install_sap = recommendations_df[ + recommendations_df["scenario_id"] == scenario_id + ][["property_id", "default", "sap_points"]] post_install_sap = post_install_sap[post_install_sap["default"]] # Sum up the sap points by property id - post_install_sap = post_install_sap.groupby(["property_id"])[["sap_points"]].sum().reset_index() + post_install_sap = ( + post_install_sap.groupby(["property_id"])[["sap_points"]].sum().reset_index() + ) # Find dupes by property id and measure type - dupes = recommended_measures_df.duplicated(subset=["property_id", "measure_type"], keep=False) + dupes = recommended_measures_df.duplicated( + subset=["property_id", "measure_type"], keep=False + ) dupe_df = recommended_measures_df[dupes] if dupe_df.shape: # Drop dupes - happened due to a funny bug recommended_measures_df = recommended_measures_df.drop_duplicates( - subset=["property_id", "measure_type"], keep='first' + subset=["property_id", "measure_type"], keep="first" ) recommendations_measures_pivot = recommended_measures_df.pivot( - index='property_id', - columns='measure_type', - values='estimated_cost' + index="property_id", columns="measure_type", values="estimated_cost" ) recommendations_measures_pivot = recommendations_measures_pivot.reset_index() # Total cost is the row sum, excluding the property_id column - recommendations_measures_pivot["total_retrofit_cost"] = recommendations_measures_pivot.drop( - columns=["property_id"] - ).sum(axis=1) + recommendations_measures_pivot["total_retrofit_cost"] = ( + recommendations_measures_pivot.drop(columns=["property_id"]).sum(axis=1) + ) - df = properties_df[ - [ - "landlord_property_id", "property_id", "uprn", "address", "postcode", "property_type", "walls", "roof", - "heating", "windows", "current_epc_rating", "current_sap_points", "total_floor_area", "number_of_rooms", - "id" + df = ( + properties_df[ + [ + "landlord_property_id", + "property_id", + "uprn", + "address", + "postcode", + "property_type", + "walls", + "roof", + "heating", + "windows", + "current_epc_rating", + "current_sap_points", + "total_floor_area", + "number_of_rooms", + "id", + ] ] - ].merge( - recommendations_measures_pivot, how="left", on="property_id" - ).merge( - post_install_sap, how="left", on="property_id" + .merge(recommendations_measures_pivot, how="left", on="property_id") + .merge(post_install_sap, how="left", on="property_id") ) # df = df.drop(columns=["property_id"]) @@ -262,21 +292,25 @@ for scenario_id in SCENARIOS: df["predicted_post_works_sap"] = df["current_sap_points"] + df["sap_points"] df["predicted_post_works_sap"] = df["predicted_post_works_sap"] - df["predicted_post_works_epc"] = df["predicted_post_works_sap"].apply(lambda x: sap_to_epc(x)) + df["predicted_post_works_epc"] = df["predicted_post_works_sap"].apply( + lambda x: sap_to_epc(x) + ) df["uprn"] = df["uprn"].astype(str) relevant_plans = plans_df[plans_df["scenario_id"] == scenario_id] df2 = df.merge( - relevant_plans[["property_id", "post_sap_points", "post_epc_rating"]], how="left", on="property_id", - suffixes=("", "_plan") + relevant_plans[["property_id", "post_sap_points", "post_epc_rating"]], + how="left", + on="property_id", + suffixes=("", "_plan"), ) print(df2["predicted_post_works_epc"].value_counts()) print(df2["post_epc_rating"].value_counts()) z = df2[ - (df2["predicted_post_works_epc"] != "D") & - (df2["post_epc_rating"].astype(str) == "Epc.D") - ] + (df2["predicted_post_works_epc"] != "D") + & (df2["post_epc_rating"].astype(str) == "Epc.D") + ] df2["predicted_post_works_epc"].value_counts() df2["post_epc_rating"].astype(str).value_counts() @@ -291,189 +325,6 @@ for scenario_id in SCENARIOS: df[df["predicted_post_works_sap"] == ""] # Create excel to store to -<<<<<<< HEAD - filename = (f"{scenario_names[scenario_id]} - 20250113 final.xlsx") + filename = f"{scenario_names[scenario_id]} - 20250113 final.xlsx" with pd.ExcelWriter(filename) as writer: df.to_excel(writer, sheet_name="properties", index=False) -======= - filename = ("/Users/khalimconn-kowlessar/Documents/hestia/Customers/Peabody/Nov 2025 Consulting " - f"Project/Final SAL/scenarios/{scenario_names[scenario_id]} - 20250114 final.xlsx") - with pd.ExcelWriter(filename) as writer: - df.to_excel(writer, sheet_name="properties", index=False) - - -# asset_list = pd.DataFrame(asset_list) -# asset_list = asset_list.rename( -# columns={ -# "postcode": "domna_postcode" -# } -# ) -# if "domna_full_address": -# # For Peabody -# asset_list["domna_full_address"] = asset_list["domna_address_1"] -# -# asset_list = asset_list[["domna_full_address", "domna_postcode", "epc_os_uprn", ]].copy() -# asset_list = asset_list.rename(columns={"epc_os_uprn": "uprn"}) -# asset_list["uprn"] = asset_list["uprn"].astype("Int64").astype(str) -# asset_list = asset_list.merge( -# df.drop(columns=["address", "postcode", "property_type", "total_floor_area"]), -# how="left", -# on="uprn" -# ) - - -# Get conservation area data from property details spatial. based on the UPRNs -def get_conservation_area_data(uprns): - session = sessionmaker(bind=db_engine)() - session.begin() - - # Query to get conservation area data - spatial_query = session.query( - PropertyDetailsSpatial - ).filter( - PropertyDetailsSpatial.uprn.in_(uprns) # Filter by UPRNs - ).all() - - # Transform spatial data to include all fields dynamically - spatial_data = [ - {col.name: getattr(spatial, col.name) for col in PropertyDetailsSpatial.__table__.columns} - for spatial in spatial_query - ] - - session.close() - return pd.DataFrame(spatial_data) - - -uprns = asset_list[ - ~pd.isna(asset_list["uprn"]) & (asset_list["uprn"] != "") - ]["uprn"].astype(int).unique().tolist() -conservation_area_data = get_conservation_area_data(uprns) -conservation_area_data["uprn"] = conservation_area_data["uprn"].astype(str) -asset_list = asset_list.merge( - conservation_area_data[["uprn", "conservation_status", "is_listed_building", "is_heritage_building"]], - how="left", - on="uprn" -) - -# For exporting -df.to_excel( - "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Lincs Rural/EPC C -without floors proposed measures - " - "with ID.xlsx", - index=False -) -# asset_list.to_excel( -# "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Lincs Rural/epc_measures.xlsx", -# index=False -# ) - -condition_costs = pd.read_excel( - "/Users/khalimconn-kowlessar/Documents/hestia/sfr/Spring JV/Condition costs.xlsx", - sheet_name="Prices - Khalim", - header=35 -) -# Remove unnamed columns and reset index -condition_costs = condition_costs.loc[:, ~condition_costs.columns.str.contains('^Unnamed')] -condition_costs = condition_costs.reset_index(drop=True) - - -# We now estimate condition cost -def simulate_condition(asset_list, condition_costs): - """ - This function is for testing, and will simulate condition cost from 1-10 for each property to see what the - costing array looks like. - :param df: - :return: - """ - - condition_df = [] - for _, row in asset_list.iterrows(): - - n_bathrooms = row["bathrooms"] - - conditions = {} - for condition in reversed(range(1, 11)): - condition_cost = condition_costs[ - condition_costs["Condition"] == condition - ].drop(columns=["Condition"]).iloc[0] - - # Each cost is scaled by floor area - condition_cost = condition_cost * row["total_floor_area"] - condition_cost["Bathroom"] = condition_cost["Bathroom"] * n_bathrooms - - total_condition_cost = condition_cost.sum() - conditions["Condition " + str(condition)] = (total_condition_cost) - - condition_df.append( - { - "uprn": row["uprn"], - **conditions - } - ) - - condition_df = pd.DataFrame(condition_df) - - asset_list = asset_list.merge( - condition_df, - how="left", - on="uprn" - ) - - return asset_list - - -# asset_list = simulate_condition(asset_list, condition_costs) - -# We calculate the condition cost based on the condition -for _, row in asset_list.iterrows(): - - condition = row["condition_score"] - if condition in [None, ""]: - continue - condition = int(float(condition)) - - condition_cost = condition_costs[ - condition_costs["Condition"] == condition - ].drop(columns=["Condition"]).iloc[0] - - # Each cost is scaled by floor area - condition_cost = condition_cost * float(row["total_floor_area"]) - n_bathrooms = row["n_bathrooms"] - condition_cost["Bathroom"] = condition_cost["Bathroom"] * float(n_bathrooms) - - total_condition_cost = condition_cost.sum() - asset_list.loc[asset_list["uprn"] == row["uprn"], "domna_condition_cost"] = total_condition_cost - -# Store output -asset_list.to_excel( - "/Users/khalimconn-kowlessar/Documents/hestia/sfr/Spring JV/20250624_portfolio_retrofit_packages.xlsx", - index=False -) - -condition_cost_comparison = asset_list[ - ["condition_score", "decoration_sum_min ", "decoration_sum_max", "domna_condition_cost"] -] - -# Testing -plans_df.head() - -example = pd.read_excel( - "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Peabody/Nov 2025 Consulting Project/Final " - "SAL/scenarios/EPC C - no solid floor, no EWI or IWI, ashp 3.0 - 20250114 final.xlsx" -) - -plans_df2 = plans_df.merge( - properties_df[["property_id", "landlord_property_id"]], - left_on="property_id", - right_on="property_id", - how="left" -) - -plans_df2 = plans_df2[plans_df2["scenario_id"] == 909] - -dupes = plans_df2[plans_df2["property_id"].duplicated()] - -# merge on plans -example = example.merge( - plans_df, how="left", -) ->>>>>>> 3874da6177cbcc37f7a488bec0a06e387906653c From 864828304384f1dc7668b2ceb317e79a57a15909 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Mon, 2 Feb 2026 16:45:06 +0000 Subject: [PATCH 07/86] modules redirectory --- infrastructure/terraform/shared/main.tf | 60 ++++++++++++------------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/infrastructure/terraform/shared/main.tf b/infrastructure/terraform/shared/main.tf index 5a67b793..2322557d 100644 --- a/infrastructure/terraform/shared/main.tf +++ b/infrastructure/terraform/shared/main.tf @@ -91,101 +91,101 @@ resource "aws_db_instance" "default" { # Set up the bucket that recieve the csv uploads of epc to be retrofit module "s3_presignable_bucket" { - source = "./modules/s3_presignable_bucket" + source = "../modules/s3_presignable_bucket" bucketname = "retrofit-plan-inputs-${var.stage}" environment = var.stage allowed_origins = var.allowed_origins } module "s3_due_considerations_bucket" { - source = "./modules/s3_presignable_bucket" + source = "../modules/s3_presignable_bucket" bucketname = "retrofit-due-considerations-${var.stage}" environment = var.stage allowed_origins = var.allowed_origins } module "s3_eco_spreadseet_bucket" { - source = "./modules/s3_presignable_bucket" + source = "../modules/s3_presignable_bucket" bucketname = "retrofit-eco-spreadsheet-${var.stage}" environment = var.stage allowed_origins = var.allowed_origins } module "s3" { - source = "./modules/s3" + source = "../modules/s3" bucketname = "retrofit-datalake-${var.stage}" allowed_origins = var.allowed_origins } module "model_directory" { - source = "./modules/s3" + source = "../modules/s3" bucketname = "retrofit-model-directory-${var.stage}" allowed_origins = var.allowed_origins } module "retrofit_sap_predictions" { - source = "./modules/s3" + source = "../modules/s3" bucketname = "retrofit-sap-predictions-${var.stage}" allowed_origins = var.allowed_origins } module "retrofit_sap_data" { - source = "./modules/s3" + source = "../modules/s3" bucketname = "retrofit-data-${var.stage}" allowed_origins = var.allowed_origins } module "retrofit_carbon_predictions" { - source = "./modules/s3" + source = "../modules/s3" bucketname = "retrofit-carbon-predictions-${var.stage}" allowed_origins = var.allowed_origins } module "retrofit_heat_predictions" { - source = "./modules/s3" + source = "../modules/s3" bucketname = "retrofit-heat-predictions-${var.stage}" allowed_origins = var.allowed_origins } module "retrofit_lighting_cost_predictions" { - source = "./modules/s3" + source = "../modules/s3" bucketname = "retrofit-lighting-cost-predictions-${var.stage}" allowed_origins = var.allowed_origins } module "retrofit_heating_cost_predictions" { - source = "./modules/s3" + source = "../modules/s3" bucketname = "retrofit-heating-cost-predictions-${var.stage}" allowed_origins = var.allowed_origins } module "retrofit_hot_water_cost_predictions" { - source = "./modules/s3" + source = "../modules/s3" bucketname = "retrofit-hot-water-cost-predictions-${var.stage}" allowed_origins = var.allowed_origins } module "retrofit_heating_kwh_predictions" { - source = "./modules/s3" + source = "../modules/s3" bucketname = "retrofit-heating-kwh-predictions-${var.stage}" allowed_origins = var.allowed_origins } module "retrofit_hotwater_kwh_predictions" { - source = "./modules/s3" + source = "../modules/s3" bucketname = "retrofit-hotwater-kwh-predictions-${var.stage}" allowed_origins = var.allowed_origins } module "retrofit_sap_baseline_predictions" { - source = "./modules/s3" + source = "../modules/s3" bucketname = "retrofit-sap-baseline-predictions-${var.stage}" allowed_origins = var.allowed_origins } // We make this bucket presignable, because we want to generate download links for the frontend module "retrofit_energy_assessments" { - source = "./modules/s3_presignable_bucket" + source = "../modules/s3_presignable_bucket" bucketname = "retrofit-energy-assessments-${var.stage}" allowed_origins = var.allowed_origins environment = var.stage @@ -193,7 +193,7 @@ module "retrofit_energy_assessments" { # Set up the route53 record for the API module "route53" { - source = "./modules/route53" + source = "../modules/route53" domain_name = var.domain_name api_url_prefix = var.api_url_prefix providers = { @@ -204,65 +204,65 @@ module "route53" { # Create an ECR repository for storage of the lambda's docker images module "ecr" { ecr_name = "fastapi-repository-${var.stage}" - source = "./modules/ecr" + source = "../modules/ecr" } module "lambda_sap_prediction_ecr" { ecr_name = "lambda-sap-prediction-${var.stage}" - source = "./modules/ecr" + source = "../modules/ecr" } module "due_considerations_ecr" { ecr_name = "due-considerations-${var.stage}" - source = "./modules/ecr" + source = "../modules/ecr" } module "eco_spreadsheet_ecr" { ecr_name = "eco-spreadsheet-${var.stage}" - source = "./modules/ecr" + source = "../modules/ecr" } module "lambda_carbon_prediction_ecr" { ecr_name = "lambda-carbon-prediction-${var.stage}" - source = "./modules/ecr" + source = "../modules/ecr" } module "lambda_heat_prediction_ecr" { ecr_name = "lambda-heat-prediction-${var.stage}" - source = "./modules/ecr" + source = "../modules/ecr" } # ECR repos for lighting cost, heating cost and hot water cost models module "lambda_lighting_cost_prediction_ecr" { ecr_name = "lighting-cost-prediction-${var.stage}" - source = "./modules/ecr" + source = "../modules/ecr" } module "lambda_heating_cost_prediction_ecr" { ecr_name = "heating-cost-prediction-${var.stage}" - source = "./modules/ecr" + source = "../modules/ecr" } module "lambda_hot_water_cost_prediction_ecr" { ecr_name = "hot-water-cost-prediction-${var.stage}" - source = "./modules/ecr" + source = "../modules/ecr" } # For heating and hot water kwh models module "lambda_heating_kwh_prediction_ecr" { ecr_name = "heating-kwh-prediction-${var.stage}" - source = "./modules/ecr" + source = "../modules/ecr" } module "lambda_hotwater_kwh_prediction_ecr" { ecr_name = "hotwater-kwh-prediction-${var.stage}" - source = "./modules/ecr" + source = "../modules/ecr" } # Baselining models module "sap_baseline_ecr" { ecr_name = "sap-baseline-prediction-${var.stage}" - source = "./modules/ecr" + source = "../modules/ecr" } ############################################## @@ -281,7 +281,7 @@ module "cloudfront_distribution" { # SES - Email sending ################################################ module "ses" { - source = "./modules/ses" + source = "../modules/ses" domain_name = "domna.homes" stage = var.stage } From 150da2a78006ee5f7fa32d3d263e3800a34fef75 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Mon, 2 Feb 2026 18:08:02 +0000 Subject: [PATCH 08/86] cloud in the wrong directory --- infrastructure/terraform/shared/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infrastructure/terraform/shared/main.tf b/infrastructure/terraform/shared/main.tf index 2322557d..9d90ae61 100644 --- a/infrastructure/terraform/shared/main.tf +++ b/infrastructure/terraform/shared/main.tf @@ -269,7 +269,7 @@ module "sap_baseline_ecr" { # CDN - Cloudfront ############################################## module "cloudfront_distribution" { - source = "./modules/cloudfront" + source = "../modules/cloudfront" bucket_name = module.s3.bucket_name bucket_id = module.s3.bucket_id bucket_arn = module.s3.bucket_arn From 1a640d6d01e3c116dd606f847287be35668a3442 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Mon, 2 Feb 2026 18:27:55 +0000 Subject: [PATCH 09/86] check plan first --- .github/workflows/deploy_terraform.yml | 8 +++++- infrastructure/terraform/modules/ecr/main.tf | 25 ++++++++++++++----- .../terraform/modules/ecr/outputs.tf | 6 +++++ infrastructure/terraform/shared/main.tf | 9 +++++++ 4 files changed, 41 insertions(+), 7 deletions(-) diff --git a/.github/workflows/deploy_terraform.yml b/.github/workflows/deploy_terraform.yml index fe52a1e2..d90d9912 100644 --- a/.github/workflows/deploy_terraform.yml +++ b/.github/workflows/deploy_terraform.yml @@ -55,4 +55,10 @@ jobs: - name: Terraform Plan (shared) run: | cd infrastructure/terraform/shared - terraform plan -var-file=dev.tfvars \ No newline at end of file + terraform plan -var-file=dev.tfvars + + # # only run once + # - name: Terraform Apply (shared) + # run: | + # cd infrastructure/terraform/shared + # terraform apply -auto-approve -var-file=dev.tfvars \ No newline at end of file diff --git a/infrastructure/terraform/modules/ecr/main.tf b/infrastructure/terraform/modules/ecr/main.tf index 468ef3d2..dfed0712 100644 --- a/infrastructure/terraform/modules/ecr/main.tf +++ b/infrastructure/terraform/modules/ecr/main.tf @@ -1,7 +1,6 @@ resource "aws_ecr_repository" "my_repository" { - name = "${var.ecr_name}" + name = var.ecr_name image_tag_mutability = "MUTABLE" - # Allows overwriting image tags, change to IMMUTABLE if you want to prevent overwriting image_scanning_configuration { scan_on_push = true @@ -13,13 +12,27 @@ resource "aws_ecr_lifecycle_policy" "my_repository_policy" { policy = jsonencode({ rules = [ + # 1️⃣ PROTECT important environment tags forever { rulePriority = 1 - description = "Retain only the last 10 images" - selection = { + description = "Keep prod, main, dev images forever" + selection = { + tagStatus = "tagged" + tagPrefixList = ["prod", "main", "dev"] + } + action = { + type = "retain" + } + }, + + # 2️⃣ Expire everything else beyond the most recent 20 images + { + rulePriority = 2 + description = "Expire old non-protected images" + selection = { tagStatus = "any" countType = "imageCountMoreThan" - countNumber = 10 + countNumber = 20 } action = { type = "expire" @@ -27,4 +40,4 @@ resource "aws_ecr_lifecycle_policy" "my_repository_policy" { } ] }) -} \ No newline at end of file +} diff --git a/infrastructure/terraform/modules/ecr/outputs.tf b/infrastructure/terraform/modules/ecr/outputs.tf index 53839718..7f045412 100644 --- a/infrastructure/terraform/modules/ecr/outputs.tf +++ b/infrastructure/terraform/modules/ecr/outputs.tf @@ -1,4 +1,10 @@ output "ecr_repository_name" { description = "Name of the EPR repo in AWS" value = aws_ecr_repository.my_repository.name +} + + +output "ecr_repository_url" { + description = "Full ECR repository URL" + value = aws_ecr_repository.my_repository.repository_url } \ No newline at end of file diff --git a/infrastructure/terraform/shared/main.tf b/infrastructure/terraform/shared/main.tf index 9d90ae61..393def46 100644 --- a/infrastructure/terraform/shared/main.tf +++ b/infrastructure/terraform/shared/main.tf @@ -288,4 +288,13 @@ module "ses" { output "ses_dns_records" { value = module.ses.dns_records +} + + +################################################ +# One ECR to rule all the lambdas +################################################ +module "lambda_shared_ecr" { + source = "../modules/ecr" + ecr_name = "lambda-shared-${var.stage}" } \ No newline at end of file From cc76f8502bb947012ae27fb58d8073d9943e2de2 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Mon, 2 Feb 2026 18:40:15 +0000 Subject: [PATCH 10/86] deploy container registry --- .github/workflows/deploy_terraform.yml | 10 +++++----- infrastructure/terraform/modules/ecr/main.tf | 6 +++--- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/deploy_terraform.yml b/.github/workflows/deploy_terraform.yml index d90d9912..51d7599e 100644 --- a/.github/workflows/deploy_terraform.yml +++ b/.github/workflows/deploy_terraform.yml @@ -57,8 +57,8 @@ jobs: cd infrastructure/terraform/shared terraform plan -var-file=dev.tfvars - # # only run once - # - name: Terraform Apply (shared) - # run: | - # cd infrastructure/terraform/shared - # terraform apply -auto-approve -var-file=dev.tfvars \ No newline at end of file + # only run once + - name: Terraform Apply (shared) + run: | + cd infrastructure/terraform/shared + terraform apply -auto-approve -var-file=dev.tfvars \ No newline at end of file diff --git a/infrastructure/terraform/modules/ecr/main.tf b/infrastructure/terraform/modules/ecr/main.tf index dfed0712..51085c5c 100644 --- a/infrastructure/terraform/modules/ecr/main.tf +++ b/infrastructure/terraform/modules/ecr/main.tf @@ -25,14 +25,14 @@ resource "aws_ecr_lifecycle_policy" "my_repository_policy" { } }, - # 2️⃣ Expire everything else beyond the most recent 20 images + # 2️⃣ Expire everything else beyond the most recent 10 images { rulePriority = 2 - description = "Expire old non-protected images" + description = "Retain only the last 10 images" -> "Keep prod, main, dev images forever" selection = { tagStatus = "any" countType = "imageCountMoreThan" - countNumber = 20 + countNumber = 10 } action = { type = "expire" From ba16afe0f0c79b2868cb10647fc4736d1fc430b9 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Mon, 2 Feb 2026 18:41:14 +0000 Subject: [PATCH 11/86] deploy container registry --- infrastructure/terraform/modules/ecr/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infrastructure/terraform/modules/ecr/main.tf b/infrastructure/terraform/modules/ecr/main.tf index 51085c5c..4f2c838e 100644 --- a/infrastructure/terraform/modules/ecr/main.tf +++ b/infrastructure/terraform/modules/ecr/main.tf @@ -28,7 +28,7 @@ resource "aws_ecr_lifecycle_policy" "my_repository_policy" { # 2️⃣ Expire everything else beyond the most recent 10 images { rulePriority = 2 - description = "Retain only the last 10 images" -> "Keep prod, main, dev images forever" + description = "Retain only the last 10 images" selection = { tagStatus = "any" countType = "imageCountMoreThan" From 92a6624b67e3c5ba7f87e0c6475722a42def2bdb Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Mon, 2 Feb 2026 18:50:14 +0000 Subject: [PATCH 12/86] got rid of description --- infrastructure/terraform/modules/ecr/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infrastructure/terraform/modules/ecr/main.tf b/infrastructure/terraform/modules/ecr/main.tf index 4f2c838e..b1fdc4bb 100644 --- a/infrastructure/terraform/modules/ecr/main.tf +++ b/infrastructure/terraform/modules/ecr/main.tf @@ -1,5 +1,5 @@ resource "aws_ecr_repository" "my_repository" { - name = var.ecr_name + name = "${var.ecr_name}" image_tag_mutability = "MUTABLE" image_scanning_configuration { From 6a89e8fc71ed7058dc40c95f1c94de68e1c37dca Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Mon, 2 Feb 2026 18:54:39 +0000 Subject: [PATCH 13/86] add count rules within docker --- infrastructure/terraform/modules/ecr/main.tf | 2 ++ 1 file changed, 2 insertions(+) diff --git a/infrastructure/terraform/modules/ecr/main.tf b/infrastructure/terraform/modules/ecr/main.tf index b1fdc4bb..67b2427b 100644 --- a/infrastructure/terraform/modules/ecr/main.tf +++ b/infrastructure/terraform/modules/ecr/main.tf @@ -19,6 +19,8 @@ resource "aws_ecr_lifecycle_policy" "my_repository_policy" { selection = { tagStatus = "tagged" tagPrefixList = ["prod", "main", "dev"] + countType = "imageCountMoreThan" + countNumber = 9999 } action = { type = "retain" From 3756bbec7e1f323b50126783d873584d7a71f6b5 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Mon, 2 Feb 2026 20:06:39 +0000 Subject: [PATCH 14/86] re run --- infrastructure/terraform/modules/ecr/main.tf | 1 + 1 file changed, 1 insertion(+) diff --git a/infrastructure/terraform/modules/ecr/main.tf b/infrastructure/terraform/modules/ecr/main.tf index 67b2427b..d0c37190 100644 --- a/infrastructure/terraform/modules/ecr/main.tf +++ b/infrastructure/terraform/modules/ecr/main.tf @@ -43,3 +43,4 @@ resource "aws_ecr_lifecycle_policy" "my_repository_policy" { ] }) } + From 68e0825295ed861e4a0a1f3349b0b7a96b7baa58 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Mon, 2 Feb 2026 23:09:28 +0000 Subject: [PATCH 15/86] rerun --- .github/workflows/deploy_terraform.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/deploy_terraform.yml b/.github/workflows/deploy_terraform.yml index 51d7599e..c306e69b 100644 --- a/.github/workflows/deploy_terraform.yml +++ b/.github/workflows/deploy_terraform.yml @@ -61,4 +61,7 @@ jobs: - name: Terraform Apply (shared) run: | cd infrastructure/terraform/shared - terraform apply -auto-approve -var-file=dev.tfvars \ No newline at end of file + terraform apply -auto-approve -var-file=dev.tfvars + + + \ No newline at end of file From 634798c5d45fdb3262a3ed08960fc33680d1bdce Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Mon, 2 Feb 2026 23:13:12 +0000 Subject: [PATCH 16/86] get rid of previous error --- infrastructure/terraform/modules/ecr/main.tf | 2 -- 1 file changed, 2 deletions(-) diff --git a/infrastructure/terraform/modules/ecr/main.tf b/infrastructure/terraform/modules/ecr/main.tf index d0c37190..813ff9b0 100644 --- a/infrastructure/terraform/modules/ecr/main.tf +++ b/infrastructure/terraform/modules/ecr/main.tf @@ -19,8 +19,6 @@ resource "aws_ecr_lifecycle_policy" "my_repository_policy" { selection = { tagStatus = "tagged" tagPrefixList = ["prod", "main", "dev"] - countType = "imageCountMoreThan" - countNumber = 9999 } action = { type = "retain" From 8b0ac85ad1079f1ff4d960236abf059a94d29034 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Mon, 2 Feb 2026 23:19:25 +0000 Subject: [PATCH 17/86] dont use cache --- .../actions/actions/lambda-deploy/action.yml | 86 ------------------- .../actions/terraform-deploy/action.yml | 55 ------------ .github/workflows/deploy_terraform.yml | 3 +- 3 files changed, 1 insertion(+), 143 deletions(-) delete mode 100644 .github/workflows/actions/actions/lambda-deploy/action.yml delete mode 100644 .github/workflows/actions/actions/terraform-deploy/action.yml diff --git a/.github/workflows/actions/actions/lambda-deploy/action.yml b/.github/workflows/actions/actions/lambda-deploy/action.yml deleted file mode 100644 index 3ca0fc8d..00000000 --- a/.github/workflows/actions/actions/lambda-deploy/action.yml +++ /dev/null @@ -1,86 +0,0 @@ -name: "Build and Push Lambda Image to ECR" -description: "Reusable action for building and pushing lambda Docker image to ECR" - -inputs: - ecr_name: - description: "Lambda name / ECR repo name" - required: true - dockerfile_path: - description: "Path to Dockerfile" - required: true - ecr_tf_dir: - description: "Path to ECR terraform directory" - required: true - lambda_tf_dir: - description: "Path to Lambda terraform directory" - required: true - aws-access-key-id: - description: "AWS access key" - required: true - aws-secret-access-key: - description: "AWS secret key" - required: true - aws-region: - description: "AWS region" - required: true - git-sha: - description: "Git commit SHA" - required: true - git-ref: - description: "Git ref name" - required: true - -runs: - using: "composite" - steps: - - uses: actions/checkout@v4 - - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - aws-access-key-id: ${{ inputs.aws-access-key-id }} - aws-secret-access-key: ${{ inputs.aws-secret-access-key }} - aws-region: ${{ inputs.aws-region }} - - - name: Log in to Amazon ECR - id: login-ecr - uses: aws-actions/amazon-ecr-login@v2 - - - name: Deploy ECR - uses: ./.github/workflows/actions/terraform-deploy - with: - working_directory: ${{ inputs.ecr_tf_dir }} - aws-access-key-id: ${{ inputs.aws-access-key-id }} - aws-secret-access-key: ${{ inputs.aws-secret-access-key }} - aws-region: ${{ inputs.aws-region }} - - name: Set Docker image tag - id: set_tag - shell: bash - run: | - SHORT_SHA=$(echo "${{ inputs.git-sha }}" | cut -c1-7) - BRANCH=$(echo "${{ inputs.git-ref }}" | tr '/' '-') - TAG="${BRANCH}-${SHORT_SHA}" - echo "IMAGE_TAG=${TAG}" >> $GITHUB_ENV - echo "tag=$TAG" >> $GITHUB_OUTPUT - - - name: Build and push Docker image - shell: bash - run: | - IMAGE_URI=${{ steps.login-ecr.outputs.registry }}/${{ inputs.ecr_name }}:${{ steps.set_tag.outputs.tag }} - echo "Building Docker image for ${{ inputs.ecr_name }}..." - docker build -t $IMAGE_URI -f ${{ inputs.dockerfile_path }} . - - echo "Pushing to ECR..." - docker push $IMAGE_URI - - - name: Deploy Lambda - uses: ./.github/workflows/actions/terraform-deploy - with: - working_directory: ${{ inputs.lambda_tf_dir }} - aws-access-key-id: ${{ inputs.aws-access-key-id }} - aws-secret-access-key: ${{ inputs.aws-secret-access-key }} - aws-region: ${{ inputs.aws-region }} - lambda-image-tag: ${{ steps.set_tag.outputs.tag }} - - - diff --git a/.github/workflows/actions/actions/terraform-deploy/action.yml b/.github/workflows/actions/actions/terraform-deploy/action.yml deleted file mode 100644 index 56133299..00000000 --- a/.github/workflows/actions/actions/terraform-deploy/action.yml +++ /dev/null @@ -1,55 +0,0 @@ -name: "Terraform Plan Shared Config" -description: "Plans shared Terraform config for Lambdas" - -inputs: - working_directory: - description: "Directory containing Terraform config" - required: true - aws-access-key-id: - description: "AWS access key" - required: true - aws-secret-access-key: - description: "AWS secret key" - required: true - aws-region: - description: "AWS region" - required: true - lambda-image-tag: - description: "Tag of the Lambda image (e.g., GitHub SHA)" - required: false - -runs: - using: "composite" - steps: - - uses: actions/checkout@v4 - - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - aws-access-key-id: ${{ inputs.aws-access-key-id }} - aws-secret-access-key: ${{ inputs.aws-secret-access-key }} - aws-region: ${{ inputs.aws-region }} - - - name: Setup Terraform - uses: hashicorp/setup-terraform@v3 - - - name: Terraform Init - working-directory: ${{ inputs.working_directory }} - shell: bash - run: terraform init -reconfigure - - - name: Terraform Plan - working-directory: ${{ inputs.working_directory }} - shell: bash - run: | - if [ -n "${{ inputs.lambda-image-tag }}" ]; then - terraform plan -out=tfplan -var="lambda_image_tag=${{ inputs.lambda-image-tag }}" - else - terraform plan -out=tfplan - fi - - - name: Terraform Apply - working-directory: ${{ inputs.working_directory }} - shell: bash - run: terraform apply -auto-approve tfplan - diff --git a/.github/workflows/deploy_terraform.yml b/.github/workflows/deploy_terraform.yml index c306e69b..fe3ca9b2 100644 --- a/.github/workflows/deploy_terraform.yml +++ b/.github/workflows/deploy_terraform.yml @@ -45,7 +45,7 @@ jobs: # Deploy shared terrform things - name: Terraform Init - run: cd infrastructure/terraform/shared && terraform init + run: cd infrastructure/terraform/shared && terraform init -upgrade - name: Terraform Workspace run: | @@ -64,4 +64,3 @@ jobs: terraform apply -auto-approve -var-file=dev.tfvars - \ No newline at end of file From 6ba5ecc3b4979af58cc640cf3a05b16f6eaa2e8d Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Mon, 2 Feb 2026 23:26:42 +0000 Subject: [PATCH 18/86] untagged image as well --- infrastructure/terraform/modules/ecr/main.tf | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/infrastructure/terraform/modules/ecr/main.tf b/infrastructure/terraform/modules/ecr/main.tf index 813ff9b0..ee4cac11 100644 --- a/infrastructure/terraform/modules/ecr/main.tf +++ b/infrastructure/terraform/modules/ecr/main.tf @@ -12,7 +12,7 @@ resource "aws_ecr_lifecycle_policy" "my_repository_policy" { policy = jsonencode({ rules = [ - # 1️⃣ PROTECT important environment tags forever + # 1️⃣ Keep important env tags forever { rulePriority = 1 description = "Keep prod, main, dev images forever" @@ -25,9 +25,23 @@ resource "aws_ecr_lifecycle_policy" "my_repository_policy" { } }, - # 2️⃣ Expire everything else beyond the most recent 10 images + # 2️⃣ Aggressively expire untagged images { rulePriority = 2 + description = "Expire untagged images" + selection = { + tagStatus = "untagged" + countType = "imageCountMoreThan" + countNumber = 1 + } + action = { + type = "expire" + } + }, + + # 3️⃣ Rotate everything else + { + rulePriority = 3 description = "Retain only the last 10 images" selection = { tagStatus = "any" @@ -41,4 +55,3 @@ resource "aws_ecr_lifecycle_policy" "my_repository_policy" { ] }) } - From 9208b3eda4cd8a9e473a11cad26ffbfbe0173c66 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Mon, 2 Feb 2026 23:27:00 +0000 Subject: [PATCH 19/86] get rid of tag --- .github/workflows/deploy_terraform.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy_terraform.yml b/.github/workflows/deploy_terraform.yml index fe3ca9b2..79adf795 100644 --- a/.github/workflows/deploy_terraform.yml +++ b/.github/workflows/deploy_terraform.yml @@ -45,7 +45,7 @@ jobs: # Deploy shared terrform things - name: Terraform Init - run: cd infrastructure/terraform/shared && terraform init -upgrade + run: cd infrastructure/terraform/shared && terraform init - name: Terraform Workspace run: | From e069f5d12983fa6e8789ad8a101ab384da8f5341 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Mon, 2 Feb 2026 23:30:45 +0000 Subject: [PATCH 20/86] go back to the old one --- infrastructure/terraform/modules/ecr/main.tf | 33 ++------------------ 1 file changed, 3 insertions(+), 30 deletions(-) diff --git a/infrastructure/terraform/modules/ecr/main.tf b/infrastructure/terraform/modules/ecr/main.tf index ee4cac11..468ef3d2 100644 --- a/infrastructure/terraform/modules/ecr/main.tf +++ b/infrastructure/terraform/modules/ecr/main.tf @@ -1,6 +1,7 @@ resource "aws_ecr_repository" "my_repository" { name = "${var.ecr_name}" image_tag_mutability = "MUTABLE" + # Allows overwriting image tags, change to IMMUTABLE if you want to prevent overwriting image_scanning_configuration { scan_on_push = true @@ -12,38 +13,10 @@ resource "aws_ecr_lifecycle_policy" "my_repository_policy" { policy = jsonencode({ rules = [ - # 1️⃣ Keep important env tags forever { rulePriority = 1 - description = "Keep prod, main, dev images forever" - selection = { - tagStatus = "tagged" - tagPrefixList = ["prod", "main", "dev"] - } - action = { - type = "retain" - } - }, - - # 2️⃣ Aggressively expire untagged images - { - rulePriority = 2 - description = "Expire untagged images" - selection = { - tagStatus = "untagged" - countType = "imageCountMoreThan" - countNumber = 1 - } - action = { - type = "expire" - } - }, - - # 3️⃣ Rotate everything else - { - rulePriority = 3 description = "Retain only the last 10 images" - selection = { + selection = { tagStatus = "any" countType = "imageCountMoreThan" countNumber = 10 @@ -54,4 +27,4 @@ resource "aws_ecr_lifecycle_policy" "my_repository_policy" { } ] }) -} +} \ No newline at end of file From a4124dce336a1979c8e5b20cf134ef2ce2b3ce13 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Tue, 3 Feb 2026 07:56:48 +0000 Subject: [PATCH 21/86] check new layout --- .github/workflows/deploy_terraform.yml | 81 ++++++++++---------- infrastructure/terraform/lamdas/backend.tf | 0 infrastructure/terraform/lamdas/dev.tfvars | 0 infrastructure/terraform/lamdas/main.tf | 0 infrastructure/terraform/lamdas/variables.tf | 0 5 files changed, 40 insertions(+), 41 deletions(-) delete mode 100644 infrastructure/terraform/lamdas/backend.tf delete mode 100644 infrastructure/terraform/lamdas/dev.tfvars delete mode 100644 infrastructure/terraform/lamdas/main.tf delete mode 100644 infrastructure/terraform/lamdas/variables.tf diff --git a/.github/workflows/deploy_terraform.yml b/.github/workflows/deploy_terraform.yml index 79adf795..3311ce45 100644 --- a/.github/workflows/deploy_terraform.yml +++ b/.github/workflows/deploy_terraform.yml @@ -6,61 +6,60 @@ on: - "**" jobs: - deploy: + Deploy shared terraform stack: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v4 - - name: Setup AWS credentials file - run: | - mkdir -p ~/.aws - echo "[DevAdmin]" > ~/.aws/credentials - echo "aws_access_key_id = ${{ secrets.DEV_AWS_ACCESS_KEY_ID }}" >> ~/.aws/credentials - echo "aws_secret_access_key = ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }}" >> ~/.aws/credentials - echo "[ProdAdmin]" >> ~/.aws/credentials - echo "aws_access_key_id = ${{ secrets.PROD_AWS_ACCESS_KEY_ID }}" >> ~/.aws/credentials - echo "aws_secret_access_key = ${{ secrets.PROD_AWS_SECRET_ACCESS_KEY }}" >> ~/.aws/credentials - - - name: Setup AWS config file - run: | - echo "[profile DevAdmin]" > ~/.aws/config - echo "region = eu-west-2" >> ~/.aws/config - echo "[profile ProdAdmin]" >> ~/.aws/config - echo "region = eu-west-2" >> ~/.aws/config - - - name: Setup Terraform - uses: hashicorp/setup-terraform@v1 + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 with: - terraform_version: 1.5.2 - - - name: Configure AWS credentials (DevAdmin) - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }} + aws-access-key-id: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }}" + aws-secret-access-key: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }}" aws-region: eu-west-2 env: AWS_PROFILE: "DevAdmin" - # Deploy shared terrform things + - name: Setup Terraform + shell: bash + uses: hashicorp/setup-terraform@v3 + - name: Terraform Init - run: cd infrastructure/terraform/shared && terraform init + working-directory: ./infrastructure/terraform/shared + shell: bash + run: terraform init -reconfigure - name: Terraform Workspace - run: | - cd infrastructure/terraform/shared - terraform workspace select dev || terraform workspace new dev + working-directory: ./infrastructure/terraform/shared + shell: bash + run: terraform workspace select dev || terraform workspace new dev - name: Terraform Plan (shared) - run: | - cd infrastructure/terraform/shared - terraform plan -var-file=dev.tfvars + working-directory: ./infrastructure/terraform/shared + shell: bash + run: terraform plan -var-file=dev.tfvars -out=tfplan - # only run once - - name: Terraform Apply (shared) - run: | - cd infrastructure/terraform/shared - terraform apply -auto-approve -var-file=dev.tfvars + # - name: Terraform Apply + # working-directory: ./infrastructure/terraform/shared + # shell: bash + # run: terraform apply -auto-approve tfplan + + # # apply shared dev + # - name: Terraform Apply (shared) + # run: | + # cd infrastructure/terraform/shared + # terraform apply -auto-approve -var-file=dev.tfvars + + # - name: Build & push image + # run: | + # IMAGE_TAG=address2uprn-${GITHUB_SHA} + # IMAGE_URI=${AWS_ACCOUNT_ID}.dkr.ecr.eu-west-2.amazonaws.com/lambda-shared-dev:${IMAGE_TAG} + + # docker build -t $IMAGE_URI . + # docker push $IMAGE_URI + + # echo "IMAGE_URI=$IMAGE_URI" >> $GITHUB_ENV + diff --git a/infrastructure/terraform/lamdas/backend.tf b/infrastructure/terraform/lamdas/backend.tf deleted file mode 100644 index e69de29b..00000000 diff --git a/infrastructure/terraform/lamdas/dev.tfvars b/infrastructure/terraform/lamdas/dev.tfvars deleted file mode 100644 index e69de29b..00000000 diff --git a/infrastructure/terraform/lamdas/main.tf b/infrastructure/terraform/lamdas/main.tf deleted file mode 100644 index e69de29b..00000000 diff --git a/infrastructure/terraform/lamdas/variables.tf b/infrastructure/terraform/lamdas/variables.tf deleted file mode 100644 index e69de29b..00000000 From eb6d07c28b1a050faa9771dbc05ce3d1644278b9 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Tue, 3 Feb 2026 07:59:13 +0000 Subject: [PATCH 22/86] fix hasicorp set up --- .github/workflows/deploy_terraform.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/deploy_terraform.yml b/.github/workflows/deploy_terraform.yml index 3311ce45..71c485ce 100644 --- a/.github/workflows/deploy_terraform.yml +++ b/.github/workflows/deploy_terraform.yml @@ -22,7 +22,6 @@ jobs: AWS_PROFILE: "DevAdmin" - name: Setup Terraform - shell: bash uses: hashicorp/setup-terraform@v3 - name: Terraform Init From 5a3634d43e9ed78ec54a10e2092277149317738b Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Tue, 3 Feb 2026 08:00:18 +0000 Subject: [PATCH 23/86] job name --- .github/workflows/deploy_terraform.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy_terraform.yml b/.github/workflows/deploy_terraform.yml index 71c485ce..b5f2c9cf 100644 --- a/.github/workflows/deploy_terraform.yml +++ b/.github/workflows/deploy_terraform.yml @@ -6,7 +6,7 @@ on: - "**" jobs: - Deploy shared terraform stack: + deploy_shared_terraform_stack: runs-on: ubuntu-latest steps: - name: Checkout From 72b2515011a40c14787df077dc636bac4682a689 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Tue, 3 Feb 2026 08:01:41 +0000 Subject: [PATCH 24/86] get rid of dev admin env --- .github/workflows/deploy_terraform.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/deploy_terraform.yml b/.github/workflows/deploy_terraform.yml index b5f2c9cf..d1f3b133 100644 --- a/.github/workflows/deploy_terraform.yml +++ b/.github/workflows/deploy_terraform.yml @@ -18,8 +18,6 @@ jobs: aws-access-key-id: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }}" aws-secret-access-key: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }}" aws-region: eu-west-2 - env: - AWS_PROFILE: "DevAdmin" - name: Setup Terraform uses: hashicorp/setup-terraform@v3 From 3d4e4e6c4c9599617ce7b59011428afe7d9cdbd6 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Tue, 3 Feb 2026 08:05:35 +0000 Subject: [PATCH 25/86] add secrets properly --- .github/workflows/deploy_terraform.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy_terraform.yml b/.github/workflows/deploy_terraform.yml index d1f3b133..f845d2a7 100644 --- a/.github/workflows/deploy_terraform.yml +++ b/.github/workflows/deploy_terraform.yml @@ -16,7 +16,7 @@ jobs: uses: aws-actions/configure-aws-credentials@v4 with: aws-access-key-id: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }}" - aws-secret-access-key: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }}" + aws-secret-access-key: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }}" aws-region: eu-west-2 - name: Setup Terraform From d686f3dad6a3e330148f69f1300fcaa806bac935 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Tue, 3 Feb 2026 08:07:01 +0000 Subject: [PATCH 26/86] add secrets properly --- .github/workflows/deploy_terraform.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploy_terraform.yml b/.github/workflows/deploy_terraform.yml index f845d2a7..28561661 100644 --- a/.github/workflows/deploy_terraform.yml +++ b/.github/workflows/deploy_terraform.yml @@ -15,8 +15,8 @@ jobs: - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v4 with: - aws-access-key-id: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }}" - aws-secret-access-key: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }}" + aws-access-key-id: "${{ secrets.DEV_AWS_ACCESS_KEY_ID }}" + aws-secret-access-key: "${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }}" aws-region: eu-west-2 - name: Setup Terraform From 558932aaa8a5ac8e70a3b8e86f284f7b3c3b2d53 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Tue, 3 Feb 2026 08:08:35 +0000 Subject: [PATCH 27/86] fix directory --- .github/workflows/deploy_terraform.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/deploy_terraform.yml b/.github/workflows/deploy_terraform.yml index 28561661..23cc4d71 100644 --- a/.github/workflows/deploy_terraform.yml +++ b/.github/workflows/deploy_terraform.yml @@ -23,17 +23,17 @@ jobs: uses: hashicorp/setup-terraform@v3 - name: Terraform Init - working-directory: ./infrastructure/terraform/shared + working-directory: ./infrastructure/terraform/shared/ shell: bash run: terraform init -reconfigure - name: Terraform Workspace - working-directory: ./infrastructure/terraform/shared + working-directory: ./infrastructure/terraform/shared/ shell: bash run: terraform workspace select dev || terraform workspace new dev - name: Terraform Plan (shared) - working-directory: ./infrastructure/terraform/shared + working-directory: ./infrastructure/terraform/shared/ shell: bash run: terraform plan -var-file=dev.tfvars -out=tfplan From 9f91a3b547cca9ee8152b320dbe1d4aaae531ecf Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Tue, 3 Feb 2026 08:11:38 +0000 Subject: [PATCH 28/86] fix directory --- .github/workflows/deploy_terraform.yml | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/.github/workflows/deploy_terraform.yml b/.github/workflows/deploy_terraform.yml index 23cc4d71..899ab0e3 100644 --- a/.github/workflows/deploy_terraform.yml +++ b/.github/workflows/deploy_terraform.yml @@ -23,19 +23,25 @@ jobs: uses: hashicorp/setup-terraform@v3 - name: Terraform Init - working-directory: ./infrastructure/terraform/shared/ + working-directory: ./infrastructure/terraform shell: bash - run: terraform init -reconfigure + run: | + cd shared + terraform init -reconfigure - name: Terraform Workspace - working-directory: ./infrastructure/terraform/shared/ + working-directory: ./infrastructure/terraform shell: bash - run: terraform workspace select dev || terraform workspace new dev + run: | + cd shared + terraform workspace select dev || terraform workspace new dev - name: Terraform Plan (shared) - working-directory: ./infrastructure/terraform/shared/ + working-directory: ./infrastructure/terraform shell: bash - run: terraform plan -var-file=dev.tfvars -out=tfplan + run: | + cd shared + terraform plan -var-file=dev.tfvars -out=tfplan # - name: Terraform Apply # working-directory: ./infrastructure/terraform/shared From ddaddb82d9e634ed226b3b77709f1c3537548b8a Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Tue, 3 Feb 2026 08:15:49 +0000 Subject: [PATCH 29/86] add working directory again --- .github/workflows/deploy_terraform.yml | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/.github/workflows/deploy_terraform.yml b/.github/workflows/deploy_terraform.yml index 899ab0e3..23cc4d71 100644 --- a/.github/workflows/deploy_terraform.yml +++ b/.github/workflows/deploy_terraform.yml @@ -23,25 +23,19 @@ jobs: uses: hashicorp/setup-terraform@v3 - name: Terraform Init - working-directory: ./infrastructure/terraform + working-directory: ./infrastructure/terraform/shared/ shell: bash - run: | - cd shared - terraform init -reconfigure + run: terraform init -reconfigure - name: Terraform Workspace - working-directory: ./infrastructure/terraform + working-directory: ./infrastructure/terraform/shared/ shell: bash - run: | - cd shared - terraform workspace select dev || terraform workspace new dev + run: terraform workspace select dev || terraform workspace new dev - name: Terraform Plan (shared) - working-directory: ./infrastructure/terraform + working-directory: ./infrastructure/terraform/shared/ shell: bash - run: | - cd shared - terraform plan -var-file=dev.tfvars -out=tfplan + run: terraform plan -var-file=dev.tfvars -out=tfplan # - name: Terraform Apply # working-directory: ./infrastructure/terraform/shared From 45c84c996674bfe583c2175e8221803e247ebf0f Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Tue, 3 Feb 2026 08:20:38 +0000 Subject: [PATCH 30/86] add working directory again --- infrastructure/terraform/shared/main.tf | 3 +++ 1 file changed, 3 insertions(+) diff --git a/infrastructure/terraform/shared/main.tf b/infrastructure/terraform/shared/main.tf index 393def46..ea250c9b 100644 --- a/infrastructure/terraform/shared/main.tf +++ b/infrastructure/terraform/shared/main.tf @@ -3,6 +3,9 @@ terraform { aws = { source = "hashicorp/aws" version = "~> 4.16" + configuration_aliases = [ + aws.aws_use1 + ] } } backend "s3" { From cd6681970ac83502ad3d3cf1829a315b271b0e48 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Tue, 3 Feb 2026 08:22:40 +0000 Subject: [PATCH 31/86] get rid of profile --- infrastructure/terraform/shared/dev.tfvars | 2 +- infrastructure/terraform/shared/main.tf | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/infrastructure/terraform/shared/dev.tfvars b/infrastructure/terraform/shared/dev.tfvars index 92b7e158..544104b1 100644 --- a/infrastructure/terraform/shared/dev.tfvars +++ b/infrastructure/terraform/shared/dev.tfvars @@ -1,5 +1,5 @@ stage = "dev" -profile = "DevAdmin" +# profile = "DevAdmin" region = "eu-west-2" # Domain diff --git a/infrastructure/terraform/shared/main.tf b/infrastructure/terraform/shared/main.tf index ea250c9b..5d5c6be9 100644 --- a/infrastructure/terraform/shared/main.tf +++ b/infrastructure/terraform/shared/main.tf @@ -3,15 +3,12 @@ terraform { aws = { source = "hashicorp/aws" version = "~> 4.16" - configuration_aliases = [ - aws.aws_use1 - ] } } backend "s3" { bucket = "assessment-model-terraform-state" region = "eu-west-2" - profile = "DevAdmin" + # profile = "DevAdmin" key = "terraform.tfstate" } @@ -204,6 +201,7 @@ module "route53" { } } + # Create an ECR repository for storage of the lambda's docker images module "ecr" { ecr_name = "fastapi-repository-${var.stage}" From 2d2498490507fbf43a9374de56e74e0af7e3addd Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Tue, 3 Feb 2026 08:25:32 +0000 Subject: [PATCH 32/86] get rid of profile --- infrastructure/terraform/shared/main.tf | 1 - 1 file changed, 1 deletion(-) diff --git a/infrastructure/terraform/shared/main.tf b/infrastructure/terraform/shared/main.tf index 5d5c6be9..facc6ea5 100644 --- a/infrastructure/terraform/shared/main.tf +++ b/infrastructure/terraform/shared/main.tf @@ -16,7 +16,6 @@ terraform { } provider "aws" { - profile = var.profile region = var.region } From 1f023d9f69a036313452ca88b4e164b59f2254e8 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Tue, 3 Feb 2026 08:28:28 +0000 Subject: [PATCH 33/86] get rid of profile --- infrastructure/terraform/shared/variables.tf | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/infrastructure/terraform/shared/variables.tf b/infrastructure/terraform/shared/variables.tf index 76734340..9a07e6bd 100644 --- a/infrastructure/terraform/shared/variables.tf +++ b/infrastructure/terraform/shared/variables.tf @@ -3,10 +3,10 @@ variable stage { type = string } -variable "profile" { - description = "AWS profile to use" - type = string -} +# variable "profile" { +# description = "AWS profile to use" +# type = string +# } variable "region" { description = "AWS region" From 5cf3644cf46b8281010afcca555b3ec9dec676aa Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Tue, 3 Feb 2026 08:29:53 +0000 Subject: [PATCH 34/86] get rid of profile --- infrastructure/terraform/shared/variables.tf | 4 ---- 1 file changed, 4 deletions(-) diff --git a/infrastructure/terraform/shared/variables.tf b/infrastructure/terraform/shared/variables.tf index 9a07e6bd..d00170d2 100644 --- a/infrastructure/terraform/shared/variables.tf +++ b/infrastructure/terraform/shared/variables.tf @@ -3,10 +3,6 @@ variable stage { type = string } -# variable "profile" { -# description = "AWS profile to use" -# type = string -# } variable "region" { description = "AWS region" From ef3e808c9cd753ccec83d3c89b3591abe47b1e84 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Tue, 3 Feb 2026 11:12:25 +0000 Subject: [PATCH 35/86] test deployment --- .github/workflows/_build_image.yml | 79 ++++++++++++++++ .github/workflows/_deploy_lambda.yml | 68 ++++++++++++++ .github/workflows/deploy_terraform.yml | 92 ++++++++++++------- .../terraform/lambda/_template/README.md | 7 ++ .../terraform/lambda/_template/main.tf | 21 +++++ .../terraform/lambda/_template/provider.tf | 20 ++++ .../terraform/lambda/_template/variables.tf | 17 ++++ .../terraform/lambda/address2UPRN/main.tf | 88 ++++++++++++++++++ .../terraform/lambda/address2UPRN/provider.tf | 20 ++++ .../lambda/address2UPRN/variables.tf | 13 +++ .../lambda/modules/lambda_with_sqs/main.tf | 44 +++++++++ .../lambda/modules/lambda_with_sqs/outputs.tf | 11 +++ .../modules/lambda_with_sqs/variables.tf | 36 ++++++++ .../modules/container_registry/main.tf | 30 ++++++ .../modules/container_registry/outputs.tf | 11 +++ .../modules/container_registry/variables.tf | 15 +++ infrastructure/terraform/modules/ecr/main.tf | 5 + .../modules/lambda_execution_role/main.tf | 37 ++++++++ .../modules/lambda_execution_role/outputs.tf | 7 ++ .../lambda_execution_role/variables.tf | 0 .../terraform/modules/lambda_service/main.tf | 15 +++ .../modules/lambda_service/outputs.tf | 3 + .../modules/lambda_service/variables.tf | 18 ++++ .../modules/lambda_sqs_trigger/main.tf | 23 +++++ .../modules/lambda_sqs_trigger/variables.tf | 8 ++ .../terraform/modules/lambda_with_sqs/main.tf | 23 ----- .../modules/lambda_with_sqs/outputs.tf | 15 --- .../modules/lambda_with_sqs/variables.tf | 32 ------- .../terraform/modules/sqs_queue/main.tf | 12 +++ .../terraform/modules/sqs_queue/outputs.tf | 7 ++ .../terraform/modules/sqs_queue/variables.tf | 6 ++ .../terraform/modules/tf_state_bucket/main.tf | 30 ++++++ .../modules/tf_state_bucket/outputs.tf | 7 ++ .../modules/tf_state_bucket/variables.tf | 3 + infrastructure/terraform/shared/locals.tf | 0 infrastructure/terraform/shared/main.tf | 25 +++-- infrastructure/terraform/shared/variables.tf | 1 - 37 files changed, 736 insertions(+), 113 deletions(-) create mode 100644 .github/workflows/_build_image.yml create mode 100644 .github/workflows/_deploy_lambda.yml create mode 100644 infrastructure/terraform/lambda/_template/README.md create mode 100644 infrastructure/terraform/lambda/_template/main.tf create mode 100644 infrastructure/terraform/lambda/_template/provider.tf create mode 100644 infrastructure/terraform/lambda/_template/variables.tf create mode 100644 infrastructure/terraform/lambda/address2UPRN/main.tf create mode 100644 infrastructure/terraform/lambda/address2UPRN/provider.tf create mode 100644 infrastructure/terraform/lambda/address2UPRN/variables.tf create mode 100644 infrastructure/terraform/lambda/modules/lambda_with_sqs/main.tf create mode 100644 infrastructure/terraform/lambda/modules/lambda_with_sqs/outputs.tf create mode 100644 infrastructure/terraform/lambda/modules/lambda_with_sqs/variables.tf create mode 100644 infrastructure/terraform/modules/container_registry/main.tf create mode 100644 infrastructure/terraform/modules/container_registry/outputs.tf create mode 100644 infrastructure/terraform/modules/container_registry/variables.tf create mode 100644 infrastructure/terraform/modules/lambda_execution_role/main.tf create mode 100644 infrastructure/terraform/modules/lambda_execution_role/outputs.tf create mode 100644 infrastructure/terraform/modules/lambda_execution_role/variables.tf create mode 100644 infrastructure/terraform/modules/lambda_service/main.tf create mode 100644 infrastructure/terraform/modules/lambda_service/outputs.tf create mode 100644 infrastructure/terraform/modules/lambda_service/variables.tf create mode 100644 infrastructure/terraform/modules/lambda_sqs_trigger/main.tf create mode 100644 infrastructure/terraform/modules/lambda_sqs_trigger/variables.tf delete mode 100644 infrastructure/terraform/modules/lambda_with_sqs/main.tf delete mode 100644 infrastructure/terraform/modules/lambda_with_sqs/outputs.tf delete mode 100644 infrastructure/terraform/modules/lambda_with_sqs/variables.tf create mode 100644 infrastructure/terraform/modules/sqs_queue/main.tf create mode 100644 infrastructure/terraform/modules/sqs_queue/outputs.tf create mode 100644 infrastructure/terraform/modules/sqs_queue/variables.tf create mode 100644 infrastructure/terraform/modules/tf_state_bucket/main.tf create mode 100644 infrastructure/terraform/modules/tf_state_bucket/outputs.tf create mode 100644 infrastructure/terraform/modules/tf_state_bucket/variables.tf create mode 100644 infrastructure/terraform/shared/locals.tf diff --git a/.github/workflows/_build_image.yml b/.github/workflows/_build_image.yml new file mode 100644 index 00000000..8ecd241e --- /dev/null +++ b/.github/workflows/_build_image.yml @@ -0,0 +1,79 @@ +name: Build Docker image + +on: + workflow_call: + inputs: + ecr_repo: + description: "ECR repository name" + required: true + type: string + + aws_region: + description: "AWS region" + required: true + type: string + + dockerfile_path: + description: "Path to Dockerfile" + required: true + type: string + + build_context: + description: "Docker build context directory" + required: false + default: "." + type: string + + outputs: + image_digest: + description: "Pushed image digest" + value: ${{ jobs.build.outputs.image_digest }} + + secrets: + AWS_ACCESS_KEY_ID: + required: true + AWS_SECRET_ACCESS_KEY: + required: true + AWS_ACCOUNT_ID: + required: true + +jobs: + build: + runs-on: ubuntu-latest + + outputs: + image_digest: ${{ steps.digest.outputs.image_digest }} + + steps: + - uses: actions/checkout@v4 + + - uses: aws-actions/configure-aws-credentials@v4 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ inputs.aws_region }} + + - uses: aws-actions/amazon-ecr-login@v2 + + - name: Build & push image + run: | + IMAGE_TAG=${GITHUB_SHA} + IMAGE_URI=${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ inputs.aws_region }}.amazonaws.com/${{ inputs.ecr_repo }}:${IMAGE_TAG} + + docker build \ + -f ${{ inputs.dockerfile_path }} \ + -t $IMAGE_URI \ + ${{ inputs.build_context }} + + docker push $IMAGE_URI + + - name: Resolve image digest + id: digest + run: | + DIGEST=$(aws ecr describe-images \ + --repository-name ${{ inputs.ecr_repo }} \ + --image-ids imageTag=${GITHUB_SHA} \ + --query 'imageDetails[0].imageDigest' \ + --output text) + + echo "image_digest=$DIGEST" >> $GITHUB_OUTPUT diff --git a/.github/workflows/_deploy_lambda.yml b/.github/workflows/_deploy_lambda.yml new file mode 100644 index 00000000..a218a3d7 --- /dev/null +++ b/.github/workflows/_deploy_lambda.yml @@ -0,0 +1,68 @@ +name: Deploy Lambda (Terraform) + +on: + workflow_call: + inputs: + lambda_name: + required: true + type: string + lambda_path: + required: true + type: string + stage: + required: true + type: string + aws_region: + required: true + type: string + image_digest: + required: true + type: string + + secrets: + AWS_ACCESS_KEY_ID: + required: true + AWS_SECRET_ACCESS_KEY: + required: true + AWS_ACCOUNT_ID: + required: true + +jobs: + deploy: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - uses: aws-actions/configure-aws-credentials@v4 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ inputs.aws_region }} + + - uses: hashicorp/setup-terraform@v3 + + - name: Terraform Init + working-directory: ${{ inputs.lambda_path }} + run: terraform init -reconfigure + + - name: Terraform Workspace + working-directory: ${{ inputs.lambda_path }} + run: | + terraform workspace select ${{ inputs.stage }} \ + || terraform workspace new ${{ inputs.stage }} + + - name: Terraform Plan + working-directory: ${{ inputs.lambda_path }} + run: | + terraform plan \ + -var="stage=${{ inputs.stage }}" \ + -var="image_digest=${{ inputs.image_digest }}" + + # - name: Terraform Apply + # working-directory: ${{ inputs.lambda_path }} + # run: | + # terraform apply \ + # -auto-approve \ + # -var="stage=${{ inputs.stage }}" \ + # -var="image_digest=${{ inputs.image_digest }}" diff --git a/.github/workflows/deploy_terraform.yml b/.github/workflows/deploy_terraform.yml index 23cc4d71..5ebe0216 100644 --- a/.github/workflows/deploy_terraform.yml +++ b/.github/workflows/deploy_terraform.yml @@ -1,13 +1,23 @@ -name: Deploy terraform stack +name: Deploy infrastructure on: push: branches: - "**" +env: + AWS_REGION: eu-west-2 + + # Temporary until we have more environemnts. You'll just need export STAGE dynamically in the future + STAGE: dev + jobs: - deploy_shared_terraform_stack: + # ============================================================ + # 1️⃣ Shared Terraform (plan only for now) + # ============================================================ + shared_terraform: runs-on: ubuntu-latest + steps: - name: Checkout uses: actions/checkout@v4 @@ -15,48 +25,60 @@ jobs: - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v4 with: - aws-access-key-id: "${{ secrets.DEV_AWS_ACCESS_KEY_ID }}" - aws-secret-access-key: "${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }}" - aws-region: eu-west-2 + # This will need to be changed to env imports when we have different env to dynamically allocate prod, staging etc + aws-access-key-id: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ env.AWS_REGION }} - name: Setup Terraform uses: hashicorp/setup-terraform@v3 - - name: Terraform Init - working-directory: ./infrastructure/terraform/shared/ - shell: bash + - name: Terraform Init (shared) + working-directory: infrastructure/terraform/shared run: terraform init -reconfigure - - name: Terraform Workspace - working-directory: ./infrastructure/terraform/shared/ - shell: bash - run: terraform workspace select dev || terraform workspace new dev + - name: Terraform Workspace (shared) + working-directory: infrastructure/terraform/shared + run: | + terraform workspace select ${STAGE} \ + || terraform workspace new ${STAGE} - name: Terraform Plan (shared) - working-directory: ./infrastructure/terraform/shared/ - shell: bash - run: terraform plan -var-file=dev.tfvars -out=tfplan + working-directory: infrastructure/terraform/shared + run: terraform plan -var-file=${STAGE}.tfvars - # - name: Terraform Apply - # working-directory: ./infrastructure/terraform/shared - # shell: bash - # run: terraform apply -auto-approve tfplan - - - - # # apply shared dev # - name: Terraform Apply (shared) - # run: | - # cd infrastructure/terraform/shared - # terraform apply -auto-approve -var-file=dev.tfvars + # working-directory: infrastructure/terraform/shared + # run: terraform apply -auto-approve -var-file=${STAGE}.tfvars - # - name: Build & push image - # run: | - # IMAGE_TAG=address2uprn-${GITHUB_SHA} - # IMAGE_URI=${AWS_ACCOUNT_ID}.dkr.ecr.eu-west-2.amazonaws.com/lambda-shared-dev:${IMAGE_TAG} - - # docker build -t $IMAGE_URI . - # docker push $IMAGE_URI - - # echo "IMAGE_URI=$IMAGE_URI" >> $GITHUB_ENV + # # ============================================================ + # # 2️⃣ Build Docker image (tag = GitHub SHA, digest resolved) + # # ============================================================ + # image: + # uses: ./.github/workflows/_build_docker_image.yml + # with: + # ecr_repo: address2uprn-dev + # aws_region: ${{ env.AWS_REGION }} + # dockerfile_path: backend/address2UPRN/Dockerfile + # build_context: backend/address2UPRN + # secrets: + # AWS_ACCESS_KEY_ID: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }} + # AWS_SECRET_ACCESS_KEY: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }} + # AWS_ACCOUNT_ID: ${{ secrets.AWS_ACCOUNT_ID }} + # # ============================================================ + # # 3️⃣ Deploy Lambda (Terraform, immutable digest) + # # ============================================================ + # deploy_lambda: + # needs: image + # uses: ./.github/workflows/_deploy_lambda.yml + # with: + # lambda_name: address2uprn + # lambda_path: infrastructure/terraform/lambda/address2uprn + # stage: ${{ env.STAGE }} + # aws_region: ${{ env.AWS_REGION }} + # image_digest: ${{ needs.image.outputs.image_digest }} + # secrets: + # AWS_ACCESS_KEY_ID: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }} + # AWS_SECRET_ACCESS_KEY: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }} + # AWS_ACCOUNT_ID: ${{ secrets.AWS_ACCOUNT_ID }} diff --git a/infrastructure/terraform/lambda/_template/README.md b/infrastructure/terraform/lambda/_template/README.md new file mode 100644 index 00000000..2ae9f9af --- /dev/null +++ b/infrastructure/terraform/lambda/_template/README.md @@ -0,0 +1,7 @@ +### Checklist for a new lambda + +- [ ] Copy cp -r lambda/_template lambda/ +- [ ] Set `state_bucket_name` +- [ ] Add ECR repo in shared/main.tf +- [ ] Add shared output for repo name/url +- [ ] Push to GitHub (CI will deploy) diff --git a/infrastructure/terraform/lambda/_template/main.tf b/infrastructure/terraform/lambda/_template/main.tf new file mode 100644 index 00000000..77476cf4 --- /dev/null +++ b/infrastructure/terraform/lambda/_template/main.tf @@ -0,0 +1,21 @@ +data "terraform_remote_state" "shared" { + backend = "s3" + config = { + bucket = "assessment-model-terraform-state" + key = "terraform.tfstate" + region = "eu-west-2" + } +} + +module "lambda" { + source = "../modules/lambda_with_sqs" + + name = "REPLACE_ME" + stage = var.stage + + image_uri = "${data.terraform_remote_state.shared.outputs.REPLACE_ME_repository_url}@${var.image_digest}" + + environment = { + STAGE = var.stage + } +} diff --git a/infrastructure/terraform/lambda/_template/provider.tf b/infrastructure/terraform/lambda/_template/provider.tf new file mode 100644 index 00000000..36cb63f1 --- /dev/null +++ b/infrastructure/terraform/lambda/_template/provider.tf @@ -0,0 +1,20 @@ +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 4.16" + } + } + + backend "s3" { + bucket = var.state_bucket_name + key = "terraform.tfstate" + region = "eu-west-2" + } + + required_version = ">= 1.2.0" +} + +provider "aws" { + region = var.region +} diff --git a/infrastructure/terraform/lambda/_template/variables.tf b/infrastructure/terraform/lambda/_template/variables.tf new file mode 100644 index 00000000..b6f9907a --- /dev/null +++ b/infrastructure/terraform/lambda/_template/variables.tf @@ -0,0 +1,17 @@ +variable "region" { + type = string + default = "eu-west-2" +} + +variable "stage" { + type = string +} + +variable "image_digest" { + type = string +} + +variable "state_bucket_name" { + type = string + description = "S3 bucket name used for this lambda's Terraform state" +} \ No newline at end of file diff --git a/infrastructure/terraform/lambda/address2UPRN/main.tf b/infrastructure/terraform/lambda/address2UPRN/main.tf new file mode 100644 index 00000000..78106c22 --- /dev/null +++ b/infrastructure/terraform/lambda/address2UPRN/main.tf @@ -0,0 +1,88 @@ +############################################ +# Read shared state to get outputs +############################################ +data "terraform_remote_state" "shared" { + backend = "s3" + + config = { + bucket = "assessment-model-terraform-state" + key = "terraform.tfstate" + region = "eu-west-2" + } +} + +############################################ +# IAM role +############################################ +module "role" { + source = "../../modules/lambda_execution_role" + name = "address2uprn-lambda-${var.stage}" +} + +############################################ +# SQS queue +############################################ +module "queue" { + source = "../../modules/sqs_queue" + name = "address2uprn-queue-${var.stage}" +} + +############################################ +# Lambda (image-based) +############################################ +module "lambda" { + source = "../../modules/lambda_service" + + name = "address2uprn-${var.stage}" + role_arn = module.role.role_arn + + image_uri = "${data.terraform_remote_state.shared.outputs.address2uprn_repository_url}@${var.image_digest}" + + timeout = 60 + memory_size = 1024 + + environment = { + STAGE = var.stage + LOG_LEVEL = "info" + } +} + +############################################ +# SQS → Lambda trigger +############################################ +module "sqs_trigger" { + source = "../../modules/lambda_sqs_trigger" + + lambda_arn = module.lambda.lambda_arn + lambda_role_name = module.role.role_name + queue_arn = module.queue.queue_arn +} +############################################ +# Read shared state to get outputs +############################################ +data "terraform_remote_state" "shared" { + backend = "s3" + + config = { + bucket = "assessment-model-terraform-state" + key = "terraform.tfstate" + region = "eu-west-2" + } +} + +############################################ +# Address2UPRN Lambda (via reusable module) +############################################ +module "address2uprn" { + source = "../modules/lambda_with_sqs" + + name = "address2uprn" + stage = var.stage + + image_uri = "${data.terraform_remote_state.shared.outputs.address2uprn_repository_url}@${var.image_digest}" + + environment = { + STAGE = var.stage + LOG_LEVEL = "info" + } +} diff --git a/infrastructure/terraform/lambda/address2UPRN/provider.tf b/infrastructure/terraform/lambda/address2UPRN/provider.tf new file mode 100644 index 00000000..2f4360ec --- /dev/null +++ b/infrastructure/terraform/lambda/address2UPRN/provider.tf @@ -0,0 +1,20 @@ +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 4.16" + } + } + + backend "s3" { + bucket = "address2uprn-terraform-state" + key = "terraform.tfstate" + region = "eu-west-2" + } + + required_version = ">= 1.2.0" +} + +provider "aws" { + region = var.region +} diff --git a/infrastructure/terraform/lambda/address2UPRN/variables.tf b/infrastructure/terraform/lambda/address2UPRN/variables.tf new file mode 100644 index 00000000..1d6c5952 --- /dev/null +++ b/infrastructure/terraform/lambda/address2UPRN/variables.tf @@ -0,0 +1,13 @@ +variable "region" { + type = string + default = "eu-west-2" +} + +variable "stage" { + type = string +} + +variable "image_digest" { + type = string + description = "sha256 image digest from CI" +} diff --git a/infrastructure/terraform/lambda/modules/lambda_with_sqs/main.tf b/infrastructure/terraform/lambda/modules/lambda_with_sqs/main.tf new file mode 100644 index 00000000..3816c206 --- /dev/null +++ b/infrastructure/terraform/lambda/modules/lambda_with_sqs/main.tf @@ -0,0 +1,44 @@ +############################################ +# IAM role +############################################ +module "role" { + source = "../../../modules/lambda_execution_role" + name = "${var.name}-lambda-${var.stage}" +} + +############################################ +# SQS queue + DLQ +############################################ +module "queue" { + source = "../../../modules/sqs_queue" + name = "${var.name}-queue-${var.stage}" +} + +############################################ +# Lambda +############################################ +module "lambda" { + source = "../../../modules/lambda_service" + + name = "${var.name}-${var.stage}" + role_arn = module.role.role_arn + image_uri = var.image_uri + + timeout = var.timeout + memory_size = var.memory_size + + environment = var.environment +} + +############################################ +# SQS → Lambda trigger +############################################ +module "sqs_trigger" { + source = "../../../modules/lambda_sqs_trigger" + + lambda_arn = module.lambda.lambda_arn + lambda_role_name = module.role.role_name + queue_arn = module.queue.queue_arn + + batch_size = var.batch_size +} diff --git a/infrastructure/terraform/lambda/modules/lambda_with_sqs/outputs.tf b/infrastructure/terraform/lambda/modules/lambda_with_sqs/outputs.tf new file mode 100644 index 00000000..afc9246d --- /dev/null +++ b/infrastructure/terraform/lambda/modules/lambda_with_sqs/outputs.tf @@ -0,0 +1,11 @@ +output "lambda_arn" { + value = module.lambda.lambda_arn +} + +output "queue_arn" { + value = module.queue.queue_arn +} + +output "queue_url" { + value = module.queue.queue_url +} diff --git a/infrastructure/terraform/lambda/modules/lambda_with_sqs/variables.tf b/infrastructure/terraform/lambda/modules/lambda_with_sqs/variables.tf new file mode 100644 index 00000000..b20ab2a8 --- /dev/null +++ b/infrastructure/terraform/lambda/modules/lambda_with_sqs/variables.tf @@ -0,0 +1,36 @@ +variable "name" { + type = string +} + +variable "stage" { + type = string +} + +variable "image_uri" { + type = string +} + +variable "region" { + type = string + default = "eu-west-2" +} + +variable "timeout" { + type = number + default = 60 +} + +variable "memory_size" { + type = number + default = 1024 +} + +variable "environment" { + type = map(string) + default = {} +} + +variable "batch_size" { + type = number + default = 10 +} diff --git a/infrastructure/terraform/modules/container_registry/main.tf b/infrastructure/terraform/modules/container_registry/main.tf new file mode 100644 index 00000000..f5ba8d5e --- /dev/null +++ b/infrastructure/terraform/modules/container_registry/main.tf @@ -0,0 +1,30 @@ +resource "aws_ecr_repository" "this" { + name = "${var.name}-${var.stage}" + + image_tag_mutability = "MUTABLE" + + image_scanning_configuration { + scan_on_push = true + } +} + +resource "aws_ecr_lifecycle_policy" "this" { + repository = aws_ecr_repository.this.name + + policy = jsonencode({ + rules = [ + { + rulePriority = 1 + description = "Expire old images" + selection = { + tagStatus = "any" + countType = "imageCountMoreThan" + countNumber = var.retain_count + } + action = { + type = "expire" + } + } + ] + }) +} diff --git a/infrastructure/terraform/modules/container_registry/outputs.tf b/infrastructure/terraform/modules/container_registry/outputs.tf new file mode 100644 index 00000000..47a4bc64 --- /dev/null +++ b/infrastructure/terraform/modules/container_registry/outputs.tf @@ -0,0 +1,11 @@ +output "repository_name" { + value = aws_ecr_repository.this.name +} + +output "repository_url" { + value = aws_ecr_repository.this.repository_url +} + +output "repository_arn" { + value = aws_ecr_repository.this.arn +} diff --git a/infrastructure/terraform/modules/container_registry/variables.tf b/infrastructure/terraform/modules/container_registry/variables.tf new file mode 100644 index 00000000..501a5044 --- /dev/null +++ b/infrastructure/terraform/modules/container_registry/variables.tf @@ -0,0 +1,15 @@ +variable "name" { + description = "Base name of the repository (without stage)" + type = string +} + +variable "stage" { + description = "Deployment stage (e.g. dev, prod)" + type = string +} + +variable "retain_count" { + description = "Number of images to retain" + type = number + default = 20 +} diff --git a/infrastructure/terraform/modules/ecr/main.tf b/infrastructure/terraform/modules/ecr/main.tf index 468ef3d2..9288a1fe 100644 --- a/infrastructure/terraform/modules/ecr/main.tf +++ b/infrastructure/terraform/modules/ecr/main.tf @@ -1,3 +1,8 @@ +# This ECR module is used in Khalim's default code, Junte tried changing it +# but decided to priotise delivariables as sales projects are coming soon +# one day we can unify ECR policies together but Junte decided to seperate +# the continaer lambda as it runs slighly differently + resource "aws_ecr_repository" "my_repository" { name = "${var.ecr_name}" image_tag_mutability = "MUTABLE" diff --git a/infrastructure/terraform/modules/lambda_execution_role/main.tf b/infrastructure/terraform/modules/lambda_execution_role/main.tf new file mode 100644 index 00000000..fa657afd --- /dev/null +++ b/infrastructure/terraform/modules/lambda_execution_role/main.tf @@ -0,0 +1,37 @@ +data "aws_iam_policy_document" "assume" { + statement { + effect = "Allow" + principals { + type = "Service" + identifiers = ["lambda.amazonaws.com"] + } + actions = ["sts:AssumeRole"] + } +} + +resource "aws_iam_role" "this" { + name = var.name + assume_role_policy = data.aws_iam_policy_document.assume.json +} + +resource "aws_iam_role_policy_attachment" "basic_logs" { + role = aws_iam_role.this.name + policy_arn = "arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole" +} + +resource "aws_iam_role_policy" "ecr_pull" { + role = aws_iam_role.this.name + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [{ + Effect = "Allow" + Action = [ + "ecr:GetAuthorizationToken", + "ecr:BatchGetImage", + "ecr:GetDownloadUrlForLayer" + ] + Resource = "*" + }] + }) +} diff --git a/infrastructure/terraform/modules/lambda_execution_role/outputs.tf b/infrastructure/terraform/modules/lambda_execution_role/outputs.tf new file mode 100644 index 00000000..1baca34d --- /dev/null +++ b/infrastructure/terraform/modules/lambda_execution_role/outputs.tf @@ -0,0 +1,7 @@ +output "role_arn" { + value = aws_iam_role.this.arn +} + +output "role_name" { + value = aws_iam_role.this.name +} diff --git a/infrastructure/terraform/modules/lambda_execution_role/variables.tf b/infrastructure/terraform/modules/lambda_execution_role/variables.tf new file mode 100644 index 00000000..e69de29b diff --git a/infrastructure/terraform/modules/lambda_service/main.tf b/infrastructure/terraform/modules/lambda_service/main.tf new file mode 100644 index 00000000..8a159db1 --- /dev/null +++ b/infrastructure/terraform/modules/lambda_service/main.tf @@ -0,0 +1,15 @@ +resource "aws_lambda_function" "this" { + function_name = var.name + role = var.role_arn + + package_type = "Image" + image_uri = var.image_uri + + timeout = var.timeout + memory_size = var.memory_size + publish = true + + environment { + variables = var.environment + } +} diff --git a/infrastructure/terraform/modules/lambda_service/outputs.tf b/infrastructure/terraform/modules/lambda_service/outputs.tf new file mode 100644 index 00000000..dd05cccf --- /dev/null +++ b/infrastructure/terraform/modules/lambda_service/outputs.tf @@ -0,0 +1,3 @@ +output "lambda_arn" { + value = aws_lambda_function.this.arn +} diff --git a/infrastructure/terraform/modules/lambda_service/variables.tf b/infrastructure/terraform/modules/lambda_service/variables.tf new file mode 100644 index 00000000..43def6ad --- /dev/null +++ b/infrastructure/terraform/modules/lambda_service/variables.tf @@ -0,0 +1,18 @@ +variable "name" { type = string } +variable "role_arn" { type = string } +variable "image_uri" { type = string } + +variable "timeout" { + type = number + default = 30 +} + +variable "memory_size" { + type = number + default = 512 +} + +variable "environment" { + type = map(string) + default = {} +} diff --git a/infrastructure/terraform/modules/lambda_sqs_trigger/main.tf b/infrastructure/terraform/modules/lambda_sqs_trigger/main.tf new file mode 100644 index 00000000..5919e10f --- /dev/null +++ b/infrastructure/terraform/modules/lambda_sqs_trigger/main.tf @@ -0,0 +1,23 @@ +resource "aws_lambda_event_source_mapping" "this" { + event_source_arn = var.queue_arn + function_name = var.lambda_arn + batch_size = var.batch_size + enabled = true +} + +resource "aws_iam_role_policy" "allow_sqs" { + role = var.lambda_role_name + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [{ + Effect = "Allow" + Action = [ + "sqs:ReceiveMessage", + "sqs:DeleteMessage", + "sqs:GetQueueAttributes" + ] + Resource = var.queue_arn + }] + }) +} diff --git a/infrastructure/terraform/modules/lambda_sqs_trigger/variables.tf b/infrastructure/terraform/modules/lambda_sqs_trigger/variables.tf new file mode 100644 index 00000000..0e50cd54 --- /dev/null +++ b/infrastructure/terraform/modules/lambda_sqs_trigger/variables.tf @@ -0,0 +1,8 @@ +variable "lambda_arn" { type = string } +variable "lambda_role_name" { type = string } +variable "queue_arn" { type = string } + +variable "batch_size" { + type = number + default = 10 +} diff --git a/infrastructure/terraform/modules/lambda_with_sqs/main.tf b/infrastructure/terraform/modules/lambda_with_sqs/main.tf deleted file mode 100644 index 1b4e1847..00000000 --- a/infrastructure/terraform/modules/lambda_with_sqs/main.tf +++ /dev/null @@ -1,23 +0,0 @@ -resource "aws_sqs_queue" "this" { - name = "${var.name}-queue" - tags = var.tags -} - -resource "aws_lambda_function" "this" { - function_name = var.name - role = var.lambda_role_arn - - package_type = "Image" - image_uri = var.image_uri - - timeout = var.timeout - - tags = var.tags -} - -resource "aws_lambda_event_source_mapping" "this" { - event_source_arn = aws_sqs_queue.this.arn - function_name = aws_lambda_function.this.arn - - batch_size = var.sqs_batch_size -} diff --git a/infrastructure/terraform/modules/lambda_with_sqs/outputs.tf b/infrastructure/terraform/modules/lambda_with_sqs/outputs.tf deleted file mode 100644 index dc755850..00000000 --- a/infrastructure/terraform/modules/lambda_with_sqs/outputs.tf +++ /dev/null @@ -1,15 +0,0 @@ -output "lambda_name" { - value = aws_lambda_function.this.function_name -} - -output "lambda_arn" { - value = aws_lambda_function.this.arn -} - -output "sqs_queue_url" { - value = aws_sqs_queue.this.url -} - -output "sqs_queue_arn" { - value = aws_sqs_queue.this.arn -} diff --git a/infrastructure/terraform/modules/lambda_with_sqs/variables.tf b/infrastructure/terraform/modules/lambda_with_sqs/variables.tf deleted file mode 100644 index 8ac24942..00000000 --- a/infrastructure/terraform/modules/lambda_with_sqs/variables.tf +++ /dev/null @@ -1,32 +0,0 @@ -variable "name" { - description = "Base name for lambda and related resources" - type = string -} - -variable "image_uri" { - description = "ECR image URI with tag" - type = string -} - -variable "lambda_role_arn" { - description = "IAM role ARN for Lambda execution" - type = string -} - -variable "timeout" { - description = "Lambda timeout in seconds" - type = number - default = 10 -} - -variable "sqs_batch_size" { - description = "Number of SQS messages per batch" - type = number - default = 1 -} - -variable "tags" { - description = "Tags to apply to resources" - type = map(string) - default = {} -} diff --git a/infrastructure/terraform/modules/sqs_queue/main.tf b/infrastructure/terraform/modules/sqs_queue/main.tf new file mode 100644 index 00000000..478c345c --- /dev/null +++ b/infrastructure/terraform/modules/sqs_queue/main.tf @@ -0,0 +1,12 @@ +resource "aws_sqs_queue" "dlq" { + name = "${var.name}-dlq" +} + +resource "aws_sqs_queue" "this" { + name = var.name + + redrive_policy = jsonencode({ + deadLetterTargetArn = aws_sqs_queue.dlq.arn + maxReceiveCount = var.max_receive_count + }) +} diff --git a/infrastructure/terraform/modules/sqs_queue/outputs.tf b/infrastructure/terraform/modules/sqs_queue/outputs.tf new file mode 100644 index 00000000..46fafe90 --- /dev/null +++ b/infrastructure/terraform/modules/sqs_queue/outputs.tf @@ -0,0 +1,7 @@ +output "queue_arn" { + value = aws_sqs_queue.this.arn +} + +output "queue_url" { + value = aws_sqs_queue.this.url +} diff --git a/infrastructure/terraform/modules/sqs_queue/variables.tf b/infrastructure/terraform/modules/sqs_queue/variables.tf new file mode 100644 index 00000000..943a7a16 --- /dev/null +++ b/infrastructure/terraform/modules/sqs_queue/variables.tf @@ -0,0 +1,6 @@ +variable "name" { type = string } + +variable "max_receive_count" { + type = number + default = 5 +} diff --git a/infrastructure/terraform/modules/tf_state_bucket/main.tf b/infrastructure/terraform/modules/tf_state_bucket/main.tf new file mode 100644 index 00000000..86c0cc21 --- /dev/null +++ b/infrastructure/terraform/modules/tf_state_bucket/main.tf @@ -0,0 +1,30 @@ +resource "aws_s3_bucket" "this" { + bucket = var.bucket_name +} + +resource "aws_s3_bucket_versioning" "this" { + bucket = aws_s3_bucket.this.id + + versioning_configuration { + status = "Enabled" + } +} + +resource "aws_s3_bucket_server_side_encryption_configuration" "this" { + bucket = aws_s3_bucket.this.id + + rule { + apply_server_side_encryption_by_default { + sse_algorithm = "AES256" + } + } +} + +resource "aws_s3_bucket_public_access_block" "this" { + bucket = aws_s3_bucket.this.id + + block_public_acls = true + block_public_policy = true + ignore_public_acls = true + restrict_public_buckets = true +} diff --git a/infrastructure/terraform/modules/tf_state_bucket/outputs.tf b/infrastructure/terraform/modules/tf_state_bucket/outputs.tf new file mode 100644 index 00000000..e8ceffd1 --- /dev/null +++ b/infrastructure/terraform/modules/tf_state_bucket/outputs.tf @@ -0,0 +1,7 @@ +output "bucket_name" { + value = aws_s3_bucket.this.bucket +} + +output "bucket_arn" { + value = aws_s3_bucket.this.arn +} diff --git a/infrastructure/terraform/modules/tf_state_bucket/variables.tf b/infrastructure/terraform/modules/tf_state_bucket/variables.tf new file mode 100644 index 00000000..b3aae9bb --- /dev/null +++ b/infrastructure/terraform/modules/tf_state_bucket/variables.tf @@ -0,0 +1,3 @@ +variable "bucket_name" { + type = string +} diff --git a/infrastructure/terraform/shared/locals.tf b/infrastructure/terraform/shared/locals.tf new file mode 100644 index 00000000..e69de29b diff --git a/infrastructure/terraform/shared/main.tf b/infrastructure/terraform/shared/main.tf index facc6ea5..f6475362 100644 --- a/infrastructure/terraform/shared/main.tf +++ b/infrastructure/terraform/shared/main.tf @@ -8,7 +8,6 @@ terraform { backend "s3" { bucket = "assessment-model-terraform-state" region = "eu-west-2" - # profile = "DevAdmin" key = "terraform.tfstate" } @@ -290,11 +289,23 @@ output "ses_dns_records" { value = module.ses.dns_records } +################################################ +# Address2UPRN – Lambda ECR +################################################ +module "address2uprn_state_bucket" { + source = "../modules/tf_state_bucket" + bucket_name = "address2uprn-terraform-state" +} -################################################ -# One ECR to rule all the lambdas -################################################ -module "lambda_shared_ecr" { - source = "../modules/ecr" - ecr_name = "lambda-shared-${var.stage}" +output "address2uprn_state_bucket_name" { + value = module.address2uprn_state_bucket.bucket_name +} + +module "address2uprn_registry" { + source = "../modules/container_registry" + name = "address2uprn-${var.stage}" +} + +output "address2uprn_repository_url" { + value = module.address2uprn_registry.repository_url } \ No newline at end of file diff --git a/infrastructure/terraform/shared/variables.tf b/infrastructure/terraform/shared/variables.tf index d00170d2..e922e465 100644 --- a/infrastructure/terraform/shared/variables.tf +++ b/infrastructure/terraform/shared/variables.tf @@ -3,7 +3,6 @@ variable stage { type = string } - variable "region" { description = "AWS region" type = string From 8c5f49976f0da86be01830297db81c662e2fd5c3 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Tue, 3 Feb 2026 11:25:45 +0000 Subject: [PATCH 36/86] pass var stage --- .github/workflows/deploy_terraform.yml | 1 + infrastructure/terraform/shared/locals.tf | 0 infrastructure/terraform/shared/main.tf | 5 ++++- 3 files changed, 5 insertions(+), 1 deletion(-) delete mode 100644 infrastructure/terraform/shared/locals.tf diff --git a/.github/workflows/deploy_terraform.yml b/.github/workflows/deploy_terraform.yml index 5ebe0216..6f95b165 100644 --- a/.github/workflows/deploy_terraform.yml +++ b/.github/workflows/deploy_terraform.yml @@ -57,6 +57,7 @@ jobs: # image: # uses: ./.github/workflows/_build_docker_image.yml # with: + # ecr_repo will need to changed to dynamic env in the future # ecr_repo: address2uprn-dev # aws_region: ${{ env.AWS_REGION }} # dockerfile_path: backend/address2UPRN/Dockerfile diff --git a/infrastructure/terraform/shared/locals.tf b/infrastructure/terraform/shared/locals.tf deleted file mode 100644 index e69de29b..00000000 diff --git a/infrastructure/terraform/shared/main.tf b/infrastructure/terraform/shared/main.tf index f6475362..acb1b070 100644 --- a/infrastructure/terraform/shared/main.tf +++ b/infrastructure/terraform/shared/main.tf @@ -294,7 +294,8 @@ output "ses_dns_records" { ################################################ module "address2uprn_state_bucket" { source = "../modules/tf_state_bucket" - bucket_name = "address2uprn-terraform-state" + bucket_name = "address2uprn-terraform-state-${var.stage}" + } output "address2uprn_state_bucket_name" { @@ -304,6 +305,8 @@ output "address2uprn_state_bucket_name" { module "address2uprn_registry" { source = "../modules/container_registry" name = "address2uprn-${var.stage}" + stage = var.stage + } output "address2uprn_repository_url" { From a8026eb490eb734f65d6ea7e866a3f6571cdd707 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Tue, 3 Feb 2026 11:29:22 +0000 Subject: [PATCH 37/86] name is reused to stage anyway --- infrastructure/terraform/shared/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infrastructure/terraform/shared/main.tf b/infrastructure/terraform/shared/main.tf index acb1b070..424204a3 100644 --- a/infrastructure/terraform/shared/main.tf +++ b/infrastructure/terraform/shared/main.tf @@ -304,7 +304,7 @@ output "address2uprn_state_bucket_name" { module "address2uprn_registry" { source = "../modules/container_registry" - name = "address2uprn-${var.stage}" + name = "address2uprn" stage = var.stage } From bf24e75580fdb8304d9079eb5b428635a0b01714 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Tue, 3 Feb 2026 11:42:36 +0000 Subject: [PATCH 38/86] apply --- .github/workflows/deploy_terraform.yml | 36 +++++++++++++------------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/.github/workflows/deploy_terraform.yml b/.github/workflows/deploy_terraform.yml index 6f95b165..58f61495 100644 --- a/.github/workflows/deploy_terraform.yml +++ b/.github/workflows/deploy_terraform.yml @@ -47,25 +47,25 @@ jobs: working-directory: infrastructure/terraform/shared run: terraform plan -var-file=${STAGE}.tfvars - # - name: Terraform Apply (shared) - # working-directory: infrastructure/terraform/shared - # run: terraform apply -auto-approve -var-file=${STAGE}.tfvars + - name: Terraform Apply (shared) + working-directory: infrastructure/terraform/shared + run: terraform apply -auto-approve -var-file=${STAGE}.tfvars - # # ============================================================ - # # 2️⃣ Build Docker image (tag = GitHub SHA, digest resolved) - # # ============================================================ - # image: - # uses: ./.github/workflows/_build_docker_image.yml - # with: - # ecr_repo will need to changed to dynamic env in the future - # ecr_repo: address2uprn-dev - # aws_region: ${{ env.AWS_REGION }} - # dockerfile_path: backend/address2UPRN/Dockerfile - # build_context: backend/address2UPRN - # secrets: - # AWS_ACCESS_KEY_ID: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }} - # AWS_SECRET_ACCESS_KEY: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }} - # AWS_ACCOUNT_ID: ${{ secrets.AWS_ACCOUNT_ID }} + # ============================================================ + # 2️⃣ Build Docker image (tag = GitHub SHA, digest resolved) + # ============================================================ + image: + uses: ./.github/workflows/_build_docker_image.yml + with: + # ecr_repo will need to changed to dynamic env in the future + ecr_repo: address2uprn-dev + aws_region: ${{ env.AWS_REGION }} + dockerfile_path: backend/address2UPRN/Dockerfile + build_context: backend/address2UPRN + secrets: + AWS_ACCESS_KEY_ID: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }} + AWS_ACCOUNT_ID: ${{ secrets.AWS_ACCOUNT_ID }} # # ============================================================ # # 3️⃣ Deploy Lambda (Terraform, immutable digest) From 0789e5022465e3b5fa44d859bb5a456a9e9e5d6d Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Tue, 3 Feb 2026 11:43:48 +0000 Subject: [PATCH 39/86] just apply --- .github/workflows/deploy_terraform.yml | 34 ++++++++++++-------------- 1 file changed, 16 insertions(+), 18 deletions(-) diff --git a/.github/workflows/deploy_terraform.yml b/.github/workflows/deploy_terraform.yml index 58f61495..8179a58a 100644 --- a/.github/workflows/deploy_terraform.yml +++ b/.github/workflows/deploy_terraform.yml @@ -39,9 +39,7 @@ jobs: - name: Terraform Workspace (shared) working-directory: infrastructure/terraform/shared - run: | - terraform workspace select ${STAGE} \ - || terraform workspace new ${STAGE} + run: terraform workspace select ${STAGE} || terraform workspace new ${STAGE} - name: Terraform Plan (shared) working-directory: infrastructure/terraform/shared @@ -51,21 +49,21 @@ jobs: working-directory: infrastructure/terraform/shared run: terraform apply -auto-approve -var-file=${STAGE}.tfvars - # ============================================================ - # 2️⃣ Build Docker image (tag = GitHub SHA, digest resolved) - # ============================================================ - image: - uses: ./.github/workflows/_build_docker_image.yml - with: - # ecr_repo will need to changed to dynamic env in the future - ecr_repo: address2uprn-dev - aws_region: ${{ env.AWS_REGION }} - dockerfile_path: backend/address2UPRN/Dockerfile - build_context: backend/address2UPRN - secrets: - AWS_ACCESS_KEY_ID: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }} - AWS_ACCOUNT_ID: ${{ secrets.AWS_ACCOUNT_ID }} + # # ============================================================ + # # 2️⃣ Build Docker image (tag = GitHub SHA, digest resolved) + # # ============================================================ + # image: + # uses: ./.github/workflows/_build_docker_image.yml + # with: + # # ecr_repo will need to changed to dynamic env in the future + # ecr_repo: address2uprn-dev + # aws_region: ${{ env.AWS_REGION }} + # dockerfile_path: backend/address2UPRN/Dockerfile + # build_context: backend/address2UPRN + # secrets: + # AWS_ACCESS_KEY_ID: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }} + # AWS_SECRET_ACCESS_KEY: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }} + # AWS_ACCOUNT_ID: ${{ secrets.AWS_ACCOUNT_ID }} # # ============================================================ # # 3️⃣ Deploy Lambda (Terraform, immutable digest) From c6a33f30156ce441bedf7567c19472a1627fdbe5 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Tue, 3 Feb 2026 11:46:33 +0000 Subject: [PATCH 40/86] add docker build --- .github/workflows/deploy_terraform.yml | 38 +++++++++++++------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/.github/workflows/deploy_terraform.yml b/.github/workflows/deploy_terraform.yml index 8179a58a..3bbcaaf5 100644 --- a/.github/workflows/deploy_terraform.yml +++ b/.github/workflows/deploy_terraform.yml @@ -43,27 +43,27 @@ jobs: - name: Terraform Plan (shared) working-directory: infrastructure/terraform/shared - run: terraform plan -var-file=${STAGE}.tfvars + run: terraform plan -var-file=${STAGE}.tfvars -out=tfplan - - name: Terraform Apply (shared) - working-directory: infrastructure/terraform/shared - run: terraform apply -auto-approve -var-file=${STAGE}.tfvars + # - name: Terraform Apply (shared) + # working-directory: infrastructure/terraform/shared + # run: terraform apply -auto-approve -var-file=${STAGE}.tfvars tfplan - # # ============================================================ - # # 2️⃣ Build Docker image (tag = GitHub SHA, digest resolved) - # # ============================================================ - # image: - # uses: ./.github/workflows/_build_docker_image.yml - # with: - # # ecr_repo will need to changed to dynamic env in the future - # ecr_repo: address2uprn-dev - # aws_region: ${{ env.AWS_REGION }} - # dockerfile_path: backend/address2UPRN/Dockerfile - # build_context: backend/address2UPRN - # secrets: - # AWS_ACCESS_KEY_ID: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }} - # AWS_SECRET_ACCESS_KEY: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }} - # AWS_ACCOUNT_ID: ${{ secrets.AWS_ACCOUNT_ID }} + # ============================================================ + # 2️⃣ Build Docker image (tag = GitHub SHA, digest resolved) + # ============================================================ + image: + uses: ./.github/workflows/_build_docker_image.yml + with: + # ecr_repo will need to changed to dynamic env in the future + ecr_repo: address2uprn-dev + aws_region: ${{ env.AWS_REGION }} + dockerfile_path: backend/address2UPRN/Dockerfile + build_context: backend/address2UPRN + secrets: + AWS_ACCESS_KEY_ID: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }} + AWS_ACCOUNT_ID: ${{ secrets.AWS_ACCOUNT_ID }} # # ============================================================ # # 3️⃣ Deploy Lambda (Terraform, immutable digest) From 7b1ebca7905237492daed01f21c3d19c4ad2ddd5 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Tue, 3 Feb 2026 12:03:32 +0000 Subject: [PATCH 41/86] build image --- .github/workflows/deploy_terraform.yml | 49 ++++++++++++++++++-------- 1 file changed, 34 insertions(+), 15 deletions(-) diff --git a/.github/workflows/deploy_terraform.yml b/.github/workflows/deploy_terraform.yml index 3bbcaaf5..b84f0bc1 100644 --- a/.github/workflows/deploy_terraform.yml +++ b/.github/workflows/deploy_terraform.yml @@ -5,18 +5,37 @@ on: branches: - "**" -env: - AWS_REGION: eu-west-2 - - # Temporary until we have more environemnts. You'll just need export STAGE dynamically in the future - STAGE: dev jobs: + determine_stage: + runs-on: ubuntu-latest + outputs: + stage: ${{ steps.set-stage.outputs.stage }} + + steps: + - name: Determine stage from branch + id: set-stage + shell: bash + run: | + BRANCH="${GITHUB_REF_NAME}" + + if [[ "$BRANCH" == "prod" ]]; then + echo "stage=prod" >> "$GITHUB_OUTPUT" + elif [[ "$BRANCH" == "dev" ]]; then + echo "stage=dev" >> "$GITHUB_OUTPUT" + else + echo "stage=dev" >> "$GITHUB_OUTPUT" + fi + + echo "Resolved STAGE=$BRANCH → $(cat $GITHUB_OUTPUT)" # ============================================================ # 1️⃣ Shared Terraform (plan only for now) # ============================================================ shared_terraform: + needs: determine_stage runs-on: ubuntu-latest + env: + STAGE: ${{ needs.determine_stage.outputs.stage }} steps: - name: Checkout @@ -28,7 +47,7 @@ jobs: # This will need to be changed to env imports when we have different env to dynamically allocate prod, staging etc aws-access-key-id: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ env.AWS_REGION }} + aws-region: ${{ secrets.DEV_AWS_REGION }} - name: Setup Terraform uses: hashicorp/setup-terraform@v3 @@ -53,11 +72,11 @@ jobs: # 2️⃣ Build Docker image (tag = GitHub SHA, digest resolved) # ============================================================ image: + needs: determine_stage uses: ./.github/workflows/_build_docker_image.yml with: - # ecr_repo will need to changed to dynamic env in the future - ecr_repo: address2uprn-dev - aws_region: ${{ env.AWS_REGION }} + ecr_repo: address2uprn-${{ needs.determine_stage.outputs.stage }} + aws_region: ${{ secrets.DEV_AWS_REGION }} dockerfile_path: backend/address2UPRN/Dockerfile build_context: backend/address2UPRN secrets: @@ -65,17 +84,17 @@ jobs: AWS_SECRET_ACCESS_KEY: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }} AWS_ACCOUNT_ID: ${{ secrets.AWS_ACCOUNT_ID }} - # # ============================================================ - # # 3️⃣ Deploy Lambda (Terraform, immutable digest) - # # ============================================================ + # # # ============================================================ + # # # 3️⃣ Deploy Lambda (Terraform, immutable digest) + # # # ============================================================ # deploy_lambda: - # needs: image + # needs: [image, determine_stage] # uses: ./.github/workflows/_deploy_lambda.yml # with: # lambda_name: address2uprn # lambda_path: infrastructure/terraform/lambda/address2uprn - # stage: ${{ env.STAGE }} - # aws_region: ${{ env.AWS_REGION }} + # stage: ${{ needs.determine_stage.outputs.stage }} + # aws_region: ${{ secrets.DEV_AWS_REGION }} # image_digest: ${{ needs.image.outputs.image_digest }} # secrets: # AWS_ACCESS_KEY_ID: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }} From 33811cbb9a8a1018adf75379c2930e57b9b5dd41 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Tue, 3 Feb 2026 12:08:36 +0000 Subject: [PATCH 42/86] use aws secrets instead --- .github/workflows/_build_image.yml | 16 +++++++--------- .github/workflows/deploy_terraform.yml | 5 ++++- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/.github/workflows/_build_image.yml b/.github/workflows/_build_image.yml index 8ecd241e..34a4c6a9 100644 --- a/.github/workflows/_build_image.yml +++ b/.github/workflows/_build_image.yml @@ -8,11 +8,6 @@ on: required: true type: string - aws_region: - description: "AWS region" - required: true - type: string - dockerfile_path: description: "Path to Dockerfile" required: true @@ -36,6 +31,8 @@ on: required: true AWS_ACCOUNT_ID: required: true + AWS_REGION: + required: true jobs: build: @@ -47,18 +44,19 @@ jobs: steps: - uses: actions/checkout@v4 - - uses: aws-actions/configure-aws-credentials@v4 + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 with: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ inputs.aws_region }} + aws-region: ${{ secrets.AWS_REGION }} - uses: aws-actions/amazon-ecr-login@v2 - name: Build & push image run: | IMAGE_TAG=${GITHUB_SHA} - IMAGE_URI=${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ inputs.aws_region }}.amazonaws.com/${{ inputs.ecr_repo }}:${IMAGE_TAG} + IMAGE_URI=${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ secrets.AWS_REGION }}.amazonaws.com/${{ inputs.ecr_repo }}:${IMAGE_TAG} docker build \ -f ${{ inputs.dockerfile_path }} \ @@ -76,4 +74,4 @@ jobs: --query 'imageDetails[0].imageDigest' \ --output text) - echo "image_digest=$DIGEST" >> $GITHUB_OUTPUT + echo "image_digest=$DIGEST" >> $GITHUB_OUTPUT \ No newline at end of file diff --git a/.github/workflows/deploy_terraform.yml b/.github/workflows/deploy_terraform.yml index b84f0bc1..2d62845a 100644 --- a/.github/workflows/deploy_terraform.yml +++ b/.github/workflows/deploy_terraform.yml @@ -76,13 +76,16 @@ jobs: uses: ./.github/workflows/_build_docker_image.yml with: ecr_repo: address2uprn-${{ needs.determine_stage.outputs.stage }} - aws_region: ${{ secrets.DEV_AWS_REGION }} dockerfile_path: backend/address2UPRN/Dockerfile build_context: backend/address2UPRN secrets: AWS_ACCESS_KEY_ID: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }} AWS_ACCOUNT_ID: ${{ secrets.AWS_ACCOUNT_ID }} + AWS_REGION: ${{ secrets.DEV_AWS_REGION }} + + + # # # ============================================================ # # # 3️⃣ Deploy Lambda (Terraform, immutable digest) From ea75c4dff3a80ec4899d44bff2f9584e15436976 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Tue, 3 Feb 2026 12:10:22 +0000 Subject: [PATCH 43/86] wrong workflow name --- .github/workflows/deploy_terraform.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy_terraform.yml b/.github/workflows/deploy_terraform.yml index 2d62845a..cf7640c8 100644 --- a/.github/workflows/deploy_terraform.yml +++ b/.github/workflows/deploy_terraform.yml @@ -73,7 +73,7 @@ jobs: # ============================================================ image: needs: determine_stage - uses: ./.github/workflows/_build_docker_image.yml + uses: ./.github/workflows/_build_image.yml with: ecr_repo: address2uprn-${{ needs.determine_stage.outputs.stage }} dockerfile_path: backend/address2UPRN/Dockerfile From c924306dcc34b41f7fee1cd2b328906800316748 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Tue, 3 Feb 2026 12:13:13 +0000 Subject: [PATCH 44/86] wrong account id --- .github/workflows/deploy_terraform.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy_terraform.yml b/.github/workflows/deploy_terraform.yml index cf7640c8..ab7c4df3 100644 --- a/.github/workflows/deploy_terraform.yml +++ b/.github/workflows/deploy_terraform.yml @@ -81,7 +81,7 @@ jobs: secrets: AWS_ACCESS_KEY_ID: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }} - AWS_ACCOUNT_ID: ${{ secrets.AWS_ACCOUNT_ID }} + AWS_ACCOUNT_ID: ${{ secrets.DEV_AWS_ACCOUNT_ID }} AWS_REGION: ${{ secrets.DEV_AWS_REGION }} From a225e248bbbcd0c5dcaad9269119fe2425fd3a43 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Tue, 3 Feb 2026 12:16:41 +0000 Subject: [PATCH 45/86] wrong account id --- .github/workflows/_build_image.yml | 8 +++++--- .github/workflows/deploy_terraform.yml | 1 - 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/_build_image.yml b/.github/workflows/_build_image.yml index 34a4c6a9..1008d592 100644 --- a/.github/workflows/_build_image.yml +++ b/.github/workflows/_build_image.yml @@ -29,8 +29,6 @@ on: required: true AWS_SECRET_ACCESS_KEY: required: true - AWS_ACCOUNT_ID: - required: true AWS_REGION: required: true @@ -56,7 +54,11 @@ jobs: - name: Build & push image run: | IMAGE_TAG=${GITHUB_SHA} - IMAGE_URI=${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ secrets.AWS_REGION }}.amazonaws.com/${{ inputs.ecr_repo }}:${IMAGE_TAG} + AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text) + + IMAGE_URI=${AWS_ACCOUNT_ID}.dkr.ecr.${{ secrets.AWS_REGION }}.amazonaws.com/${{ inputs.ecr_repo }}:${IMAGE_TAG} + + echo "Using IMAGE_URI=$IMAGE_URI" docker build \ -f ${{ inputs.dockerfile_path }} \ diff --git a/.github/workflows/deploy_terraform.yml b/.github/workflows/deploy_terraform.yml index ab7c4df3..4713c8c7 100644 --- a/.github/workflows/deploy_terraform.yml +++ b/.github/workflows/deploy_terraform.yml @@ -81,7 +81,6 @@ jobs: secrets: AWS_ACCESS_KEY_ID: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }} - AWS_ACCOUNT_ID: ${{ secrets.DEV_AWS_ACCOUNT_ID }} AWS_REGION: ${{ secrets.DEV_AWS_REGION }} From c675bf9c12aa233eb727120ea2f31bc7c5fd6e57 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Tue, 3 Feb 2026 12:18:21 +0000 Subject: [PATCH 46/86] fix dockerfile --- backend/address2UPRN/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/address2UPRN/Dockerfile b/backend/address2UPRN/Dockerfile index d7485a3f..ac6af2a5 100644 --- a/backend/address2UPRN/Dockerfile +++ b/backend/address2UPRN/Dockerfile @@ -1,7 +1,7 @@ FROM public.ecr.aws/lambda/python:3.10 # Copy function code -COPY app.py ${LAMBDA_TASK_ROOT} +COPY main.py . # Set the handler CMD ["main.handler"] From d96ad1c6b0876ab57b17443230cc45873a465fb3 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Tue, 3 Feb 2026 12:32:48 +0000 Subject: [PATCH 47/86] check if lambda plan works --- .github/workflows/_deploy_lambda.yml | 16 ++++++------ .github/workflows/deploy_terraform.yml | 34 ++++++++++++-------------- 2 files changed, 23 insertions(+), 27 deletions(-) diff --git a/.github/workflows/_deploy_lambda.yml b/.github/workflows/_deploy_lambda.yml index a218a3d7..5abbbd99 100644 --- a/.github/workflows/_deploy_lambda.yml +++ b/.github/workflows/_deploy_lambda.yml @@ -12,9 +12,6 @@ on: stage: required: true type: string - aws_region: - required: true - type: string image_digest: required: true type: string @@ -24,8 +21,9 @@ on: required: true AWS_SECRET_ACCESS_KEY: required: true - AWS_ACCOUNT_ID: - required: true + AWS_REGION: + required: true + jobs: deploy: @@ -38,7 +36,7 @@ jobs: with: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ inputs.aws_region }} + aws-region: ${{ secrets.AWS_REGION }} - uses: hashicorp/setup-terraform@v3 @@ -57,7 +55,8 @@ jobs: run: | terraform plan \ -var="stage=${{ inputs.stage }}" \ - -var="image_digest=${{ inputs.image_digest }}" + -var="image_digest=${{ inputs.image_digest }}" \ + -out=lambdaplan # - name: Terraform Apply # working-directory: ${{ inputs.lambda_path }} @@ -65,4 +64,5 @@ jobs: # terraform apply \ # -auto-approve \ # -var="stage=${{ inputs.stage }}" \ - # -var="image_digest=${{ inputs.image_digest }}" + # -var="image_digest=${{ inputs.image_digest }}" \ + # lambdaplan diff --git a/.github/workflows/deploy_terraform.yml b/.github/workflows/deploy_terraform.yml index 4713c8c7..9fb3787f 100644 --- a/.github/workflows/deploy_terraform.yml +++ b/.github/workflows/deploy_terraform.yml @@ -83,22 +83,18 @@ jobs: AWS_SECRET_ACCESS_KEY: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }} AWS_REGION: ${{ secrets.DEV_AWS_REGION }} - - - - # # # ============================================================ - # # # 3️⃣ Deploy Lambda (Terraform, immutable digest) - # # # ============================================================ - # deploy_lambda: - # needs: [image, determine_stage] - # uses: ./.github/workflows/_deploy_lambda.yml - # with: - # lambda_name: address2uprn - # lambda_path: infrastructure/terraform/lambda/address2uprn - # stage: ${{ needs.determine_stage.outputs.stage }} - # aws_region: ${{ secrets.DEV_AWS_REGION }} - # image_digest: ${{ needs.image.outputs.image_digest }} - # secrets: - # AWS_ACCESS_KEY_ID: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }} - # AWS_SECRET_ACCESS_KEY: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }} - # AWS_ACCOUNT_ID: ${{ secrets.AWS_ACCOUNT_ID }} + # # ============================================================ + # # 3️⃣ Deploy Lambda (Terraform, immutable digest) + # # ============================================================ + deploy_lambda: + needs: [image, determine_stage] + uses: ./.github/workflows/_deploy_lambda.yml + with: + lambda_name: address2uprn + lambda_path: infrastructure/terraform/lambda/address2uprn + stage: ${{ needs.determine_stage.outputs.stage }} + image_digest: ${{ needs.image.outputs.image_digest }} + secrets: + AWS_ACCESS_KEY_ID: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }} + AWS_REGION: ${{ secrets.DEV_AWS_REGION }} From 75146e67391923c87efc18929d4ebe6f31123383 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Tue, 3 Feb 2026 12:35:54 +0000 Subject: [PATCH 48/86] filename --- .github/workflows/deploy_terraform.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/deploy_terraform.yml b/.github/workflows/deploy_terraform.yml index 9fb3787f..6864570d 100644 --- a/.github/workflows/deploy_terraform.yml +++ b/.github/workflows/deploy_terraform.yml @@ -71,7 +71,7 @@ jobs: # ============================================================ # 2️⃣ Build Docker image (tag = GitHub SHA, digest resolved) # ============================================================ - image: + address2uprn_image: needs: determine_stage uses: ./.github/workflows/_build_image.yml with: @@ -86,12 +86,12 @@ jobs: # # ============================================================ # # 3️⃣ Deploy Lambda (Terraform, immutable digest) # # ============================================================ - deploy_lambda: - needs: [image, determine_stage] + deploy_address2uprn_lambda: + needs: [address2uprn_image, determine_stage] uses: ./.github/workflows/_deploy_lambda.yml with: - lambda_name: address2uprn - lambda_path: infrastructure/terraform/lambda/address2uprn + lambda_name: address2UPRN + lambda_path: infrastructure/terraform/lambda/address2UPRN stage: ${{ needs.determine_stage.outputs.stage }} image_digest: ${{ needs.image.outputs.image_digest }} secrets: From 1b649455091ab79b33561228bc61e1d246170420 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Tue, 3 Feb 2026 12:39:21 +0000 Subject: [PATCH 49/86] remove duplicate --- .../terraform/lambda/address2UPRN/main.tf | 29 ------------------- 1 file changed, 29 deletions(-) diff --git a/infrastructure/terraform/lambda/address2UPRN/main.tf b/infrastructure/terraform/lambda/address2UPRN/main.tf index 78106c22..a0c4d3af 100644 --- a/infrastructure/terraform/lambda/address2UPRN/main.tf +++ b/infrastructure/terraform/lambda/address2UPRN/main.tf @@ -57,32 +57,3 @@ module "sqs_trigger" { lambda_role_name = module.role.role_name queue_arn = module.queue.queue_arn } -############################################ -# Read shared state to get outputs -############################################ -data "terraform_remote_state" "shared" { - backend = "s3" - - config = { - bucket = "assessment-model-terraform-state" - key = "terraform.tfstate" - region = "eu-west-2" - } -} - -############################################ -# Address2UPRN Lambda (via reusable module) -############################################ -module "address2uprn" { - source = "../modules/lambda_with_sqs" - - name = "address2uprn" - stage = var.stage - - image_uri = "${data.terraform_remote_state.shared.outputs.address2uprn_repository_url}@${var.image_digest}" - - environment = { - STAGE = var.stage - LOG_LEVEL = "info" - } -} From 41595940427921c0994bf052d7164b1207c3e37d Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Tue, 3 Feb 2026 12:42:35 +0000 Subject: [PATCH 50/86] make it more modular --- .../terraform/lambda/_template/README.md | 3 ++ .../terraform/lambda/address2UPRN/main.tf | 40 +++---------------- 2 files changed, 8 insertions(+), 35 deletions(-) diff --git a/infrastructure/terraform/lambda/_template/README.md b/infrastructure/terraform/lambda/_template/README.md index 2ae9f9af..0a2f07af 100644 --- a/infrastructure/terraform/lambda/_template/README.md +++ b/infrastructure/terraform/lambda/_template/README.md @@ -5,3 +5,6 @@ - [ ] Add ECR repo in shared/main.tf - [ ] Add shared output for repo name/url - [ ] Push to GitHub (CI will deploy) + + +Note: By default this does a sqs to lamba. Configure the terraform file for other connections between sqs->lambda \ No newline at end of file diff --git a/infrastructure/terraform/lambda/address2UPRN/main.tf b/infrastructure/terraform/lambda/address2UPRN/main.tf index a0c4d3af..a5978186 100644 --- a/infrastructure/terraform/lambda/address2UPRN/main.tf +++ b/infrastructure/terraform/lambda/address2UPRN/main.tf @@ -12,48 +12,18 @@ data "terraform_remote_state" "shared" { } ############################################ -# IAM role +# Address2UPRN Lambda (via reusable module) ############################################ -module "role" { - source = "../../modules/lambda_execution_role" - name = "address2uprn-lambda-${var.stage}" -} +module "address2uprn" { + source = "../modules/lambda_with_sqs" -############################################ -# SQS queue -############################################ -module "queue" { - source = "../../modules/sqs_queue" - name = "address2uprn-queue-${var.stage}" -} - -############################################ -# Lambda (image-based) -############################################ -module "lambda" { - source = "../../modules/lambda_service" - - name = "address2uprn-${var.stage}" - role_arn = module.role.role_arn + name = "address2uprn" + stage = var.stage image_uri = "${data.terraform_remote_state.shared.outputs.address2uprn_repository_url}@${var.image_digest}" - timeout = 60 - memory_size = 1024 - environment = { STAGE = var.stage LOG_LEVEL = "info" } } - -############################################ -# SQS → Lambda trigger -############################################ -module "sqs_trigger" { - source = "../../modules/lambda_sqs_trigger" - - lambda_arn = module.lambda.lambda_arn - lambda_role_name = module.role.role_name - queue_arn = module.queue.queue_arn -} From 19872bf451e0c2a616927160a0580f7c58f415ff Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Tue, 3 Feb 2026 12:45:34 +0000 Subject: [PATCH 51/86] use the template --- infrastructure/terraform/lambda/address2UPRN/provider.tf | 2 +- infrastructure/terraform/lambda/address2UPRN/variables.tf | 8 ++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/infrastructure/terraform/lambda/address2UPRN/provider.tf b/infrastructure/terraform/lambda/address2UPRN/provider.tf index 2f4360ec..36cb63f1 100644 --- a/infrastructure/terraform/lambda/address2UPRN/provider.tf +++ b/infrastructure/terraform/lambda/address2UPRN/provider.tf @@ -7,7 +7,7 @@ terraform { } backend "s3" { - bucket = "address2uprn-terraform-state" + bucket = var.state_bucket_name key = "terraform.tfstate" region = "eu-west-2" } diff --git a/infrastructure/terraform/lambda/address2UPRN/variables.tf b/infrastructure/terraform/lambda/address2UPRN/variables.tf index 1d6c5952..b6f9907a 100644 --- a/infrastructure/terraform/lambda/address2UPRN/variables.tf +++ b/infrastructure/terraform/lambda/address2UPRN/variables.tf @@ -8,6 +8,10 @@ variable "stage" { } variable "image_digest" { - type = string - description = "sha256 image digest from CI" + type = string } + +variable "state_bucket_name" { + type = string + description = "S3 bucket name used for this lambda's Terraform state" +} \ No newline at end of file From f6255c105bcc25beebdad8ca99cfab0950bd4b2f Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Tue, 3 Feb 2026 13:36:55 +0000 Subject: [PATCH 52/86] check plan --- .github/workflows/deploy_terraform.yml | 58 +++++++++---------- .../terraform/lambda/_template/README.md | 2 +- .../terraform/lambda/_template/main.tf | 2 +- .../terraform/lambda/_template/provider.tf | 2 +- .../terraform/lambda/_template/variables.tf | 5 -- .../terraform/lambda/address2UPRN/provider.tf | 2 +- .../lambda/address2UPRN/variables.tf | 5 -- infrastructure/terraform/shared/main.tf | 2 +- 8 files changed, 34 insertions(+), 44 deletions(-) diff --git a/.github/workflows/deploy_terraform.yml b/.github/workflows/deploy_terraform.yml index 6864570d..52187f1e 100644 --- a/.github/workflows/deploy_terraform.yml +++ b/.github/workflows/deploy_terraform.yml @@ -68,33 +68,33 @@ jobs: # working-directory: infrastructure/terraform/shared # run: terraform apply -auto-approve -var-file=${STAGE}.tfvars tfplan - # ============================================================ - # 2️⃣ Build Docker image (tag = GitHub SHA, digest resolved) - # ============================================================ - address2uprn_image: - needs: determine_stage - uses: ./.github/workflows/_build_image.yml - with: - ecr_repo: address2uprn-${{ needs.determine_stage.outputs.stage }} - dockerfile_path: backend/address2UPRN/Dockerfile - build_context: backend/address2UPRN - secrets: - AWS_ACCESS_KEY_ID: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }} - AWS_REGION: ${{ secrets.DEV_AWS_REGION }} + # # ============================================================ + # # 2️⃣ Build Docker image (tag = GitHub SHA, digest resolved) + # # ============================================================ + # address2uprn_image: + # needs: determine_stage + # uses: ./.github/workflows/_build_image.yml + # with: + # ecr_repo: address2uprn-${{ needs.determine_stage.outputs.stage }} + # dockerfile_path: backend/address2UPRN/Dockerfile + # build_context: backend/address2UPRN + # secrets: + # AWS_ACCESS_KEY_ID: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }} + # AWS_SECRET_ACCESS_KEY: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }} + # AWS_REGION: ${{ secrets.DEV_AWS_REGION }} - # # ============================================================ - # # 3️⃣ Deploy Lambda (Terraform, immutable digest) - # # ============================================================ - deploy_address2uprn_lambda: - needs: [address2uprn_image, determine_stage] - uses: ./.github/workflows/_deploy_lambda.yml - with: - lambda_name: address2UPRN - lambda_path: infrastructure/terraform/lambda/address2UPRN - stage: ${{ needs.determine_stage.outputs.stage }} - image_digest: ${{ needs.image.outputs.image_digest }} - secrets: - AWS_ACCESS_KEY_ID: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }} - AWS_REGION: ${{ secrets.DEV_AWS_REGION }} + # # # ============================================================ + # # # 3️⃣ Deploy Lambda (Terraform, immutable digest) + # # # ============================================================ + # deploy_address2uprn_lambda: + # needs: [address2uprn_image, determine_stage] + # uses: ./.github/workflows/_deploy_lambda.yml + # with: + # lambda_name: address2UPRN + # lambda_path: infrastructure/terraform/lambda/address2UPRN + # stage: ${{ needs.determine_stage.outputs.stage }} + # image_digest: ${{ needs.image.outputs.image_digest }} + # secrets: + # AWS_ACCESS_KEY_ID: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }} + # AWS_SECRET_ACCESS_KEY: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }} + # AWS_REGION: ${{ secrets.DEV_AWS_REGION }} diff --git a/infrastructure/terraform/lambda/_template/README.md b/infrastructure/terraform/lambda/_template/README.md index 0a2f07af..1f519a90 100644 --- a/infrastructure/terraform/lambda/_template/README.md +++ b/infrastructure/terraform/lambda/_template/README.md @@ -1,8 +1,8 @@ ### Checklist for a new lambda - [ ] Copy cp -r lambda/_template lambda/ -- [ ] Set `state_bucket_name` - [ ] Add ECR repo in shared/main.tf +- [ ] Set bucket name in provider.tf - [ ] Add shared output for repo name/url - [ ] Push to GitHub (CI will deploy) diff --git a/infrastructure/terraform/lambda/_template/main.tf b/infrastructure/terraform/lambda/_template/main.tf index 77476cf4..0b3f008a 100644 --- a/infrastructure/terraform/lambda/_template/main.tf +++ b/infrastructure/terraform/lambda/_template/main.tf @@ -10,7 +10,7 @@ data "terraform_remote_state" "shared" { module "lambda" { source = "../modules/lambda_with_sqs" - name = "REPLACE_ME" + name = REPLACE ME #"address2uprn" for example stage = var.stage image_uri = "${data.terraform_remote_state.shared.outputs.REPLACE_ME_repository_url}@${var.image_digest}" diff --git a/infrastructure/terraform/lambda/_template/provider.tf b/infrastructure/terraform/lambda/_template/provider.tf index 36cb63f1..244935de 100644 --- a/infrastructure/terraform/lambda/_template/provider.tf +++ b/infrastructure/terraform/lambda/_template/provider.tf @@ -7,7 +7,7 @@ terraform { } backend "s3" { - bucket = var.state_bucket_name + bucket = REPLACE_ME key = "terraform.tfstate" region = "eu-west-2" } diff --git a/infrastructure/terraform/lambda/_template/variables.tf b/infrastructure/terraform/lambda/_template/variables.tf index b6f9907a..42ac1047 100644 --- a/infrastructure/terraform/lambda/_template/variables.tf +++ b/infrastructure/terraform/lambda/_template/variables.tf @@ -9,9 +9,4 @@ variable "stage" { variable "image_digest" { type = string -} - -variable "state_bucket_name" { - type = string - description = "S3 bucket name used for this lambda's Terraform state" } \ No newline at end of file diff --git a/infrastructure/terraform/lambda/address2UPRN/provider.tf b/infrastructure/terraform/lambda/address2UPRN/provider.tf index 36cb63f1..2f4360ec 100644 --- a/infrastructure/terraform/lambda/address2UPRN/provider.tf +++ b/infrastructure/terraform/lambda/address2UPRN/provider.tf @@ -7,7 +7,7 @@ terraform { } backend "s3" { - bucket = var.state_bucket_name + bucket = "address2uprn-terraform-state" key = "terraform.tfstate" region = "eu-west-2" } diff --git a/infrastructure/terraform/lambda/address2UPRN/variables.tf b/infrastructure/terraform/lambda/address2UPRN/variables.tf index b6f9907a..208b82b5 100644 --- a/infrastructure/terraform/lambda/address2UPRN/variables.tf +++ b/infrastructure/terraform/lambda/address2UPRN/variables.tf @@ -10,8 +10,3 @@ variable "stage" { variable "image_digest" { type = string } - -variable "state_bucket_name" { - type = string - description = "S3 bucket name used for this lambda's Terraform state" -} \ No newline at end of file diff --git a/infrastructure/terraform/shared/main.tf b/infrastructure/terraform/shared/main.tf index 424204a3..3ba78ef3 100644 --- a/infrastructure/terraform/shared/main.tf +++ b/infrastructure/terraform/shared/main.tf @@ -294,7 +294,7 @@ output "ses_dns_records" { ################################################ module "address2uprn_state_bucket" { source = "../modules/tf_state_bucket" - bucket_name = "address2uprn-terraform-state-${var.stage}" + bucket_name = "address2uprn-terraform-state" } From 3221899299f6b70c0def73788f9bdb3fda725604 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Tue, 3 Feb 2026 13:38:37 +0000 Subject: [PATCH 53/86] apply --- .github/workflows/deploy_terraform.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/deploy_terraform.yml b/.github/workflows/deploy_terraform.yml index 52187f1e..c35e3744 100644 --- a/.github/workflows/deploy_terraform.yml +++ b/.github/workflows/deploy_terraform.yml @@ -64,9 +64,10 @@ jobs: working-directory: infrastructure/terraform/shared run: terraform plan -var-file=${STAGE}.tfvars -out=tfplan - # - name: Terraform Apply (shared) - # working-directory: infrastructure/terraform/shared - # run: terraform apply -auto-approve -var-file=${STAGE}.tfvars tfplan + - name: Terraform Apply (shared) + # if: env.STAGE == 'prod' + working-directory: infrastructure/terraform/shared + run: terraform apply -auto-approve -var-file=${STAGE}.tfvars tfplan # # ============================================================ # # 2️⃣ Build Docker image (tag = GitHub SHA, digest resolved) From 9b631c1b8bbd1e5e7a041249e00816207a41b1fc Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Tue, 3 Feb 2026 13:40:50 +0000 Subject: [PATCH 54/86] address 2 uprn lambda deployment --- .github/workflows/deploy_terraform.yml | 60 +++++++++++++------------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/.github/workflows/deploy_terraform.yml b/.github/workflows/deploy_terraform.yml index c35e3744..d92bb696 100644 --- a/.github/workflows/deploy_terraform.yml +++ b/.github/workflows/deploy_terraform.yml @@ -65,37 +65,37 @@ jobs: run: terraform plan -var-file=${STAGE}.tfvars -out=tfplan - name: Terraform Apply (shared) - # if: env.STAGE == 'prod' + if: env.STAGE == 'prod' working-directory: infrastructure/terraform/shared run: terraform apply -auto-approve -var-file=${STAGE}.tfvars tfplan - # # ============================================================ - # # 2️⃣ Build Docker image (tag = GitHub SHA, digest resolved) - # # ============================================================ - # address2uprn_image: - # needs: determine_stage - # uses: ./.github/workflows/_build_image.yml - # with: - # ecr_repo: address2uprn-${{ needs.determine_stage.outputs.stage }} - # dockerfile_path: backend/address2UPRN/Dockerfile - # build_context: backend/address2UPRN - # secrets: - # AWS_ACCESS_KEY_ID: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }} - # AWS_SECRET_ACCESS_KEY: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }} - # AWS_REGION: ${{ secrets.DEV_AWS_REGION }} + # ============================================================ + # 2️⃣ Build Docker image (tag = GitHub SHA, digest resolved) + # ============================================================ + address2uprn_image: + needs: determine_stage + uses: ./.github/workflows/_build_image.yml + with: + ecr_repo: address2uprn-${{ needs.determine_stage.outputs.stage }} + dockerfile_path: backend/address2UPRN/Dockerfile + build_context: backend/address2UPRN + secrets: + AWS_ACCESS_KEY_ID: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }} + AWS_REGION: ${{ secrets.DEV_AWS_REGION }} - # # # ============================================================ - # # # 3️⃣ Deploy Lambda (Terraform, immutable digest) - # # # ============================================================ - # deploy_address2uprn_lambda: - # needs: [address2uprn_image, determine_stage] - # uses: ./.github/workflows/_deploy_lambda.yml - # with: - # lambda_name: address2UPRN - # lambda_path: infrastructure/terraform/lambda/address2UPRN - # stage: ${{ needs.determine_stage.outputs.stage }} - # image_digest: ${{ needs.image.outputs.image_digest }} - # secrets: - # AWS_ACCESS_KEY_ID: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }} - # AWS_SECRET_ACCESS_KEY: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }} - # AWS_REGION: ${{ secrets.DEV_AWS_REGION }} + # # ============================================================ + # # 3️⃣ Deploy Lambda (Terraform, immutable digest) + # # ============================================================ + deploy_address2uprn_lambda: + needs: [address2uprn_image, determine_stage] + uses: ./.github/workflows/_deploy_lambda.yml + with: + lambda_name: address2UPRN + lambda_path: infrastructure/terraform/lambda/address2UPRN + stage: ${{ needs.determine_stage.outputs.stage }} + image_digest: ${{ needs.image.outputs.image_digest }} + secrets: + AWS_ACCESS_KEY_ID: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }} + AWS_REGION: ${{ secrets.DEV_AWS_REGION }} From 7a7de42a0b4964ef3b3eabb889db797bce8011c9 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Tue, 3 Feb 2026 13:45:10 +0000 Subject: [PATCH 55/86] forgot to add variables --- .github/workflows/deploy_terraform.yml | 2 +- .../terraform/modules/lambda_execution_role/variables.tf | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/deploy_terraform.yml b/.github/workflows/deploy_terraform.yml index d92bb696..662c4b98 100644 --- a/.github/workflows/deploy_terraform.yml +++ b/.github/workflows/deploy_terraform.yml @@ -87,7 +87,7 @@ jobs: # # ============================================================ # # 3️⃣ Deploy Lambda (Terraform, immutable digest) # # ============================================================ - deploy_address2uprn_lambda: + address2uprn_lambda: needs: [address2uprn_image, determine_stage] uses: ./.github/workflows/_deploy_lambda.yml with: diff --git a/infrastructure/terraform/modules/lambda_execution_role/variables.tf b/infrastructure/terraform/modules/lambda_execution_role/variables.tf index e69de29b..f9f512ff 100644 --- a/infrastructure/terraform/modules/lambda_execution_role/variables.tf +++ b/infrastructure/terraform/modules/lambda_execution_role/variables.tf @@ -0,0 +1,4 @@ +variable "name" { + description = "IAM role name for the Lambda execution role" + type = string +} From f8f71079bf54c1518977664b24c2a5f99f60ffe0 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Tue, 3 Feb 2026 13:52:13 +0000 Subject: [PATCH 56/86] add workspaces --- .github/workflows/deploy_terraform.yml | 2 +- infrastructure/terraform/lambda/_template/main.tf | 1 + infrastructure/terraform/lambda/address2UPRN/main.tf | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/deploy_terraform.yml b/.github/workflows/deploy_terraform.yml index 662c4b98..b7743acd 100644 --- a/.github/workflows/deploy_terraform.yml +++ b/.github/workflows/deploy_terraform.yml @@ -73,7 +73,7 @@ jobs: # 2️⃣ Build Docker image (tag = GitHub SHA, digest resolved) # ============================================================ address2uprn_image: - needs: determine_stage + needs: [determine_stage, shared_terraform] uses: ./.github/workflows/_build_image.yml with: ecr_repo: address2uprn-${{ needs.determine_stage.outputs.stage }} diff --git a/infrastructure/terraform/lambda/_template/main.tf b/infrastructure/terraform/lambda/_template/main.tf index 0b3f008a..46c54207 100644 --- a/infrastructure/terraform/lambda/_template/main.tf +++ b/infrastructure/terraform/lambda/_template/main.tf @@ -4,6 +4,7 @@ data "terraform_remote_state" "shared" { bucket = "assessment-model-terraform-state" key = "terraform.tfstate" region = "eu-west-2" + workspace = var.stage } } diff --git a/infrastructure/terraform/lambda/address2UPRN/main.tf b/infrastructure/terraform/lambda/address2UPRN/main.tf index a5978186..b3d41925 100644 --- a/infrastructure/terraform/lambda/address2UPRN/main.tf +++ b/infrastructure/terraform/lambda/address2UPRN/main.tf @@ -8,6 +8,7 @@ data "terraform_remote_state" "shared" { bucket = "assessment-model-terraform-state" key = "terraform.tfstate" region = "eu-west-2" + workspace = var.stage } } From c52cd37a74c1da662858266f8de526ee40e5ec41 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Tue, 3 Feb 2026 13:56:25 +0000 Subject: [PATCH 57/86] remove workspaces --- infrastructure/terraform/lambda/_template/main.tf | 1 - infrastructure/terraform/lambda/address2UPRN/main.tf | 1 - 2 files changed, 2 deletions(-) diff --git a/infrastructure/terraform/lambda/_template/main.tf b/infrastructure/terraform/lambda/_template/main.tf index 46c54207..0b3f008a 100644 --- a/infrastructure/terraform/lambda/_template/main.tf +++ b/infrastructure/terraform/lambda/_template/main.tf @@ -4,7 +4,6 @@ data "terraform_remote_state" "shared" { bucket = "assessment-model-terraform-state" key = "terraform.tfstate" region = "eu-west-2" - workspace = var.stage } } diff --git a/infrastructure/terraform/lambda/address2UPRN/main.tf b/infrastructure/terraform/lambda/address2UPRN/main.tf index b3d41925..a5978186 100644 --- a/infrastructure/terraform/lambda/address2UPRN/main.tf +++ b/infrastructure/terraform/lambda/address2UPRN/main.tf @@ -8,7 +8,6 @@ data "terraform_remote_state" "shared" { bucket = "assessment-model-terraform-state" key = "terraform.tfstate" region = "eu-west-2" - workspace = var.stage } } From 5db4d2cfbed4adbe81847ff08cdc802f2a69c2f5 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Tue, 3 Feb 2026 14:23:20 +0000 Subject: [PATCH 58/86] deploy --- .github/workflows/_build_image.yml | 27 ++++--- .github/workflows/_deploy_lambda.yml | 20 +++--- .github/workflows/deploy_terraform.yml | 38 +++++----- .../terraform/lambda/_template/README.md | 72 +++++++++++++++++-- .../terraform/lambda/_template/main.tf | 12 +--- .../terraform/lambda/_template/variables.tf | 15 ++-- .../terraform/lambda/address2UPRN/main.tf | 18 +---- .../lambda/address2UPRN/variables.tf | 15 ++-- 8 files changed, 124 insertions(+), 93 deletions(-) diff --git a/.github/workflows/_build_image.yml b/.github/workflows/_build_image.yml index 1008d592..9414b959 100644 --- a/.github/workflows/_build_image.yml +++ b/.github/workflows/_build_image.yml @@ -21,16 +21,20 @@ on: outputs: image_digest: - description: "Pushed image digest" + description: "Pushed image digest (sha256:...)" value: ${{ jobs.build.outputs.image_digest }} + ecr_repo_url: + description: "ECR repository URL (no tag, no digest)" + value: ${{ jobs.build.outputs.ecr_repo_url }} + secrets: AWS_ACCESS_KEY_ID: required: true AWS_SECRET_ACCESS_KEY: required: true AWS_REGION: - required: true + required: true jobs: build: @@ -38,6 +42,7 @@ jobs: outputs: image_digest: ${{ steps.digest.outputs.image_digest }} + ecr_repo_url: ${{ steps.repo.outputs.ecr_repo_url }} steps: - uses: actions/checkout@v4 @@ -49,16 +54,20 @@ jobs: aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} aws-region: ${{ secrets.AWS_REGION }} - - uses: aws-actions/amazon-ecr-login@v2 + - name: Login to ECR + uses: aws-actions/amazon-ecr-login@v2 + + - name: Resolve ECR repo URL + id: repo + run: | + AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text) + ECR_REPO_URL="${AWS_ACCOUNT_ID}.dkr.ecr.${{ secrets.AWS_REGION }}.amazonaws.com/${{ inputs.ecr_repo }}" + echo "ecr_repo_url=$ECR_REPO_URL" >> "$GITHUB_OUTPUT" - name: Build & push image run: | IMAGE_TAG=${GITHUB_SHA} - AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text) - - IMAGE_URI=${AWS_ACCOUNT_ID}.dkr.ecr.${{ secrets.AWS_REGION }}.amazonaws.com/${{ inputs.ecr_repo }}:${IMAGE_TAG} - - echo "Using IMAGE_URI=$IMAGE_URI" + IMAGE_URI="${{ steps.repo.outputs.ecr_repo_url }}:${IMAGE_TAG}" docker build \ -f ${{ inputs.dockerfile_path }} \ @@ -76,4 +85,4 @@ jobs: --query 'imageDetails[0].imageDigest' \ --output text) - echo "image_digest=$DIGEST" >> $GITHUB_OUTPUT \ No newline at end of file + echo "image_digest=$DIGEST" >> "$GITHUB_OUTPUT" diff --git a/.github/workflows/_deploy_lambda.yml b/.github/workflows/_deploy_lambda.yml index 5abbbd99..5887da65 100644 --- a/.github/workflows/_deploy_lambda.yml +++ b/.github/workflows/_deploy_lambda.yml @@ -6,13 +6,17 @@ on: lambda_name: required: true type: string + lambda_path: required: true type: string + stage: required: true type: string - image_digest: + + image_uri: + description: "Full ECR image URI including digest" required: true type: string @@ -22,8 +26,7 @@ on: AWS_SECRET_ACCESS_KEY: required: true AWS_REGION: - required: true - + required: true jobs: deploy: @@ -55,14 +58,11 @@ jobs: run: | terraform plan \ -var="stage=${{ inputs.stage }}" \ - -var="image_digest=${{ inputs.image_digest }}" \ + -var="lambda_name=${{ inputs.lambda_name }}" \ + -var="image_uri=${{ inputs.image_uri }}" \ -out=lambdaplan + # Uncomment when ready # - name: Terraform Apply # working-directory: ${{ inputs.lambda_path }} - # run: | - # terraform apply \ - # -auto-approve \ - # -var="stage=${{ inputs.stage }}" \ - # -var="image_digest=${{ inputs.image_digest }}" \ - # lambdaplan + # run: terraform apply -auto-approve lambdaplan diff --git a/.github/workflows/deploy_terraform.yml b/.github/workflows/deploy_terraform.yml index b7743acd..339e742d 100644 --- a/.github/workflows/deploy_terraform.yml +++ b/.github/workflows/deploy_terraform.yml @@ -5,7 +5,6 @@ on: branches: - "**" - jobs: determine_stage: runs-on: ubuntu-latest @@ -27,9 +26,8 @@ jobs: echo "stage=dev" >> "$GITHUB_OUTPUT" fi - echo "Resolved STAGE=$BRANCH → $(cat $GITHUB_OUTPUT)" # ============================================================ - # 1️⃣ Shared Terraform (plan only for now) + # 1️⃣ Shared Terraform (infra) # ============================================================ shared_terraform: needs: determine_stage @@ -38,39 +36,35 @@ jobs: STAGE: ${{ needs.determine_stage.outputs.stage }} steps: - - name: Checkout - uses: actions/checkout@v4 + - uses: actions/checkout@v4 - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v4 + - uses: aws-actions/configure-aws-credentials@v4 with: - # This will need to be changed to env imports when we have different env to dynamically allocate prod, staging etc aws-access-key-id: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }} aws-region: ${{ secrets.DEV_AWS_REGION }} - - name: Setup Terraform - uses: hashicorp/setup-terraform@v3 + - uses: hashicorp/setup-terraform@v3 - - name: Terraform Init (shared) + - name: Terraform Init working-directory: infrastructure/terraform/shared run: terraform init -reconfigure - - name: Terraform Workspace (shared) + - name: Terraform Workspace working-directory: infrastructure/terraform/shared run: terraform workspace select ${STAGE} || terraform workspace new ${STAGE} - - name: Terraform Plan (shared) + - name: Terraform Plan working-directory: infrastructure/terraform/shared run: terraform plan -var-file=${STAGE}.tfvars -out=tfplan - - name: Terraform Apply (shared) + - name: Terraform Apply if: env.STAGE == 'prod' working-directory: infrastructure/terraform/shared - run: terraform apply -auto-approve -var-file=${STAGE}.tfvars tfplan + run: terraform apply -auto-approve tfplan # ============================================================ - # 2️⃣ Build Docker image (tag = GitHub SHA, digest resolved) + # 2️⃣ Build image # ============================================================ address2uprn_image: needs: [determine_stage, shared_terraform] @@ -84,17 +78,17 @@ jobs: AWS_SECRET_ACCESS_KEY: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }} AWS_REGION: ${{ secrets.DEV_AWS_REGION }} - # # ============================================================ - # # 3️⃣ Deploy Lambda (Terraform, immutable digest) - # # ============================================================ + # ============================================================ + # 3️⃣ Deploy Lambda + # ============================================================ address2uprn_lambda: needs: [address2uprn_image, determine_stage] uses: ./.github/workflows/_deploy_lambda.yml with: - lambda_name: address2UPRN - lambda_path: infrastructure/terraform/lambda/address2UPRN + lambda_name: address2uprn + lambda_path: infrastructure/terraform/lambda/address2uprn stage: ${{ needs.determine_stage.outputs.stage }} - image_digest: ${{ needs.image.outputs.image_digest }} + image_uri: ${{ needs.address2uprn_image.outputs.ecr_repo_url }}@${{ needs.address2uprn_image.outputs.image_digest }} secrets: AWS_ACCESS_KEY_ID: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }} diff --git a/infrastructure/terraform/lambda/_template/README.md b/infrastructure/terraform/lambda/_template/README.md index 1f519a90..0dd612ee 100644 --- a/infrastructure/terraform/lambda/_template/README.md +++ b/infrastructure/terraform/lambda/_template/README.md @@ -1,10 +1,68 @@ -### Checklist for a new lambda +## Checklist for adding a new Lambda -- [ ] Copy cp -r lambda/_template lambda/ -- [ ] Add ECR repo in shared/main.tf -- [ ] Set bucket name in provider.tf -- [ ] Add shared output for repo name/url -- [ ] Push to GitHub (CI will deploy) +### 1. Create the Lambda scaffold +- Copy the template: + cp -r lambda/_template lambda/ -Note: By default this does a sqs to lamba. Configure the terraform file for other connections between sqs->lambda \ No newline at end of file +- Update `variables.tf` if required +- Ensure the Terraform module accepts: + - `lambda_name` + - `stage` + - `image_uri` + +--- + +### 2. Add infrastructure prerequisites (shared stack) +- Add a new ECR repository in: + + infrastructure/terraform/shared/main.tf + +- Apply the shared stack (or let CI apply it on `prod`) +- Verify the ECR repository exists in AWS + +Note: +No Terraform outputs are required for the ECR repo. +The CI pipeline resolves the repository URL dynamically at build time. + +--- + +### 3. Add Docker build configuration +- Create a `Dockerfile` for the Lambda +- Verify the Dockerfile path and build context +- Add a new image build job in `deploy_terraform.yml` using `_build_image.yml` + +--- + +### 4. Wire the Lambda deploy job (CI) +- Add a deploy job using `_deploy_lambda.yml` +- Pass the following inputs: + - `lambda_name` + - `lambda_path` + - `stage` + - `image_uri` (constructed as `repo@sha256:digest`) +- Ensure the deploy job depends on the image build job + +--- + +### 5. Deploy +- Push changes to GitHub +- CI will: + 1. Build and push the Docker image + 2. Resolve the image digest + 3. Deploy the Lambda using the immutable `image_uri` + +--- + +## Notes +- Terraform remote state is not used for image wiring +- Image tags are not used; deployments are digest-based +- By default, the template wires SQS → Lambda + To change this, update the Terraform in `lambda/` + (e.g. EventBridge, API Gateway, direct invoke) + +--- + +## Rule of thumb +CI decides what image to deploy. +Terraform only deploys it. diff --git a/infrastructure/terraform/lambda/_template/main.tf b/infrastructure/terraform/lambda/_template/main.tf index 0b3f008a..a0e8eb80 100644 --- a/infrastructure/terraform/lambda/_template/main.tf +++ b/infrastructure/terraform/lambda/_template/main.tf @@ -1,19 +1,11 @@ -data "terraform_remote_state" "shared" { - backend = "s3" - config = { - bucket = "assessment-model-terraform-state" - key = "terraform.tfstate" - region = "eu-west-2" - } -} - module "lambda" { source = "../modules/lambda_with_sqs" name = REPLACE ME #"address2uprn" for example stage = var.stage - image_uri = "${data.terraform_remote_state.shared.outputs.REPLACE_ME_repository_url}@${var.image_digest}" + image_uri = var.image_uri + environment = { STAGE = var.stage diff --git a/infrastructure/terraform/lambda/_template/variables.tf b/infrastructure/terraform/lambda/_template/variables.tf index 42ac1047..7ba0bedb 100644 --- a/infrastructure/terraform/lambda/_template/variables.tf +++ b/infrastructure/terraform/lambda/_template/variables.tf @@ -1,12 +1,9 @@ -variable "region" { - type = string - default = "eu-west-2" +variable "lambda_name" { + type = string + description = "Logical name of the lambda (e.g. address2uprn)" } -variable "stage" { - type = string +variable "image_uri" { + type = string + description = "Full ECR image URI including digest" } - -variable "image_digest" { - type = string -} \ No newline at end of file diff --git a/infrastructure/terraform/lambda/address2UPRN/main.tf b/infrastructure/terraform/lambda/address2UPRN/main.tf index a5978186..8d8e489f 100644 --- a/infrastructure/terraform/lambda/address2UPRN/main.tf +++ b/infrastructure/terraform/lambda/address2UPRN/main.tf @@ -1,26 +1,10 @@ -############################################ -# Read shared state to get outputs -############################################ -data "terraform_remote_state" "shared" { - backend = "s3" - - config = { - bucket = "assessment-model-terraform-state" - key = "terraform.tfstate" - region = "eu-west-2" - } -} - -############################################ -# Address2UPRN Lambda (via reusable module) -############################################ module "address2uprn" { source = "../modules/lambda_with_sqs" name = "address2uprn" stage = var.stage - image_uri = "${data.terraform_remote_state.shared.outputs.address2uprn_repository_url}@${var.image_digest}" + image_uri = var.image_uri environment = { STAGE = var.stage diff --git a/infrastructure/terraform/lambda/address2UPRN/variables.tf b/infrastructure/terraform/lambda/address2UPRN/variables.tf index 208b82b5..7ba0bedb 100644 --- a/infrastructure/terraform/lambda/address2UPRN/variables.tf +++ b/infrastructure/terraform/lambda/address2UPRN/variables.tf @@ -1,12 +1,9 @@ -variable "region" { - type = string - default = "eu-west-2" +variable "lambda_name" { + type = string + description = "Logical name of the lambda (e.g. address2uprn)" } -variable "stage" { - type = string -} - -variable "image_digest" { - type = string +variable "image_uri" { + type = string + description = "Full ECR image URI including digest" } From 323badc5fc4d092f2742c92abe4caa2421136744 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Tue, 3 Feb 2026 14:26:47 +0000 Subject: [PATCH 59/86] file name --- .github/workflows/deploy_terraform.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy_terraform.yml b/.github/workflows/deploy_terraform.yml index 339e742d..1aab25cf 100644 --- a/.github/workflows/deploy_terraform.yml +++ b/.github/workflows/deploy_terraform.yml @@ -86,7 +86,7 @@ jobs: uses: ./.github/workflows/_deploy_lambda.yml with: lambda_name: address2uprn - lambda_path: infrastructure/terraform/lambda/address2uprn + lambda_path: infrastructure/terraform/lambda/address2UPRN stage: ${{ needs.determine_stage.outputs.stage }} image_uri: ${{ needs.address2uprn_image.outputs.ecr_repo_url }}@${{ needs.address2uprn_image.outputs.image_digest }} secrets: From eeee5fa03e5420b0f5ca7daa959e64ea04d5a351 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Tue, 3 Feb 2026 14:31:16 +0000 Subject: [PATCH 60/86] add stage --- infrastructure/terraform/lambda/_template/variables.tf | 5 +++++ infrastructure/terraform/lambda/address2UPRN/variables.tf | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/infrastructure/terraform/lambda/_template/variables.tf b/infrastructure/terraform/lambda/_template/variables.tf index 7ba0bedb..5876cf3a 100644 --- a/infrastructure/terraform/lambda/_template/variables.tf +++ b/infrastructure/terraform/lambda/_template/variables.tf @@ -7,3 +7,8 @@ variable "image_uri" { type = string description = "Full ECR image URI including digest" } + +variable "stage" { + description = "Deployment stage (e.g. dev, prod)" + type = string +} \ No newline at end of file diff --git a/infrastructure/terraform/lambda/address2UPRN/variables.tf b/infrastructure/terraform/lambda/address2UPRN/variables.tf index 7ba0bedb..5876cf3a 100644 --- a/infrastructure/terraform/lambda/address2UPRN/variables.tf +++ b/infrastructure/terraform/lambda/address2UPRN/variables.tf @@ -7,3 +7,8 @@ variable "image_uri" { type = string description = "Full ECR image URI including digest" } + +variable "stage" { + description = "Deployment stage (e.g. dev, prod)" + type = string +} \ No newline at end of file From 0b22adfb652cb2e2893496dc33766f529ed00df8 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Tue, 3 Feb 2026 14:39:42 +0000 Subject: [PATCH 61/86] add region --- infrastructure/terraform/lambda/_template/variables.tf | 5 +++++ infrastructure/terraform/lambda/address2UPRN/variables.tf | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/infrastructure/terraform/lambda/_template/variables.tf b/infrastructure/terraform/lambda/_template/variables.tf index 5876cf3a..a82a0859 100644 --- a/infrastructure/terraform/lambda/_template/variables.tf +++ b/infrastructure/terraform/lambda/_template/variables.tf @@ -11,4 +11,9 @@ variable "image_uri" { variable "stage" { description = "Deployment stage (e.g. dev, prod)" type = string +} + +variable "region" { + description = "Deployment stage (e.g. dev, prod)" + type = string } \ No newline at end of file diff --git a/infrastructure/terraform/lambda/address2UPRN/variables.tf b/infrastructure/terraform/lambda/address2UPRN/variables.tf index 5876cf3a..a82a0859 100644 --- a/infrastructure/terraform/lambda/address2UPRN/variables.tf +++ b/infrastructure/terraform/lambda/address2UPRN/variables.tf @@ -11,4 +11,9 @@ variable "image_uri" { variable "stage" { description = "Deployment stage (e.g. dev, prod)" type = string +} + +variable "region" { + description = "Deployment stage (e.g. dev, prod)" + type = string } \ No newline at end of file From 7cbb5526b143809365318ccfe26ec3873e02c885 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Tue, 3 Feb 2026 14:40:28 +0000 Subject: [PATCH 62/86] region remove --- infrastructure/terraform/lambda/_template/variables.tf | 5 ----- 1 file changed, 5 deletions(-) diff --git a/infrastructure/terraform/lambda/_template/variables.tf b/infrastructure/terraform/lambda/_template/variables.tf index a82a0859..41a99950 100644 --- a/infrastructure/terraform/lambda/_template/variables.tf +++ b/infrastructure/terraform/lambda/_template/variables.tf @@ -12,8 +12,3 @@ variable "stage" { description = "Deployment stage (e.g. dev, prod)" type = string } - -variable "region" { - description = "Deployment stage (e.g. dev, prod)" - type = string -} \ No newline at end of file From 15e147e55e791df73e9f4c74db40fd6ee038bbb1 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Tue, 3 Feb 2026 14:46:40 +0000 Subject: [PATCH 63/86] region remove again --- infrastructure/terraform/lambda/address2UPRN/provider.tf | 3 --- infrastructure/terraform/lambda/address2UPRN/variables.tf | 5 ----- 2 files changed, 8 deletions(-) diff --git a/infrastructure/terraform/lambda/address2UPRN/provider.tf b/infrastructure/terraform/lambda/address2UPRN/provider.tf index 2f4360ec..ad873717 100644 --- a/infrastructure/terraform/lambda/address2UPRN/provider.tf +++ b/infrastructure/terraform/lambda/address2UPRN/provider.tf @@ -15,6 +15,3 @@ terraform { required_version = ">= 1.2.0" } -provider "aws" { - region = var.region -} diff --git a/infrastructure/terraform/lambda/address2UPRN/variables.tf b/infrastructure/terraform/lambda/address2UPRN/variables.tf index a82a0859..41a99950 100644 --- a/infrastructure/terraform/lambda/address2UPRN/variables.tf +++ b/infrastructure/terraform/lambda/address2UPRN/variables.tf @@ -12,8 +12,3 @@ variable "stage" { description = "Deployment stage (e.g. dev, prod)" type = string } - -variable "region" { - description = "Deployment stage (e.g. dev, prod)" - type = string -} \ No newline at end of file From ca5fe56c8eb162fc86e1c66ca4b0720d3f36248a Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Tue, 3 Feb 2026 14:46:53 +0000 Subject: [PATCH 64/86] forgot on e file --- infrastructure/terraform/lambda/_template/provider.tf | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/infrastructure/terraform/lambda/_template/provider.tf b/infrastructure/terraform/lambda/_template/provider.tf index 244935de..37c412ce 100644 --- a/infrastructure/terraform/lambda/_template/provider.tf +++ b/infrastructure/terraform/lambda/_template/provider.tf @@ -13,8 +13,4 @@ terraform { } required_version = ">= 1.2.0" -} - -provider "aws" { - region = var.region -} +} \ No newline at end of file From c8ca5a83780eeed5a81e7d49cffc437ee385d2f3 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Tue, 3 Feb 2026 14:51:36 +0000 Subject: [PATCH 65/86] reruun --- infrastructure/terraform/lambda/_template/variables.tf | 1 + 1 file changed, 1 insertion(+) diff --git a/infrastructure/terraform/lambda/_template/variables.tf b/infrastructure/terraform/lambda/_template/variables.tf index 41a99950..4c9fd47d 100644 --- a/infrastructure/terraform/lambda/_template/variables.tf +++ b/infrastructure/terraform/lambda/_template/variables.tf @@ -12,3 +12,4 @@ variable "stage" { description = "Deployment stage (e.g. dev, prod)" type = string } + From 03e7c941589e62b81d9717be53999af937212303 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Tue, 3 Feb 2026 15:01:01 +0000 Subject: [PATCH 66/86] apply and check aws --- .github/workflows/_deploy_lambda.yml | 7 +++---- .github/workflows/deploy_terraform.yml | 4 ++-- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/workflows/_deploy_lambda.yml b/.github/workflows/_deploy_lambda.yml index 5887da65..8161dbbf 100644 --- a/.github/workflows/_deploy_lambda.yml +++ b/.github/workflows/_deploy_lambda.yml @@ -62,7 +62,6 @@ jobs: -var="image_uri=${{ inputs.image_uri }}" \ -out=lambdaplan - # Uncomment when ready - # - name: Terraform Apply - # working-directory: ${{ inputs.lambda_path }} - # run: terraform apply -auto-approve lambdaplan + - name: Terraform Apply + working-directory: ${{ inputs.lambda_path }} + run: terraform apply -auto-approve lambdaplan diff --git a/.github/workflows/deploy_terraform.yml b/.github/workflows/deploy_terraform.yml index 1aab25cf..eede1048 100644 --- a/.github/workflows/deploy_terraform.yml +++ b/.github/workflows/deploy_terraform.yml @@ -64,7 +64,7 @@ jobs: run: terraform apply -auto-approve tfplan # ============================================================ - # 2️⃣ Build image + # 2️⃣ Build Address 2 UPRN image and Push # ============================================================ address2uprn_image: needs: [determine_stage, shared_terraform] @@ -79,7 +79,7 @@ jobs: AWS_REGION: ${{ secrets.DEV_AWS_REGION }} # ============================================================ - # 3️⃣ Deploy Lambda + # 3️⃣ Deploy Address 2 UPRN Lambda # ============================================================ address2uprn_lambda: needs: [address2uprn_image, determine_stage] From 1d8fbdc4a825eea4c6d3d9d0c669d140f6f9a620 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Tue, 3 Feb 2026 16:09:08 +0000 Subject: [PATCH 67/86] ecr_repo_url was empty for some reason --- .github/workflows/_deploy_lambda.yml | 10 +++++--- .github/workflows/deploy_terraform.yml | 5 +++- .../terraform/lambda/address2UPRN/main.tf | 3 ++- .../lambda/address2UPRN/variables.tf | 23 +++++++++++++++---- 4 files changed, 31 insertions(+), 10 deletions(-) diff --git a/.github/workflows/_deploy_lambda.yml b/.github/workflows/_deploy_lambda.yml index 8161dbbf..07bfe331 100644 --- a/.github/workflows/_deploy_lambda.yml +++ b/.github/workflows/_deploy_lambda.yml @@ -15,8 +15,11 @@ on: required: true type: string - image_uri: - description: "Full ECR image URI including digest" + ecr_repo_url: + required: true + type: string + + image_digest: required: true type: string @@ -59,7 +62,8 @@ jobs: terraform plan \ -var="stage=${{ inputs.stage }}" \ -var="lambda_name=${{ inputs.lambda_name }}" \ - -var="image_uri=${{ inputs.image_uri }}" \ + -var="ecr_repo_url=${{ inputs.ecr_repo_url }}" \ + -var="image_digest=${{ inputs.image_digest }}" \ -out=lambdaplan - name: Terraform Apply diff --git a/.github/workflows/deploy_terraform.yml b/.github/workflows/deploy_terraform.yml index eede1048..38f841e7 100644 --- a/.github/workflows/deploy_terraform.yml +++ b/.github/workflows/deploy_terraform.yml @@ -20,8 +20,10 @@ jobs: if [[ "$BRANCH" == "prod" ]]; then echo "stage=prod" >> "$GITHUB_OUTPUT" + elif [[ "$BRANCH" == "dev" ]]; then echo "stage=dev" >> "$GITHUB_OUTPUT" + else echo "stage=dev" >> "$GITHUB_OUTPUT" fi @@ -88,7 +90,8 @@ jobs: lambda_name: address2uprn lambda_path: infrastructure/terraform/lambda/address2UPRN stage: ${{ needs.determine_stage.outputs.stage }} - image_uri: ${{ needs.address2uprn_image.outputs.ecr_repo_url }}@${{ needs.address2uprn_image.outputs.image_digest }} + ecr_repo_url: ${{ needs.address2uprn_image.outputs.ecr_repo_url }} + image_digest: ${{ needs.address2uprn_image.outputs.image_digest }} secrets: AWS_ACCESS_KEY_ID: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }} diff --git a/infrastructure/terraform/lambda/address2UPRN/main.tf b/infrastructure/terraform/lambda/address2UPRN/main.tf index 8d8e489f..46b193f2 100644 --- a/infrastructure/terraform/lambda/address2UPRN/main.tf +++ b/infrastructure/terraform/lambda/address2UPRN/main.tf @@ -4,7 +4,8 @@ module "address2uprn" { name = "address2uprn" stage = var.stage - image_uri = var.image_uri + image_uri = local.image_uri + environment = { STAGE = var.stage diff --git a/infrastructure/terraform/lambda/address2UPRN/variables.tf b/infrastructure/terraform/lambda/address2UPRN/variables.tf index 41a99950..e4bab243 100644 --- a/infrastructure/terraform/lambda/address2UPRN/variables.tf +++ b/infrastructure/terraform/lambda/address2UPRN/variables.tf @@ -3,12 +3,25 @@ variable "lambda_name" { description = "Logical name of the lambda (e.g. address2uprn)" } -variable "image_uri" { - type = string - description = "Full ECR image URI including digest" -} - variable "stage" { description = "Deployment stage (e.g. dev, prod)" type = string } +variable "ecr_repo_url" { + type = string + description = "ECR repository URL (no tag, no digest)" +} + +variable "image_digest" { + type = string + description = "Image digest (sha256:...)" +} + + +locals { + image_uri = "${var.ecr_repo_url}@${var.image_digest}" +} + +output "resolved_image_uri" { + value = local.image_uri +} From 8a24ddce29a6f8ac0203e86275e955c7d735dcaa Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Tue, 3 Feb 2026 16:19:56 +0000 Subject: [PATCH 68/86] added changes to template --- .../terraform/lambda/_template/main.tf | 2 +- .../terraform/lambda/_template/variables.tf | 18 +++++++++++++----- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/infrastructure/terraform/lambda/_template/main.tf b/infrastructure/terraform/lambda/_template/main.tf index a0e8eb80..32d92e00 100644 --- a/infrastructure/terraform/lambda/_template/main.tf +++ b/infrastructure/terraform/lambda/_template/main.tf @@ -4,7 +4,7 @@ module "lambda" { name = REPLACE ME #"address2uprn" for example stage = var.stage - image_uri = var.image_uri + image_uri = local.image_uri environment = { diff --git a/infrastructure/terraform/lambda/_template/variables.tf b/infrastructure/terraform/lambda/_template/variables.tf index 4c9fd47d..c0b8780d 100644 --- a/infrastructure/terraform/lambda/_template/variables.tf +++ b/infrastructure/terraform/lambda/_template/variables.tf @@ -3,13 +3,21 @@ variable "lambda_name" { description = "Logical name of the lambda (e.g. address2uprn)" } -variable "image_uri" { - type = string - description = "Full ECR image URI including digest" -} - variable "stage" { description = "Deployment stage (e.g. dev, prod)" type = string } +variable "ecr_repo_url" { + type = string + description = "ECR repository URL (no tag, no digest)" +} +variable "image_digest" { + type = string + description = "Image digest (sha256:...)" +} + + +locals { + image_uri = "${var.ecr_repo_url}@${var.image_digest}" +} From 9e4a372dd65e309ab80ade136a34371fd0bf53aa Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Tue, 3 Feb 2026 16:34:04 +0000 Subject: [PATCH 69/86] check input and outputs --- .github/workflows/_build_image.yml | 8 ++++++++ .github/workflows/_deploy_lambda.yml | 9 +++++++++ 2 files changed, 17 insertions(+) diff --git a/.github/workflows/_build_image.yml b/.github/workflows/_build_image.yml index 9414b959..abef60a7 100644 --- a/.github/workflows/_build_image.yml +++ b/.github/workflows/_build_image.yml @@ -86,3 +86,11 @@ jobs: --output text) echo "image_digest=$DIGEST" >> "$GITHUB_OUTPUT" + + - name: Debug job outputs + run: | + echo "repo(step)=${{ steps.repo.outputs.ecr_repo_url }}" + echo "digest(step)=${{ steps.digest.outputs.image_digest }}" + echo "repo(job)=${{ jobs.build.outputs.ecr_repo_url }}" + echo "digest(job)=${{ jobs.build.outputs.image_digest }}" + diff --git a/.github/workflows/_deploy_lambda.yml b/.github/workflows/_deploy_lambda.yml index 07bfe331..787e9cd3 100644 --- a/.github/workflows/_deploy_lambda.yml +++ b/.github/workflows/_deploy_lambda.yml @@ -38,6 +38,15 @@ jobs: steps: - uses: actions/checkout@v4 + - name: Debug inputs + run: | + echo "lambda_name=${{ inputs.lambda_name }}" + echo "lambda_path=${{ inputs.lambda_path }}" + echo "stage=${{ inputs.stage }}" + echo "ecr_repo_url=${{ inputs.ecr_repo_url }}" + echo "image_digest=${{ inputs.image_digest }}" + + - uses: aws-actions/configure-aws-credentials@v4 with: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} From d3f019a0fb646cf89c8aa0b2045f4a5518d43bb8 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Tue, 3 Feb 2026 16:43:19 +0000 Subject: [PATCH 70/86] force run --- .github/workflows/_build_image.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/_build_image.yml b/.github/workflows/_build_image.yml index abef60a7..7ece5941 100644 --- a/.github/workflows/_build_image.yml +++ b/.github/workflows/_build_image.yml @@ -94,3 +94,4 @@ jobs: echo "repo(job)=${{ jobs.build.outputs.ecr_repo_url }}" echo "digest(job)=${{ jobs.build.outputs.image_digest }}" + From a8150e3c91d9316b4c323f5abae2e50c86910c04 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Tue, 3 Feb 2026 17:01:37 +0000 Subject: [PATCH 71/86] merged from main --- asset_list/app.py | 188 +++++++++++++++++++++++++++++----------------- 1 file changed, 120 insertions(+), 68 deletions(-) diff --git a/asset_list/app.py b/asset_list/app.py index 9907a609..201e3ca8 100644 --- a/asset_list/app.py +++ b/asset_list/app.py @@ -12,23 +12,35 @@ from asset_list.utils import get_data from dotenv import load_dotenv from backend.SearchEpc import SearchEpc + load_dotenv(dotenv_path="backend/.env") -EPC_AUTH_TOKEN = os.getenv("EPC_AUTH_TOKEN", "a2Nvbm5rb3dsZXNzYXJAZ21haWwuY29tOjY5MGJiMWM0NmIyOGI5ZDUxYzAxMzQzYzNiZGNlZGJjZDNmODQwMzA=") +EPC_AUTH_TOKEN = os.getenv( + "EPC_AUTH_TOKEN", + "a2Nvbm5rb3dsZXNzYXJAZ21haWwuY29tOjY5MGJiMWM0NmIyOGI5ZDUxYzAxMzQzYzNiZGNlZGJjZDNmODQwMzA=", +) -def extract_address1(asset_list, full_address_col, postcode_col, method="first_two_words"): +def extract_address1( + asset_list, full_address_col, postcode_col, method="first_two_words" +): if method == "first_two_words": - asset_list["address1_extracted"] = asset_list[full_address_col].str.split(" ").str[:2].str.join(" ") + asset_list["address1_extracted"] = ( + asset_list[full_address_col].str.split(" ").str[:2].str.join(" ") + ) return asset_list if method == "first_word": - asset_list["address1_extracted"] = asset_list[full_address_col].str.split(" ").str[0] + asset_list["address1_extracted"] = ( + asset_list[full_address_col].str.split(" ").str[0] + ) return asset_list if method == "house_number_extraction": asset_list["address1_extracted"] = asset_list.apply( - lambda x: SearchEpc.get_house_number(address=x[full_address_col], postcode=x[postcode_col]), - axis=1 + lambda x: SearchEpc.get_house_number( + address=x[full_address_col], postcode=x[postcode_col] + ), + axis=1, ) return asset_list @@ -57,15 +69,11 @@ def app(): EPC recommendations Property UPRN """ -<<<<<<< HEAD - data_folder = ("/workspaces/model/asset_list") - data_filename = "assets.xlsx" -======= data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Hackney" data_filename = "Domna SHF Wave 3 (3).xlsx" sheet_name = "Domna Wave 3" - postcode_column = 'Postcode' + postcode_column = "Postcode" address1_column = "Address 1" address1_method = None fulladdress_column = None @@ -96,15 +104,16 @@ def app(): landlord_block_reference = None # Peabody data for cleaning - data_folder = ("/Users/khalimconn-kowlessar/Documents/hestia/Customers/Peabody/Nov 2025 Consulting " - "Project/data_validation") + data_folder = ( + "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Peabody/Nov 2025 Consulting " + "Project/data_validation" + ) data_filename = "to_standardise_uprns.xlsx" ->>>>>>> 3874da6177cbcc37f7a488bec0a06e387906653c sheet_name = "Sheet1" - postcode_column = 'Postcode' + postcode_column = "Postcode" address1_column = None - address1_method = 'house_number_extraction' - fulladdress_column = 'Address' + address1_method = "house_number_extraction" + fulladdress_column = "Address" address_cols_to_concat = None missing_postcodes_method = None landlord_year_built = None @@ -155,49 +164,62 @@ def app(): landlord_existing_pv=landlord_existing_pv, landlord_sap=landlord_sap, landlord_block_reference=landlord_block_reference, - phase=phase + phase=phase, ) asset_list.init_standardise() # We produce the new maps, which can be saved for future useage new_property_type_map = { - k: v for k, v in ( - asset_list.variable_mappings[asset_list.landlord_property_type] if - asset_list.landlord_property_type else {} + k: v + for k, v in ( + asset_list.variable_mappings[asset_list.landlord_property_type] + if asset_list.landlord_property_type + else {} ).items() if k not in PROPERTY_MAPPING } new_built_form_map = { - k: v for k, v in ( - asset_list.variable_mappings[asset_list.landlord_built_form] if - asset_list.landlord_built_form else {} + k: v + for k, v in ( + asset_list.variable_mappings[asset_list.landlord_built_form] + if asset_list.landlord_built_form + else {} ).items() if k not in BUILT_FORM_MAPPINGS } new_wall_map = { - k: v for k, v in ( - asset_list.variable_mappings[asset_list.landlord_wall_construction] if - asset_list.landlord_wall_construction else {} + k: v + for k, v in ( + asset_list.variable_mappings[asset_list.landlord_wall_construction] + if asset_list.landlord_wall_construction + else {} ).items() if k not in WALL_CONSTRUCTION_MAPPINGS } new_heating_map = { - k: v for k, v in ( - asset_list.variable_mappings[asset_list.landlord_heating_system] if - asset_list.landlord_heating_system else {} + k: v + for k, v in ( + asset_list.variable_mappings[asset_list.landlord_heating_system] + if asset_list.landlord_heating_system + else {} ).items() if k not in HEATING_MAPPINGS } new_existing_pv_map = { - k: v for k, v in ( - asset_list.variable_mappings[asset_list.landlord_existing_pv] if asset_list.landlord_existing_pv else {} + k: v + for k, v in ( + asset_list.variable_mappings[asset_list.landlord_existing_pv] + if asset_list.landlord_existing_pv + else {} ).items() if k not in EXISTING_PV_MAPPINGS } new_roof_construction_map = { - k: v for k, v in ( - asset_list.variable_mappings[asset_list.landlord_roof_construction] if - asset_list.landlord_roof_construction else {} + k: v + for k, v in ( + asset_list.variable_mappings[asset_list.landlord_roof_construction] + if asset_list.landlord_roof_construction + else {} ).items() if k not in ROOF_CONSTRUCTION_MAPPINGS } @@ -211,7 +233,7 @@ def app(): outcomes_address=outcomes_address, outcomes_postcode=outcomes_postcode, outcomes_houseno=outcomes_houseno, - outcomes_id=outcomes_id + outcomes_id=outcomes_id, ) asset_list.flag_survey_master( @@ -245,14 +267,16 @@ def app(): skip = max(chunk_indexes) if any(x in folder_contents for x in downloaded_files): - skip = max([i for i in chunk_indexes if filename.format(i=i) in folder_contents]) + skip = max( + [i for i in chunk_indexes if filename.format(i=i) in folder_contents] + ) for i in range(0, len(asset_list.standardised_asset_list), chunk_size): print(f"Processing chunk {i} to {i + chunk_size}") if skip is not None and not force_retrieve_data: if i <= skip: continue - chunk = asset_list.standardised_asset_list[i:i + chunk_size] + chunk = asset_list.standardised_asset_list[i : i + chunk_size] epc_data_chunk, errors_chunk, no_epc_chunk = get_data( df=chunk, row_id_name=asset_list.DOMNA_PROPERTY_ID, @@ -264,7 +288,7 @@ def app(): built_form_column=AssetList.STANDARD_BUILT_FORM, manual_uprn_map=manual_uprn_map, epc_api_only=epc_api_only, - epc_auth_token=EPC_AUTH_TOKEN + epc_auth_token=EPC_AUTH_TOKEN, ) # We now retrieve any failed properties @@ -287,7 +311,9 @@ def app(): # Append the failed data to the main data # Store the chunk locally as a csv - pd.DataFrame(epc_data_chunk).to_csv(os.path.join(data_folder, f"Chunks/Chunk {i}.csv"), index=False) + pd.DataFrame(epc_data_chunk).to_csv( + os.path.join(data_folder, f"Chunks/Chunk {i}.csv"), index=False + ) # Store the errors and no-data locally with open(os.path.join(data_folder, f"Chunks/Chunk {i} errors.json"), "w") as f: json.dump(errors_chunk, f) @@ -318,7 +344,9 @@ def app(): unique_recommendations = set() for _, row in recommendations_df.iterrows(): - unique_recommendations.update([rec["improvement-summary-text"] for rec in row["recommendations"]]) + unique_recommendations.update( + [rec["improvement-summary-text"] for rec in row["recommendations"]] + ) columns = [asset_list.DOMNA_PROPERTY_ID] + list(unique_recommendations) transformed_data = [] @@ -338,20 +366,24 @@ def app(): transformed_df = pd.DataFrame(transformed_data) for col in [ "Floor insulation (solid floor)", - "Floor insulation", "Floor insulation (suspended floor)" + "Floor insulation", + "Floor insulation (suspended floor)", ]: if col not in transformed_df.columns: transformed_df[col] = False transformed_df = transformed_df[ [ - asset_list.DOMNA_PROPERTY_ID, "Floor insulation (solid floor)", - "Floor insulation", "Floor insulation (suspended floor)" + asset_list.DOMNA_PROPERTY_ID, + "Floor insulation (solid floor)", + "Floor insulation", + "Floor insulation (suspended floor)", ] ] transformed_df["epc_has_floor_recommendation"] = ( - transformed_df["Floor insulation (solid floor)"] | transformed_df["Floor insulation"] | - transformed_df["Floor insulation (suspended floor)"] + transformed_df["Floor insulation (solid floor)"] + | transformed_df["Floor insulation"] + | transformed_df["Floor insulation (suspended floor)"] ) # Get the find my epc data @@ -364,21 +396,20 @@ def app(): find_my_epc_data.append( { asset_list.DOMNA_PROPERTY_ID: x[asset_list.DOMNA_PROPERTY_ID], - **x["find_my_epc_data"] + **x["find_my_epc_data"], } ) else: find_my_epc_data.append( - { - asset_list.DOMNA_PROPERTY_ID: x[asset_list.DOMNA_PROPERTY_ID] - } + {asset_list.DOMNA_PROPERTY_ID: x[asset_list.DOMNA_PROPERTY_ID]} ) find_my_epc_data = pd.DataFrame(find_my_epc_data) find_my_epc_data = find_my_epc_data.merge( transformed_df[[asset_list.DOMNA_PROPERTY_ID, "epc_has_floor_recommendation"]], - how="left", on=asset_list.DOMNA_PROPERTY_ID + how="left", + on=asset_list.DOMNA_PROPERTY_ID, ) # We check if we get the solar pv column: @@ -388,24 +419,26 @@ def app(): # Retrieve just the data we need epc_df = epc_df[ [asset_list.DOMNA_PROPERTY_ID] + list(asset_list.EPC_API_DATA_NAMES.keys()) - ].rename( - columns=asset_list.EPC_API_DATA_NAMES - ) + ].rename(columns=asset_list.EPC_API_DATA_NAMES) # Look for columns not in the find my EPC data, which will have happened if we didn't # retrieve it in the first place - missed_find_epc_cols = [c for c in list(asset_list.FIND_EPC_DATA_NAMES.keys()) if c not in find_my_epc_data.columns] + missed_find_epc_cols = [ + c + for c in list(asset_list.FIND_EPC_DATA_NAMES.keys()) + if c not in find_my_epc_data.columns + ] if missed_find_epc_cols: for c in missed_find_epc_cols: find_my_epc_data[c] = None epc_df = epc_df.merge( find_my_epc_data[ - [asset_list.DOMNA_PROPERTY_ID, "epc_has_floor_recommendation"] + list(asset_list.FIND_EPC_DATA_NAMES.keys()) - ] - .rename(columns=asset_list.FIND_EPC_DATA_NAMES), + [asset_list.DOMNA_PROPERTY_ID, "epc_has_floor_recommendation"] + + list(asset_list.FIND_EPC_DATA_NAMES.keys()) + ].rename(columns=asset_list.FIND_EPC_DATA_NAMES), how="left", - on=asset_list.DOMNA_PROPERTY_ID + on=asset_list.DOMNA_PROPERTY_ID, ) asset_list.merge_data(epc_df) @@ -422,7 +455,10 @@ def app(): asset_list.get_work_figures() # Store as an excel - filename = os.path.join(data_folder, ".".join(data_filename.split(".")[:-1])) + " - Standardised.xlsx" + filename = ( + os.path.join(data_folder, ".".join(data_filename.split(".")[:-1])) + + " - Standardised.xlsx" + ) # Store the data in two tabs. One for the asset list with the EPC data and the second with the flat data # Determine inspections priority @@ -446,26 +482,42 @@ def app(): # ) with pd.ExcelWriter(filename) as writer: - asset_list.standardised_asset_list.to_excel(writer, sheet_name="Standardised Asset List", index=False) + asset_list.standardised_asset_list.to_excel( + writer, sheet_name="Standardised Asset List", index=False + ) if asset_list.block_analysis_df is not None: - asset_list.block_analysis_df.to_excel(writer, sheet_name="Block Analysis", index=False) + asset_list.block_analysis_df.to_excel( + writer, sheet_name="Block Analysis", index=False + ) # If we have outcomes, we add a tab with the outcomes if not asset_list.outcomes_for_output.empty: - asset_list.outcomes_for_output.to_excel(writer, sheet_name="Outcomes", index=False) + asset_list.outcomes_for_output.to_excel( + writer, sheet_name="Outcomes", index=False + ) if not asset_list.unmatched_submissions.empty: - asset_list.unmatched_submissions.to_excel(writer, sheet_name="Unmatched Submissions", index=False) + asset_list.unmatched_submissions.to_excel( + writer, sheet_name="Unmatched Submissions", index=False + ) if not asset_list.outcomes_no_match.empty: - asset_list.outcomes_no_match.to_excel(writer, sheet_name="Unmatched Outcomes", index=False) + asset_list.outcomes_no_match.to_excel( + writer, sheet_name="Unmatched Outcomes", index=False + ) if not asset_list.ecosurv_no_match.empty: - asset_list.ecosurv_no_match.to_excel(writer, sheet_name="Unmatched Ecosurv", index=False) + asset_list.ecosurv_no_match.to_excel( + writer, sheet_name="Unmatched Ecosurv", index=False + ) if not asset_list.geographical_areas.empty: - asset_list.geographical_areas.to_excel(writer, sheet_name="Geographical Areas", index=False) + asset_list.geographical_areas.to_excel( + writer, sheet_name="Geographical Areas", index=False + ) # Store dupes if asset_list.duplicated_addresses is not None: if not asset_list.duplicated_addresses.empty: - asset_list.duplicated_addresses.to_excel(writer, sheet_name="Duplicate Properties", index=False) + asset_list.duplicated_addresses.to_excel( + writer, sheet_name="Duplicate Properties", index=False + ) From 03ba2dc231d39fa97f2e9bcdde7f10db7c91a026 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Tue, 3 Feb 2026 17:06:46 +0000 Subject: [PATCH 72/86] logs --- .github/workflows/_build_image.yml | 9 --------- 1 file changed, 9 deletions(-) diff --git a/.github/workflows/_build_image.yml b/.github/workflows/_build_image.yml index 7ece5941..9414b959 100644 --- a/.github/workflows/_build_image.yml +++ b/.github/workflows/_build_image.yml @@ -86,12 +86,3 @@ jobs: --output text) echo "image_digest=$DIGEST" >> "$GITHUB_OUTPUT" - - - name: Debug job outputs - run: | - echo "repo(step)=${{ steps.repo.outputs.ecr_repo_url }}" - echo "digest(step)=${{ steps.digest.outputs.image_digest }}" - echo "repo(job)=${{ jobs.build.outputs.ecr_repo_url }}" - echo "digest(job)=${{ jobs.build.outputs.image_digest }}" - - From 33d1288d55e76a3e501ee3cd8e1b7d329a6fee2f Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Tue, 3 Feb 2026 17:21:06 +0000 Subject: [PATCH 73/86] make it more explicit --- .github/workflows/_build_image.yml | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/.github/workflows/_build_image.yml b/.github/workflows/_build_image.yml index 9414b959..9f929b4d 100644 --- a/.github/workflows/_build_image.yml +++ b/.github/workflows/_build_image.yml @@ -20,13 +20,8 @@ on: type: string outputs: - image_digest: - description: "Pushed image digest (sha256:...)" - value: ${{ jobs.build.outputs.image_digest }} - - ecr_repo_url: - description: "ECR repository URL (no tag, no digest)" - value: ${{ jobs.build.outputs.ecr_repo_url }} + image_digest: ${{ steps.digest.outputs.image_digest }} + ecr_repo_url: ${{ steps.repo.outputs.ecr_repo_url }} secrets: AWS_ACCESS_KEY_ID: From 8c0866a84ea9ab409ab061e5e831993753c832a4 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Tue, 3 Feb 2026 17:27:23 +0000 Subject: [PATCH 74/86] added certainity to output --- .github/workflows/_build_image.yml | 33 ++++++++++-------------------- 1 file changed, 11 insertions(+), 22 deletions(-) diff --git a/.github/workflows/_build_image.yml b/.github/workflows/_build_image.yml index 9f929b4d..b6ebcd52 100644 --- a/.github/workflows/_build_image.yml +++ b/.github/workflows/_build_image.yml @@ -4,24 +4,23 @@ on: workflow_call: inputs: ecr_repo: - description: "ECR repository name" required: true type: string - dockerfile_path: - description: "Path to Dockerfile" required: true type: string - build_context: - description: "Docker build context directory" required: false default: "." type: string outputs: - image_digest: ${{ steps.digest.outputs.image_digest }} - ecr_repo_url: ${{ steps.repo.outputs.ecr_repo_url }} + image_digest: + description: "Pushed image digest" + value: ${{ jobs.build.outputs.image_digest }} + ecr_repo_url: + description: "ECR repository URL" + value: ${{ jobs.build.outputs.ecr_repo_url }} secrets: AWS_ACCESS_KEY_ID: @@ -42,33 +41,24 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v4 + - uses: aws-actions/configure-aws-credentials@v4 with: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} aws-region: ${{ secrets.AWS_REGION }} - - name: Login to ECR - uses: aws-actions/amazon-ecr-login@v2 + - uses: aws-actions/amazon-ecr-login@v2 - name: Resolve ECR repo URL id: repo run: | AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text) - ECR_REPO_URL="${AWS_ACCOUNT_ID}.dkr.ecr.${{ secrets.AWS_REGION }}.amazonaws.com/${{ inputs.ecr_repo }}" - echo "ecr_repo_url=$ECR_REPO_URL" >> "$GITHUB_OUTPUT" + echo "ecr_repo_url=${AWS_ACCOUNT_ID}.dkr.ecr.${{ secrets.AWS_REGION }}.amazonaws.com/${{ inputs.ecr_repo }}" >> "$GITHUB_OUTPUT" - name: Build & push image run: | - IMAGE_TAG=${GITHUB_SHA} - IMAGE_URI="${{ steps.repo.outputs.ecr_repo_url }}:${IMAGE_TAG}" - - docker build \ - -f ${{ inputs.dockerfile_path }} \ - -t $IMAGE_URI \ - ${{ inputs.build_context }} - + IMAGE_URI="${{ steps.repo.outputs.ecr_repo_url }}:${GITHUB_SHA}" + docker build -f ${{ inputs.dockerfile_path }} -t $IMAGE_URI ${{ inputs.build_context }} docker push $IMAGE_URI - name: Resolve image digest @@ -79,5 +69,4 @@ jobs: --image-ids imageTag=${GITHUB_SHA} \ --query 'imageDetails[0].imageDigest' \ --output text) - echo "image_digest=$DIGEST" >> "$GITHUB_OUTPUT" From 388b74f58a32cdcf14b2a996de82f81bfbbe9faa Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Tue, 3 Feb 2026 17:37:18 +0000 Subject: [PATCH 75/86] more logs --- .github/workflows/deploy_terraform.yml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/.github/workflows/deploy_terraform.yml b/.github/workflows/deploy_terraform.yml index 38f841e7..274fc196 100644 --- a/.github/workflows/deploy_terraform.yml +++ b/.github/workflows/deploy_terraform.yml @@ -80,6 +80,26 @@ jobs: AWS_SECRET_ACCESS_KEY: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }} AWS_REGION: ${{ secrets.DEV_AWS_REGION }} + # ============================================================ + # 2.5️⃣ Check Address2UPRN image outputs + # ============================================================ + check_address2uprn_image: + needs: address2uprn_image + runs-on: ubuntu-latest + + steps: + - name: Dump image outputs + run: | + echo "ECR repo URL:" + echo "${{ needs.address2uprn_image.outputs.ecr_repo_url }}" + echo + echo "Image digest:" + echo "${{ needs.address2uprn_image.outputs.image_digest }}" + echo + echo "Full image URI:" + echo "${{ needs.address2uprn_image.outputs.ecr_repo_url }}@${{ needs.address2uprn_image.outputs.image_digest }}" + + # ============================================================ # 3️⃣ Deploy Address 2 UPRN Lambda # ============================================================ From 94682ccd5fcc0ed06953df9f6a4a8a9021938c8c Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Tue, 3 Feb 2026 19:01:46 +0000 Subject: [PATCH 76/86] more logs --- .github/workflows/_build_image.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/_build_image.yml b/.github/workflows/_build_image.yml index b6ebcd52..d29f1f62 100644 --- a/.github/workflows/_build_image.yml +++ b/.github/workflows/_build_image.yml @@ -55,6 +55,9 @@ jobs: AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text) echo "ecr_repo_url=${AWS_ACCOUNT_ID}.dkr.ecr.${{ secrets.AWS_REGION }}.amazonaws.com/${{ inputs.ecr_repo }}" >> "$GITHUB_OUTPUT" + - name: GITHUB_OUTPUT + run: echo $GITHUB_OUTPUT + - name: Build & push image run: | IMAGE_URI="${{ steps.repo.outputs.ecr_repo_url }}:${GITHUB_SHA}" From 2359131ff9f41d0bca96fe99c8ab0f5d73a49253 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Tue, 3 Feb 2026 19:09:02 +0000 Subject: [PATCH 77/86] even greater amount of logs --- .github/workflows/_build_image.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/_build_image.yml b/.github/workflows/_build_image.yml index d29f1f62..6b6c4994 100644 --- a/.github/workflows/_build_image.yml +++ b/.github/workflows/_build_image.yml @@ -53,10 +53,13 @@ jobs: id: repo run: | AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text) - echo "ecr_repo_url=${AWS_ACCOUNT_ID}.dkr.ecr.${{ secrets.AWS_REGION }}.amazonaws.com/${{ inputs.ecr_repo }}" >> "$GITHUB_OUTPUT" - - name: GITHUB_OUTPUT - run: echo $GITHUB_OUTPUT + ECR_REPO_URL="${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_REGION}.amazonaws.com/${{ inputs.ecr_repo }}" + + echo "Resolved ECR repo URL (local var):" + echo "$ECR_REPO_URL" + + echo "ecr_repo_url=$ECR_REPO_URL" >> "$GITHUB_OUTPUT" - name: Build & push image run: | From 813f6e0bf34bf99391fd5d8478474a1b5f8753b4 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Tue, 3 Feb 2026 19:19:25 +0000 Subject: [PATCH 78/86] work out github ecr within lambda --- .github/workflows/_deploy_lambda.yml | 15 +++++++++++++-- .github/workflows/deploy_terraform.yml | 22 +--------------------- 2 files changed, 14 insertions(+), 23 deletions(-) diff --git a/.github/workflows/_deploy_lambda.yml b/.github/workflows/_deploy_lambda.yml index 787e9cd3..bff106c5 100644 --- a/.github/workflows/_deploy_lambda.yml +++ b/.github/workflows/_deploy_lambda.yml @@ -15,7 +15,7 @@ on: required: true type: string - ecr_repo_url: + ecr_repo: required: true type: string @@ -55,6 +55,17 @@ jobs: - uses: hashicorp/setup-terraform@v3 + - uses: aws-actions/amazon-ecr-login@v2 + + - name: Resolve ECR repo URL + id: repo + env: + AWS_REGION: ${{ secrets.AWS_REGION }} + run: | + AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text) + ECR_REPO_URL="${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_REGION}.amazonaws.com/${{ inputs.ecr_repo }}" + echo "ecr_repo_url=$ECR_REPO_URL" >> "$GITHUB_OUTPUT" + - name: Terraform Init working-directory: ${{ inputs.lambda_path }} run: terraform init -reconfigure @@ -71,7 +82,7 @@ jobs: terraform plan \ -var="stage=${{ inputs.stage }}" \ -var="lambda_name=${{ inputs.lambda_name }}" \ - -var="ecr_repo_url=${{ inputs.ecr_repo_url }}" \ + -var="ecr_repo_url=${{ steps.repo.outputs.ecr_repo_url }}" \ -var="image_digest=${{ inputs.image_digest }}" \ -out=lambdaplan diff --git a/.github/workflows/deploy_terraform.yml b/.github/workflows/deploy_terraform.yml index 274fc196..41a551c4 100644 --- a/.github/workflows/deploy_terraform.yml +++ b/.github/workflows/deploy_terraform.yml @@ -80,26 +80,6 @@ jobs: AWS_SECRET_ACCESS_KEY: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }} AWS_REGION: ${{ secrets.DEV_AWS_REGION }} - # ============================================================ - # 2.5️⃣ Check Address2UPRN image outputs - # ============================================================ - check_address2uprn_image: - needs: address2uprn_image - runs-on: ubuntu-latest - - steps: - - name: Dump image outputs - run: | - echo "ECR repo URL:" - echo "${{ needs.address2uprn_image.outputs.ecr_repo_url }}" - echo - echo "Image digest:" - echo "${{ needs.address2uprn_image.outputs.image_digest }}" - echo - echo "Full image URI:" - echo "${{ needs.address2uprn_image.outputs.ecr_repo_url }}@${{ needs.address2uprn_image.outputs.image_digest }}" - - # ============================================================ # 3️⃣ Deploy Address 2 UPRN Lambda # ============================================================ @@ -110,7 +90,7 @@ jobs: lambda_name: address2uprn lambda_path: infrastructure/terraform/lambda/address2UPRN stage: ${{ needs.determine_stage.outputs.stage }} - ecr_repo_url: ${{ needs.address2uprn_image.outputs.ecr_repo_url }} + ecr_repo: address2uprn-${{ needs.determine_stage.outputs.stage }} image_digest: ${{ needs.address2uprn_image.outputs.image_digest }} secrets: AWS_ACCESS_KEY_ID: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }} From 3539c186abc47d867e02c80ff2ce7401ba59bd1a Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Tue, 3 Feb 2026 19:23:51 +0000 Subject: [PATCH 79/86] increase timeout for now --- infrastructure/terraform/modules/sqs_queue/main.tf | 2 ++ 1 file changed, 2 insertions(+) diff --git a/infrastructure/terraform/modules/sqs_queue/main.tf b/infrastructure/terraform/modules/sqs_queue/main.tf index 478c345c..580e67bd 100644 --- a/infrastructure/terraform/modules/sqs_queue/main.tf +++ b/infrastructure/terraform/modules/sqs_queue/main.tf @@ -5,6 +5,8 @@ resource "aws_sqs_queue" "dlq" { resource "aws_sqs_queue" "this" { name = var.name + visibility_timeout_seconds = 120 + redrive_policy = jsonencode({ deadLetterTargetArn = aws_sqs_queue.dlq.arn maxReceiveCount = var.max_receive_count From dc2a070145dddd8ef72f0f96e667b0ee75a23047 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Tue, 3 Feb 2026 19:48:59 +0000 Subject: [PATCH 80/86] update template so dan can just cp and paste --- infrastructure/terraform/lambda/_template/main.tf | 1 + infrastructure/terraform/lambda/_template/variables.tf | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/infrastructure/terraform/lambda/_template/main.tf b/infrastructure/terraform/lambda/_template/main.tf index 32d92e00..3010aa8a 100644 --- a/infrastructure/terraform/lambda/_template/main.tf +++ b/infrastructure/terraform/lambda/_template/main.tf @@ -9,5 +9,6 @@ module "lambda" { environment = { STAGE = var.stage + LOG_LEVEL = "info" } } diff --git a/infrastructure/terraform/lambda/_template/variables.tf b/infrastructure/terraform/lambda/_template/variables.tf index c0b8780d..e4bab243 100644 --- a/infrastructure/terraform/lambda/_template/variables.tf +++ b/infrastructure/terraform/lambda/_template/variables.tf @@ -21,3 +21,7 @@ variable "image_digest" { locals { image_uri = "${var.ecr_repo_url}@${var.image_digest}" } + +output "resolved_image_uri" { + value = local.image_uri +} From e24fe641f931d328f4ff4759fd6893a40de22480 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Tue, 3 Feb 2026 19:53:49 +0000 Subject: [PATCH 81/86] remove epc token --- asset_list/app.py | 1 - 1 file changed, 1 deletion(-) diff --git a/asset_list/app.py b/asset_list/app.py index 201e3ca8..b46254f9 100644 --- a/asset_list/app.py +++ b/asset_list/app.py @@ -16,7 +16,6 @@ from backend.SearchEpc import SearchEpc load_dotenv(dotenv_path="backend/.env") EPC_AUTH_TOKEN = os.getenv( "EPC_AUTH_TOKEN", - "a2Nvbm5rb3dsZXNzYXJAZ21haWwuY29tOjY5MGJiMWM0NmIyOGI5ZDUxYzAxMzQzYzNiZGNlZGJjZDNmODQwMzA=", ) From 360f06df83aa0f2b4077e718eb848d93a8131b5a Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Tue, 3 Feb 2026 19:57:07 +0000 Subject: [PATCH 82/86] more instructions --- infrastructure/terraform/lambda/_template/README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/infrastructure/terraform/lambda/_template/README.md b/infrastructure/terraform/lambda/_template/README.md index 0dd612ee..2c3ddd43 100644 --- a/infrastructure/terraform/lambda/_template/README.md +++ b/infrastructure/terraform/lambda/_template/README.md @@ -52,6 +52,10 @@ The CI pipeline resolves the repository URL dynamically at build time. 2. Resolve the image digest 3. Deploy the Lambda using the immutable `image_uri` +--- +### 5. Delete + 1. Delete README if you used cp -r + --- ## Notes From 440165dc2d9ab475346af3fc67c245291161b2e9 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Tue, 3 Feb 2026 20:03:25 +0000 Subject: [PATCH 83/86] better logs --- .../terraform/lambda/_template/README.md | 41 +++++-------------- 1 file changed, 10 insertions(+), 31 deletions(-) diff --git a/infrastructure/terraform/lambda/_template/README.md b/infrastructure/terraform/lambda/_template/README.md index 2c3ddd43..a7282fc9 100644 --- a/infrastructure/terraform/lambda/_template/README.md +++ b/infrastructure/terraform/lambda/_template/README.md @@ -5,12 +5,6 @@ cp -r lambda/_template lambda/ -- Update `variables.tf` if required -- Ensure the Terraform module accepts: - - `lambda_name` - - `stage` - - `image_uri` - --- ### 2. Add infrastructure prerequisites (shared stack) @@ -18,12 +12,10 @@ infrastructure/terraform/shared/main.tf -- Apply the shared stack (or let CI apply it on `prod`) -- Verify the ECR repository exists in AWS +- Apply the shared stack + - This requires commenting 'if env.stage == "prod"' in .github/workflows/deploy_terraform.yml -Note: -No Terraform outputs are required for the ECR repo. -The CI pipeline resolves the repository URL dynamically at build time. +- Verify the ECR repository exists in AWS --- @@ -36,11 +28,6 @@ The CI pipeline resolves the repository URL dynamically at build time. ### 4. Wire the Lambda deploy job (CI) - Add a deploy job using `_deploy_lambda.yml` -- Pass the following inputs: - - `lambda_name` - - `lambda_path` - - `stage` - - `image_uri` (constructed as `repo@sha256:digest`) - Ensure the deploy job depends on the image build job --- @@ -49,24 +36,16 @@ The CI pipeline resolves the repository URL dynamically at build time. - Push changes to GitHub - CI will: 1. Build and push the Docker image - 2. Resolve the image digest - 3. Deploy the Lambda using the immutable `image_uri` - + 2. Deploy the Lambda + 3. Verify everything deployed. Good things to check: + - ECR with image + - SQS + - Trigger SQS + - Cloud watch logs --- ### 5. Delete 1. Delete README if you used cp -r --- -## Notes -- Terraform remote state is not used for image wiring -- Image tags are not used; deployments are digest-based -- By default, the template wires SQS → Lambda - To change this, update the Terraform in `lambda/` - (e.g. EventBridge, API Gateway, direct invoke) - ---- - -## Rule of thumb -CI decides what image to deploy. -Terraform only deploys it. +## Please feel free to update this document to make it easier for the next person \ No newline at end of file From af9f73e9ee939068deec4829dec2f63f6a1ce5f4 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Tue, 3 Feb 2026 20:04:38 +0000 Subject: [PATCH 84/86] reduce default to 10 --- .../terraform/modules/container_registry/variables.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infrastructure/terraform/modules/container_registry/variables.tf b/infrastructure/terraform/modules/container_registry/variables.tf index 501a5044..11821b31 100644 --- a/infrastructure/terraform/modules/container_registry/variables.tf +++ b/infrastructure/terraform/modules/container_registry/variables.tf @@ -11,5 +11,5 @@ variable "stage" { variable "retain_count" { description = "Number of images to retain" type = number - default = 20 + default = 10 } From 2e122598c145645050bf91a3956a5ea46a2c526b Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Tue, 3 Feb 2026 20:05:42 +0000 Subject: [PATCH 85/86] better comments --- infrastructure/terraform/modules/ecr/main.tf | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/infrastructure/terraform/modules/ecr/main.tf b/infrastructure/terraform/modules/ecr/main.tf index 9288a1fe..d93d1340 100644 --- a/infrastructure/terraform/modules/ecr/main.tf +++ b/infrastructure/terraform/modules/ecr/main.tf @@ -1,7 +1,5 @@ -# This ECR module is used in Khalim's default code, Junte tried changing it -# but decided to priotise delivariables as sales projects are coming soon -# one day we can unify ECR policies together but Junte decided to seperate -# the continaer lambda as it runs slighly differently +# This ecr works for things deployed by serverless. +# TODO: unify ecr and container_registry to one resource "aws_ecr_repository" "my_repository" { name = "${var.ecr_name}" From 14b108cd94e63deea35fcd4599651bc7c0dc86a1 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Tue, 3 Feb 2026 20:07:00 +0000 Subject: [PATCH 86/86] get rid of comment --- infrastructure/terraform/shared/dev.tfvars | 1 - 1 file changed, 1 deletion(-) diff --git a/infrastructure/terraform/shared/dev.tfvars b/infrastructure/terraform/shared/dev.tfvars index 544104b1..53ca6d9e 100644 --- a/infrastructure/terraform/shared/dev.tfvars +++ b/infrastructure/terraform/shared/dev.tfvars @@ -1,5 +1,4 @@ stage = "dev" -# profile = "DevAdmin" region = "eu-west-2" # Domain