junte/Model
1
0
Fork 0
mirror of https://github.com/Hestia-Homes/Model.git synced 2026-06-08 11:17:27 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'main' into feature/epc-dataclasses

Browse source
This commit is contained in:
Daniel Roth 2026-04-14 10:57:48 +00:00
commit 1f6fabd171
5 changed files with 45 additions and 31 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

18
asset_list/app.py
View file

@ -74,23 +74,23 @@ def app():
"""
data_folder = "/workspaces/model/asset_list"
data_filename = "Calico ARA Upload Review.xlsx"
sheet_name = "Sheet1"
postcode_column = "Postcode"
address1_column = "Units"
data_filename = "Waverley UPRN Match.xlsx"
sheet_name = "in"
postcode_column = "postcode_clean"
address1_column = "domna_found_address"
address1_method = None
fulladdress_column = "Units"
address_cols_to_concat = ["Units"]
fulladdress_column = "domna_found_address"
address_cols_to_concat = []
missing_postcodes_method = None
landlord_year_built = None
landlord_os_uprn = None
landlord_property_type = None # Good to include if landlord gave
landlord_os_uprn = "domna_found_uprn"
landlord_property_type = "Property Type 1" # Good to include if landlord gave
landlord_built_form = None # Good to include if landlord gave
landlord_wall_construction = None
landlord_roof_construction = None
landlord_heating_system = None
landlord_existing_pv = None
landlord_property_id = "llid"
landlord_property_id = "WBC Ref"
landlord_sap = None
outcomes_filename = None
outcomes_sheetname = None

6
backend/address2UPRN/main.py
View file

@ -351,9 +351,9 @@ def handler(event, context, local=False):
{
"body": json.dumps(
{
"task_id": "e31f2f21-175b-4a91-a3ec-a6baa325e917",
"sub_task_id": "6a427b6e-1ece-4983-b1e5-9bffccc53d1d",
"s3_uri": "s3://retrofit-data-dev/ara_postcode_splitter_batches/e31f2f21-175b-4a91-a3ec-a6baa325e917/8673913b-1a88-42d7-8578-0449123d94b0/2026-02-18T11:47:00.822579_f95467f5.csv",
"sub_task_id": "d7363c83-2ef7-4474-b30f-980fd587350c",
"task_id": "a042af13-8b57-4709-ad22-ecac1ccca4bd",
"s3_uri": "s3://retrofit-data-dev/ara_raw_inputs/essex/Copy of EPC register Essex(August 2025)(in) (2).csv",
}
)
}

2
backend/pashub_fetcher/handler/handler.py
View file

@ -20,6 +20,7 @@ from backend.pashub_fetcher.pashub_to_ara_trigger_request import (
)
from backend.pashub_fetcher.sharepoint_subfolders import SharepointSubfolders
from backend.pashub_fetcher.token_getter import get_token_from_local_storage
from backend.utils.subtasks import task_handler
from utils.logger import setup_logger
from utils.s3 import upload_file_to_s3
from utils.sharepoint.domna_sharepoint_client import DomnaSharepointClient
@ -165,6 +166,7 @@ def process_job(
return job_files
@task_handler()
def handler(event: Mapping[str, Any], context: Any) -> None:
logger.info("Received message")
logger.info(f"Number of events: {len(event.get('Records', []))}")

48
etl/hubspot/scripts/scraper/main.py
View file

@ -22,7 +22,6 @@ def handler(body: dict[str, Any], context: Any) -> None:
hubspot_client = HubspotClient()
sqs_client = boto3.client("sqs")
PASHUB_TRIGGER_QUEUE_URL = get_settings().PASHUB_TO_ARA_SQS_URL
payload = HubspotTriggerOrchestratorTriggerRequest.model_validate(body)
hubspot_deal_id: str = payload.hubspot_deal_id
@ -41,7 +40,6 @@ def handler(body: dict[str, Any], context: Any) -> None:
deal_changed = False
if not db_deal:
# New hubspot deal, no diffing to do
logger.info(f"New HubSpot deal of ID {hubspot_deal_id}. Loading to database...")
if company:
company_data: CompanyData = hubspot_client.get_company_information(company)
@ -49,6 +47,15 @@ def handler(body: dict[str, Any], context: Any) -> None:
db_client.upsert_organisation(company_data)
db_client.upsert_deal(hubspot_deal, company, listing, hubspot_client)
# ==============================
# Orchestration of other lambdas
# ==============================
if hubspot_deal["pashub_link"]:
logger.info(
f"Triggering Pas Hub file fetcher for HubSpot deal ID {hubspot_deal_id}"
)
_trigger_pashub_fetcher(sqs_client, hubspot_deal)
else:
# Deal already in db, check whether anything has changed
logger.info(
@ -84,22 +91,7 @@ def handler(body: dict[str, Any], context: Any) -> None:
logger.info(
f"Triggering Pas Hub file fetcher for HubSpot deal ID {hubspot_deal_id}"
)
message_body: Dict[str, Optional[str]] = {
"pashub_link": hubspot_deal["pashub_link"],
"address": None, # potentially available from Listing, leave as None for now
"sharepoint_link": hubspot_deal["sharepoint_link"],
"uprn": hubspot_deal["national_uprn"],
"landlord_property_id": hubspot_deal["owner_property_id"],
"deal_stage": hubspot_deal["deal_stage"],
}
response = sqs_client.send_message(
QueueUrl=PASHUB_TRIGGER_QUEUE_URL, MessageBody=json.dumps(message_body)
)
logger.info(
f"Sent message to Pashub To Ara queue. MessageId: {response['MessageId']}"
)
_trigger_pashub_fetcher(sqs_client, hubspot_deal)
else:
logger.info(
f"Not Triggering PasHub file fetcher for HubSpot deal ID {hubspot_deal_id}"
@ -108,6 +100,26 @@ def handler(body: dict[str, Any], context: Any) -> None:
print("done")
def _trigger_pashub_fetcher(sqs_client: Any, hubspot_deal: Dict[str, str]) -> None:
message_body: Dict[str, Optional[str]] = {
"pashub_link": hubspot_deal["pashub_link"],
"address": None, # potentially available from Listing, leave as None for now
"sharepoint_link": hubspot_deal["sharepoint_link"],
"uprn": hubspot_deal["national_uprn"],
"landlord_property_id": hubspot_deal["owner_property_id"],
"deal_stage": hubspot_deal["deal_stage"],
}
response = sqs_client.send_message(
QueueUrl=get_settings().PASHUB_TO_ARA_SQS_URL,
MessageBody=json.dumps(message_body),
)
logger.info(
f"Sent message to Pashub To Ara queue. MessageId: {response['MessageId']}"
)
if __name__ == "__main__":
handler({"hubspot_deal_id": "371470706915"}, "")
print("beep")

2
utils/s3.py
View file

@ -330,7 +330,7 @@ def read_csv_from_s3(bucket_name: str, filepath: str) -> list[dict[str, str]]:
body = s3_object["Body"].read()
# Use StringIO to create a file-like object from the string
csv_data = StringIO(body.decode("utf-8"))
csv_data = StringIO(body.decode("utf-8-sig"))
# Use csv library to read it into a list of dictionaries
reader = csv.DictReader(csv_data)