From eb72b0223f629e2e3499b887598fd72142fe282a Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Thu, 4 Jun 2026 09:11:08 +0000 Subject: [PATCH 1/3] add ECO_SHAREPOINT_ID env var --- .github/workflows/_deploy_lambda.yml | 4 ++++ .github/workflows/deploy_terraform.yml | 1 + backend/app/config.py | 1 + backend/pashub_fetcher/trigger_pashub_sqs_from_file.py | 8 +++++--- deployment/terraform/lambda/pashub_to_ara/main.tf | 1 + deployment/terraform/lambda/pashub_to_ara/variables.tf | 5 +++++ utils/sharepoint/domna_sites.py | 1 + 7 files changed, 18 insertions(+), 3 deletions(-) diff --git a/.github/workflows/_deploy_lambda.yml b/.github/workflows/_deploy_lambda.yml index 70f9eabe..0035a579 100644 --- a/.github/workflows/_deploy_lambda.yml +++ b/.github/workflows/_deploy_lambda.yml @@ -76,6 +76,8 @@ on: required: false TF_VAR_social_housing_wave_3_sharepoint_id: required: false + TF_VAR_eco_sharepoint_id: + required: false TF_VAR_pashub_email: required: false TF_VAR_pashub_password: @@ -159,6 +161,7 @@ jobs: TF_VAR_osmosis_acd_sharepoint_id: ${{ secrets.TF_VAR_osmosis_acd_sharepoint_id }} TF_VAR_private_pay_sharepoint_id: ${{ secrets.TF_VAR_private_pay_sharepoint_id }} TF_VAR_social_housing_wave_3_sharepoint_id: ${{ secrets.TF_VAR_social_housing_wave_3_sharepoint_id }} + TF_VAR_eco_sharepoint_id: ${{ secrets.TF_VAR_eco_sharepoint_id }} TF_VAR_pashub_email: ${{ secrets.TF_VAR_pashub_email }} TF_VAR_pashub_password: ${{ secrets.TF_VAR_pashub_password }} TF_VAR_pashub_coordination_email: ${{ secrets.TF_VAR_pashub_coordination_email }} @@ -210,6 +213,7 @@ jobs: TF_VAR_osmosis_acd_sharepoint_id: ${{ secrets.TF_VAR_osmosis_acd_sharepoint_id }} TF_VAR_private_pay_sharepoint_id: ${{ secrets.TF_VAR_private_pay_sharepoint_id }} TF_VAR_social_housing_wave_3_sharepoint_id: ${{ secrets.TF_VAR_social_housing_wave_3_sharepoint_id }} + TF_VAR_eco_sharepoint_id: ${{ secrets.TF_VAR_eco_sharepoint_id }} TF_VAR_pashub_email: ${{ secrets.TF_VAR_pashub_email }} TF_VAR_pashub_password: ${{ secrets.TF_VAR_pashub_password }} TF_VAR_pashub_coordination_email: ${{ secrets.TF_VAR_pashub_coordination_email }} diff --git a/.github/workflows/deploy_terraform.yml b/.github/workflows/deploy_terraform.yml index fc999bc0..fd003c8c 100644 --- a/.github/workflows/deploy_terraform.yml +++ b/.github/workflows/deploy_terraform.yml @@ -448,6 +448,7 @@ jobs: TF_VAR_osmosis_acd_sharepoint_id: ${{ secrets.OSMOSIS_ACD_SHAREPOINT_ID }} TF_VAR_private_pay_sharepoint_id: ${{ secrets.PRIVATE_PAY_SHAREPOINT_ID }} TF_VAR_social_housing_wave_3_sharepoint_id: ${{ secrets.SOCIAL_HOUSING_WAVE_3_SHAREPOINT_ID }} + TF_VAR_eco_sharepoint_id: ${{ secrets.TF_VAR_eco_sharepoint_id }} TF_VAR_pashub_email: ${{ secrets.PASHUB_EMAIL }} TF_VAR_pashub_password: ${{ secrets.PASHUB_PASSWORD }} TF_VAR_pashub_coordination_email: ${{ secrets.PASHUB_COORDINATION_EMAIL }} diff --git a/backend/app/config.py b/backend/app/config.py index f969518d..1dc3daaf 100644 --- a/backend/app/config.py +++ b/backend/app/config.py @@ -82,6 +82,7 @@ class Settings(BaseSettings): OSMOSIS_ACD_SHAREPOINT_ID: Optional[str] = None PRIVATE_PAY_SHAREPOINT_ID: Optional[str] = None SOCIAL_HOUSING_WAVE_3_SHAREPOINT_ID: Optional[str] = None + ECO_SHAREPOINT_ID: Optional[str] = None OPENAI_API_KEY: Optional[str] = None # Pas Hub diff --git a/backend/pashub_fetcher/trigger_pashub_sqs_from_file.py b/backend/pashub_fetcher/trigger_pashub_sqs_from_file.py index 118c0aca..fe3ec7d8 100644 --- a/backend/pashub_fetcher/trigger_pashub_sqs_from_file.py +++ b/backend/pashub_fetcher/trigger_pashub_sqs_from_file.py @@ -52,12 +52,14 @@ EXCEL_PATH: str = os.path.join( "local_run_02-06-2026/ECO_Approach_Coordination_Design_KN.xlsx", ) -SHAREPOINT_PROPERTIES_FOLDER: str = "" +SHAREPOINT_PROPERTIES_FOLDER: str = ( + "https://domnagroup.sharepoint.com/:f:/s/ECO/IgCCLUg3PJ-eS7BLDAwiNlelAW_WXLCb9wcjkigbLfHhPOY?e=DTbNNv" +) def _build_requests(excel_path: str) -> list[PashubToAraTriggerRequest]: wb = load_workbook(excel_path, data_only=True) - ws = wb.worksheets[0] + ws = wb.worksheets[1] headers: dict[str, int] = {} for col in range(1, ws.max_column + 1): @@ -123,7 +125,7 @@ def main() -> None: for request in trigger_requests: action: str = "DRY RUN" if DRY_RUN else "SENDING" logger.info( - f"[{action}] deal_id={request.hubspot_deal_id} pashub_link={request.pashub_link}" + f"[{action}] deal_id={request.hubspot_deal_id} pashub_link={request.pashub_link} sharepoint_link={request.sharepoint_link}" ) if not DRY_RUN: diff --git a/deployment/terraform/lambda/pashub_to_ara/main.tf b/deployment/terraform/lambda/pashub_to_ara/main.tf index eba9c874..b5714055 100644 --- a/deployment/terraform/lambda/pashub_to_ara/main.tf +++ b/deployment/terraform/lambda/pashub_to_ara/main.tf @@ -47,6 +47,7 @@ module "lambda" { OSMOSIS_ACD_SHAREPOINT_ID = var.osmosis_acd_sharepoint_id PRIVATE_PAY_SHAREPOINT_ID = var.private_pay_sharepoint_id SOCIAL_HOUSING_WAVE_3_SHAREPOINT_ID = var.social_housing_wave_3_sharepoint_id + ECO_SHAREPOINT_ID = var.eco_sharepoint_id PASHUB_EMAIL = var.pashub_email PASHUB_PASSWORD = var.pashub_password PASHUB_COORDINATION_EMAIL = var.pashub_coordination_email diff --git a/deployment/terraform/lambda/pashub_to_ara/variables.tf b/deployment/terraform/lambda/pashub_to_ara/variables.tf index cdeff256..29b7af70 100644 --- a/deployment/terraform/lambda/pashub_to_ara/variables.tf +++ b/deployment/terraform/lambda/pashub_to_ara/variables.tf @@ -92,6 +92,11 @@ variable "social_housing_wave_3_sharepoint_id" { sensitive = true } +variable "eco_sharepoint_id" { + type = string + sensitive = true +} + variable "pashub_email" { type = string sensitive = true diff --git a/utils/sharepoint/domna_sites.py b/utils/sharepoint/domna_sites.py index e5efb82c..ce579af6 100644 --- a/utils/sharepoint/domna_sites.py +++ b/utils/sharepoint/domna_sites.py @@ -9,3 +9,4 @@ class DomnaSites(Enum): OSMOSIS_ACD = os.getenv("OSMOSIS_ACD_SHAREPOINT_ID") PRIVATE_PAY = os.getenv("PRIVATE_PAY_SHAREPOINT_ID") SOCIAL_HOUSING_WAVE_3 = os.getenv("SOCIAL_HOUSING_WAVE_3_SHAREPOINT_ID") + ECO = os.getenv("ECO_SHAREPOINT_ID") From 416dac0512b13146975c04af1e30476670b8737f Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Thu, 4 Jun 2026 09:29:24 +0000 Subject: [PATCH 2/3] =?UTF-8?q?SharePoint=20upload=20is=20skipped=20when?= =?UTF-8?q?=20client=20is=20None=20=F0=9F=9F=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../tests/test_pashub_service.py | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/backend/pashub_fetcher/tests/test_pashub_service.py b/backend/pashub_fetcher/tests/test_pashub_service.py index 4ceeb832..ccb80ac4 100644 --- a/backend/pashub_fetcher/tests/test_pashub_service.py +++ b/backend/pashub_fetcher/tests/test_pashub_service.py @@ -616,6 +616,33 @@ def test_sharepoint_skips_upload_when_folder_not_found() -> None: mock_logger.warning.assert_called() +def test_sharepoint_skips_upload_when_sharepoint_client_is_none() -> None: + # Arrange + mock_client = MagicMock(spec=PashubClient) + mock_client.get_uprn_by_job_id.return_value = None + mock_client.get_evidence_files_by_job_id.return_value = make_downloaded( + core=["/tmp/core.pdf"] + ) + + service = PashubService( + pashub_client=mock_client, + sharepoint_client=None, + s3_bucket="test-bucket", + ) + + # Act — should not raise AttributeError on None._sharepoint_client + with patch("backend.pashub_fetcher.pashub_service.os.remove"): + result = service.run( + make_request( + sharepoint_link="Retrofit/Properties", + address="123 Main St | deal", + ) + ) + + # Assert + assert result == ["/tmp/core.pdf"] + + def test_run_warns_and_continues_when_site_notes_parsing_fails() -> None: mock_client = MagicMock(spec=PashubClient) mock_client.get_uprn_by_job_id.return_value = None From db2cbc9c574ea2a77008d44ac8b2842c0b58c096 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Thu, 4 Jun 2026 09:33:07 +0000 Subject: [PATCH 3/3] =?UTF-8?q?SharePoint=20upload=20is=20skipped=20when?= =?UTF-8?q?=20client=20is=20None=20=F0=9F=9F=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/pashub_fetcher/handler/handler.py | 14 ++++++++++---- backend/pashub_fetcher/pashub_service.py | 5 +++-- .../pashub_to_ara_trigger_request.py | 2 ++ .../pashub_fetcher/trigger_pashub_sqs_from_file.py | 5 ++++- 4 files changed, 19 insertions(+), 7 deletions(-) diff --git a/backend/pashub_fetcher/handler/handler.py b/backend/pashub_fetcher/handler/handler.py index 626ce59d..00f0ddea 100644 --- a/backend/pashub_fetcher/handler/handler.py +++ b/backend/pashub_fetcher/handler/handler.py @@ -40,10 +40,6 @@ def handler(body: Dict[str, Any], context: Any) -> List[str]: if (not pashub_email) or (not pashub_password): raise ValueError("Pas Hub credentials not provided") - sharepoint_client = DomnaSharepointClient( - sharepoint_location=DomnaSites.SOCIAL_HOUSING_WAVE_3 - ) - if coordination_hub_email and coordination_hub_password: _coord_email, _coord_password = ( coordination_hub_email, @@ -57,6 +53,16 @@ def handler(body: Dict[str, Any], context: Any) -> List[str]: payload = PashubToAraTriggerRequest.model_validate(body) logger.debug("Successfully validated request body") + sharepoint_client: Optional[DomnaSharepointClient] = None + if payload.sharepoint_site is not None: + try: + resolved_site = DomnaSites[payload.sharepoint_site] + sharepoint_client = DomnaSharepointClient(sharepoint_location=resolved_site) + except KeyError: + logger.warning( + f"Unrecognised sharepoint_site '{payload.sharepoint_site}'; skipping SharePoint upload" + ) + service = PashubService( pashub_client=get_pashub_client(pashub_email, pashub_password), sharepoint_client=sharepoint_client, diff --git a/backend/pashub_fetcher/pashub_service.py b/backend/pashub_fetcher/pashub_service.py index 2b7a7f03..86a553f0 100644 --- a/backend/pashub_fetcher/pashub_service.py +++ b/backend/pashub_fetcher/pashub_service.py @@ -38,7 +38,7 @@ class PashubService: def __init__( self, pashub_client: PashubClient, - sharepoint_client: DomnaSharepointClient, + sharepoint_client: Optional[DomnaSharepointClient], s3_bucket: str, coordination_client_factory: Optional[Callable[[], PashubClient]] = None, ) -> None: @@ -111,7 +111,7 @@ class PashubService: default_file_type=FileTypeEnum.OTHER.value, ) - if request.sharepoint_link and request.address: + if self._sharepoint_client and request.sharepoint_link and request.address: folder_name = request.address.split("|")[0].strip() folders = self._sharepoint_client.get_folders_in_path(request.sharepoint_link) match = next( @@ -207,6 +207,7 @@ class PashubService: property_folder_path: str, files: List[DownloadedFile], ) -> None: + assert self._sharepoint_client is not None for df in files: filename = os.path.basename(df.file_path) try: diff --git a/backend/pashub_fetcher/pashub_to_ara_trigger_request.py b/backend/pashub_fetcher/pashub_to_ara_trigger_request.py index 5f6ce37d..d747a388 100644 --- a/backend/pashub_fetcher/pashub_to_ara_trigger_request.py +++ b/backend/pashub_fetcher/pashub_to_ara_trigger_request.py @@ -14,6 +14,8 @@ class PashubToAraTriggerRequest(BaseModel): hubspot_listing_id: Optional[int] = None hubspot_deal_id: Optional[str] = None + sharepoint_site: Optional[str] = None + get_other_files: bool = False @property diff --git a/backend/pashub_fetcher/trigger_pashub_sqs_from_file.py b/backend/pashub_fetcher/trigger_pashub_sqs_from_file.py index fe3ec7d8..d6736eda 100644 --- a/backend/pashub_fetcher/trigger_pashub_sqs_from_file.py +++ b/backend/pashub_fetcher/trigger_pashub_sqs_from_file.py @@ -53,9 +53,11 @@ EXCEL_PATH: str = os.path.join( ) SHAREPOINT_PROPERTIES_FOLDER: str = ( - "https://domnagroup.sharepoint.com/:f:/s/ECO/IgCCLUg3PJ-eS7BLDAwiNlelAW_WXLCb9wcjkigbLfHhPOY?e=DTbNNv" + "Housing Associations/- Client Shared Folders/Abri/Abri Property Folders (Full PAS Info)" ) +SHAREPOINT_SITE: str = "ECO" + def _build_requests(excel_path: str) -> list[PashubToAraTriggerRequest]: wb = load_workbook(excel_path, data_only=True) @@ -104,6 +106,7 @@ def _build_requests(excel_path: str) -> list[PashubToAraTriggerRequest]: address=address, deal_stage=deal_stage, sharepoint_link=SHAREPOINT_PROPERTIES_FOLDER or None, + sharepoint_site=SHAREPOINT_SITE, ) )