Model/orchestration/sharepoint_renamer_orchestrator.py
Daniel Roth ae1da26558 Move the PasHub fetcher into the DDD layer structure 🟪
The service was the last one living wholly under backend/. It now follows
the same layering as abri and the other newer services:

  domain/pashub_fetcher/          core file classification, subfolders
  infrastructure/pashub_fetcher/  PasHub client, token getter, wire DTOs
  orchestration/                  PashubFetcherOrchestrator (was PashubService)
  applications/pashub_fetcher/    lambda handler, trigger request, dev tooling

core_files.py is split along the layer boundary: the domain module keeps the
filename/evidence-category classification rules and no longer imports
infrastructure.postgres, while the CoreFiles -> FileTypeEnum translation moves
to infrastructure/pashub_fetcher/core_file_types.py.

Tests move into the tests/ tree by layer. Note this puts them in the only
suite CI currently runs (unit_tests.yml is disabled), so these 73 tests now
execute on PRs for the first time; they were previously reachable only via
the legacy pytest.ini testpaths.

sharepoint_renamer's image now copies just domain/pashub_fetcher/ rather than
the whole service, since SharepointSubfolders is all it needed.

Behaviour is unchanged. tests/ goes 9927 -> 10000 passed (+73, exactly the
tests that moved in); the legacy suite keeps its same 17 pre-existing failures
and 11 errors. tests/test_lambda_packaging.py confirms both changed
Dockerfiles still copy their handler's full import closure.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-07-22 13:35:54 +00:00

152 lines
5.2 KiB
Python

import csv
import os
from typing import Optional
from domain.pashub_fetcher.sharepoint_subfolders import SharepointSubfolders
from utilities.logger import setup_logger
from utils.sharepoint.domna_sharepoint_client import DomnaSharepointClient
BASE_PATH = (
"Osmosis-ACD Projects/Sero-Clarion Housing/"
"Sero Project Documents/Property Folders"
)
ASSESSMENT_SUBFOLDER = "A. Assessment"
BATCH_FOLDER_PREFIX = "_Sero Batch"
logger = setup_logger()
def build_canonical_filename(
uprn: str, address: str, postcode: str, original_name: str
) -> Optional[str]:
"""
Returns the canonical filename, or None if the file is already renamed.
Already-renamed: name starts with "{uprn}_".
Strips any existing address prefix (address+postcode first, then address alone)
before inserting the canonical prefix.
"""
if original_name.startswith(f"{uprn}_"):
return None
stem, ext = os.path.splitext(original_name)
stem_lower = stem.lower()
street = address.split(",")[0].strip()
prefixes = [
f"{address} {postcode}",
address,
f"{street} {postcode}",
street,
]
doc_name = stem
for prefix in prefixes:
if stem_lower.startswith(prefix.lower()):
doc_name = stem[len(prefix) :]
break
if doc_name.startswith(" - "):
doc_name = doc_name[3:]
elif doc_name.startswith(" _ "):
doc_name = doc_name[3:]
doc_name = doc_name.strip()
street_post = f"{street} {postcode}"
if doc_name:
return f"{uprn}_{street_post}_{doc_name}{ext}"
return f"{uprn}_{street_post}{ext}"
class SharepointRenamerOrchestrator:
def __init__(
self, sp_client: DomnaSharepointClient, csv_path: str, dry_run: bool = False
) -> None:
self._sp_client = sp_client
self._csv_path = csv_path
self._dry_run = dry_run
def _discover_roots(self) -> list[str]:
contents = self._sp_client.get_folders_in_path(BASE_PATH)
batch_roots = sorted(
f"{BASE_PATH}/{item['name']}"
for item in contents.get("value", [])
if "folder" in item
and item["name"].lower().startswith(BATCH_FOLDER_PREFIX.lower())
)
return [BASE_PATH, *batch_roots]
def run(self) -> None:
roots = self._discover_roots()
with open(self._csv_path, newline="", encoding="utf-8-sig") as f:
reader = csv.DictReader(f)
required = {"UPRN", "Address", "Postcode"}
if not reader.fieldnames or not required.issubset(set(reader.fieldnames)):
raise ValueError(
f"CSV missing required columns. Expected {required}, got {reader.fieldnames}"
)
for row in reader:
self._process_row(
roots,
uprn=row["UPRN"].strip(),
address=row["Address"].strip(),
postcode=row["Postcode"].strip(),
)
def _process_row(
self, roots: list[str], uprn: str, address: str, postcode: str
) -> None:
# Batch folders name properties "{uprn}_{address}"; filenames must not
# repeat the UPRN, so the folder lookup and the canonical name diverge.
display_address = address.removeprefix(f"{uprn}_")
for root in roots:
folder_path = (
f"{root}/{address}, {postcode}"
f"/{SharepointSubfolders.ASSESSMENT.value}/{ASSESSMENT_SUBFOLDER}"
)
if self._process_folder(folder_path, uprn, display_address, postcode):
return
logger.warning(
f"Missing folder for UPRN {uprn} in any root: {address}, {postcode}"
)
def _process_folder(
self, folder_path: str, uprn: str, address: str, postcode: str
) -> bool:
try:
contents = self._sp_client.get_folders_in_path(folder_path)
except ValueError:
return False
for item in contents.get("value", []):
if "folder" in item:
self._process_folder(
f"{folder_path}/{item['name']}", uprn, address, postcode
)
elif "file" in item:
original_name: str = item["name"]
if os.path.splitext(original_name)[1].lower() in {".jpg", ".heic"}:
continue
new_name = build_canonical_filename(
uprn, address, postcode, original_name
)
if new_name is None:
continue
if self._dry_run:
logger.info(
f'Would rename: "{original_name}""{new_name}" (UPRN: {uprn})'
)
else:
try:
self._sp_client.rename_file(item["id"], new_name)
logger.info(
f'Renamed: "{original_name}""{new_name}" (UPRN: {uprn})'
)
except Exception as e:
logger.error(
f'Failed to rename "{original_name}""{new_name}" (UPRN: {uprn}): {e}'
)
return True