diff --git a/.devcontainer/backend/docker-compose.yml b/.devcontainer/backend/docker-compose.yml index 75526e79..683b4489 100644 --- a/.devcontainer/backend/docker-compose.yml +++ b/.devcontainer/backend/docker-compose.yml @@ -9,10 +9,20 @@ services: command: sleep infinity volumes: - ../../:/workspaces/model - networks: - - model-net -networks: - model-net: - driver: bridge + db: + image: postgres:17.4 + restart: unless-stopped + ports: + - 5432:5432 + environment: + - PGDATABASE=tech_team_local_db + - POSTGRES_USER=postgres + - POSTGRES_PASSWORD=makingwarmerhomes + volumes: + - postgres-data-two:/var/lib/postgresql/data + + +volumes: + postgres-data-two: \ No newline at end of file diff --git a/backend/.env.local b/backend/.env.local new file mode 100644 index 00000000..a05c93a3 --- /dev/null +++ b/backend/.env.local @@ -0,0 +1,22 @@ +DB_HOST=db +DB_PORT=5432 +DB_NAME=tech_team_local_db +DB_USERNAME=postgres +DB_PASSWORD=makingwarmerhomes + + +#not used +GOOGLE_SOLAR_API_KEY="test" +SAP_PREDICTIONS_BUCKET="test" +CARBON_PREDICTIONS_BUCKET="test" +HEAT_PREDICTIONS_BUCKET="test" +HEATING_KWH_PREDICTIONS_BUCKET="test" +HOTWATER_KWH_PREDICTIONS_BUCKET="test" +API_KEY="test" +ENVIRONMENT="test" +SECRET_KEY="test" +PLAN_TRIGGER_BUCKET="test" +DATA_BUCKET="test" +EPC_AUTH_TOKEN="test" +ENGINE_SQS_URL="test" +ENERGY_ASSESSMENTS_BUCKET="test" \ No newline at end of file diff --git a/backend/app/config.py b/backend/app/config.py index dd3f5db1..b335c215 100644 --- a/backend/app/config.py +++ b/backend/app/config.py @@ -42,7 +42,7 @@ class Settings(BaseSettings): AWS_DEFAULT_REGION: Optional[str] = None class Config: - env_file = "backend/.env" + env_file = "backend/.env.local" @lru_cache() diff --git a/backend/app/db/models/condition.py b/backend/app/db/models/condition.py index cecf1fde..77043366 100644 --- a/backend/app/db/models/condition.py +++ b/backend/app/db/models/condition.py @@ -18,12 +18,14 @@ ElementTypeDb = SqlEnum( ElementType, name="element_type", native_enum=True, + values_callable=lambda enum: [e.value for e in enum], ) AspectTypeDb = SqlEnum( AspectType, name="aspect_type", native_enum=True, + values_callable=lambda enum: [a.value for a in enum], ) diff --git a/backend/condition/local_runner.py b/backend/condition/local_runner.py index f7580139..fc6516cc 100644 --- a/backend/condition/local_runner.py +++ b/backend/condition/local_runner.py @@ -21,7 +21,8 @@ def main() -> None: / "2026_01_06 - Peabody - Stock Condition Data - Survey Records - D Lower.xlsx" ) # filepaths = [lbwf_path, peabody_path] - filepaths = [lbwf_path] + # filepaths = [lbwf_path] + filepaths = [peabody_path] for fp in filepaths: with fp.open("rb") as f: diff --git a/backend/condition/parsing/peabody_parser.py b/backend/condition/parsing/peabody_parser.py index b8a548a7..423c0cda 100644 --- a/backend/condition/parsing/peabody_parser.py +++ b/backend/condition/parsing/peabody_parser.py @@ -3,17 +3,22 @@ from openpyxl import Workbook, load_workbook from collections import defaultdict from backend.condition.parsing.parser import Parser -from backend.condition.parsing.records.peabody.peabody_asset_condition import PeabodyAssetCondition +from backend.condition.parsing.records.peabody.peabody_asset_condition import ( + PeabodyAssetCondition, +) from backend.condition.parsing.records.peabody.peabody_property import PeabodyProperty from utils.logger import setup_logger logger = setup_logger() + class PeabodyParser(Parser): def parse(self, file_stream: BinaryIO) -> Any: wb: Workbook = load_workbook(file_stream) - address_to_uprn_map: Dict[str, int] = PeabodyParser._generate_address_to_uprn_dict(wb) - + address_to_uprn_map: Dict[str, int] = ( + PeabodyParser._generate_address_to_uprn_dict(wb) + ) + assets = self._parse_assets(wb) return self._group_assets_into_properties( @@ -21,7 +26,6 @@ class PeabodyParser(Parser): address_to_uprn_map=address_to_uprn_map, ) - @staticmethod def _parse_assets(wb: Workbook) -> List[PeabodyAssetCondition]: assets_sheet = wb["Survey Records - D & Lower"] @@ -33,24 +37,28 @@ class PeabodyParser(Parser): assets: List[PeabodyAssetCondition] = [] for row in asset_rows: try: - asset = PeabodyParser._map_row_to_asset_record(row, asset_header_indexes) + asset = PeabodyParser._map_row_to_asset_record( + row, asset_header_indexes + ) if not asset.is_block_level: # Block-level condition surveys are out of scope for now - # until we have a wider think on how to handle block - assets.append(asset) # TODO: handle block-level assets + # until we have a wider think on how to handle block + assets.append(asset) # TODO: handle block-level assets except Exception as e: logger.error(f"Error mapping Peabody row to asset record: {e}") continue return assets - + @staticmethod def _group_assets_into_properties( assets: List[PeabodyAssetCondition], address_to_uprn_map: Dict[str, int], ) -> List[PeabodyProperty]: - assets_by_address: DefaultDict[str, List[PeabodyAssetCondition]] = defaultdict(list) + assets_by_address: DefaultDict[str, List[PeabodyAssetCondition]] = defaultdict( + list + ) for asset in assets: if asset.full_address is None: @@ -62,6 +70,7 @@ class PeabodyParser(Parser): properties: List[PeabodyProperty] = [] for address, grouped_assets in assets_by_address.items(): + uprn = address_to_uprn_map.get(address) if uprn is None: @@ -77,7 +86,6 @@ class PeabodyParser(Parser): return properties - @staticmethod def _map_row_to_asset_record( row: Any | Tuple[object | None, ...], @@ -108,14 +116,16 @@ class PeabodyParser(Parser): rows: Iterator[Tuple[object | None, ...]] = sheet.iter_rows(values_only=True) headers = next(rows) - header_indexes: Dict[str, int] = PeabodyParser._get_column_indexes_by_name(headers) + header_indexes: Dict[str, int] = PeabodyParser._get_column_indexes_by_name( + headers + ) address_idx = header_indexes["full_address"] - address_to_uprn: Dict[str, int] = {} # Generate random UPRNs for now - next_uprn = 1 # TODO: get real UPRNs + + next_uprn = 1 # TODO: get real UPRNs for row in rows: address = row[address_idx] @@ -131,10 +141,9 @@ class PeabodyParser(Parser): return address_to_uprn - @staticmethod def _get_column_indexes_by_name( - headers: Tuple[object | None, ...] + headers: Tuple[object | None, ...], ) -> Dict[str, int]: index: Dict[str, int] = {} @@ -142,4 +151,4 @@ class PeabodyParser(Parser): if isinstance(header, str): index[header] = i - return index \ No newline at end of file + return index diff --git a/backend/condition/persistence/condition_postgres.py b/backend/condition/persistence/condition_postgres.py index 4d5f2add..9d7895f0 100644 --- a/backend/condition/persistence/condition_postgres.py +++ b/backend/condition/persistence/condition_postgres.py @@ -17,7 +17,7 @@ logger = setup_logger() class ConditionPostgres: def bulk_insert_surveys( - surveys: List[PropertyConditionSurvey], batch_size: Optional[int] = 100 + self, surveys: List[PropertyConditionSurvey], batch_size: Optional[int] = 100 ) -> None: logger.info( f"Preparing to load {len(surveys)} property surveys to Postgres. Mapping to SQLModel objects..."