diff --git a/etl/daily_script.py b/etl/daily_script.py index 4365e4a..27e3df1 100644 --- a/etl/daily_script.py +++ b/etl/daily_script.py @@ -1,10 +1,12 @@ import os from pdfReader.pdfReaderToText import pdfReaderToText -from etl.scraper.scraper import SharePointScraper, SharePointInstaller +from etl.scraper.scraper import SharePointScraper, SharePointInstaller, WEEK_COMMENCING from pprint import pprint, pformat import logging from etl.utils.logger import Logger from etl.validator.validator import DomnaSharePointValidator +from collections import Counter + logger = Logger(name="main.py", level=logging.INFO).get_logger() @@ -16,17 +18,23 @@ def main(): # Correct dates format checker south_coast_scraper = SharePointScraper(SharePointInstaller.SOUTH_COAST_INSULATION) south_coast_names = south_coast_scraper.list_of_names_that_has_the_wrong_date_format() + south_coast_submissions = south_coast_scraper.get_number_of_surverys_completed() jjc_scraper = SharePointScraper(SharePointInstaller.JJC) jjc_names = jjc_scraper.list_of_names_that_has_the_wrong_date_format() + jjc_coast_submission = jjc_scraper.get_number_of_surverys_completed() SGEC = SharePointScraper(SharePointInstaller.SGEC) sgec_names = SGEC.list_of_names_that_has_the_wrong_date_format() + sgec_submission = SGEC.get_number_of_surverys_completed() BAXTER_KELLY = SharePointScraper(SharePointInstaller.BAXTER_KELLY) b_names = BAXTER_KELLY.list_of_names_that_has_the_wrong_date_format() + BAXTER_KELLY_submissions = BAXTER_KELLY.get_number_of_surverys_completed() - + total_dict = dict(Counter(south_coast_submissions) + Counter(jjc_coast_submission) + Counter(sgec_submission) + Counter(BAXTER_KELLY_submissions)) + + logger.info("-------------------------------------------") logger.info("Good morning Cyrus") if south_coast_names: logger.info("South Coast with wrong date format:") @@ -44,6 +52,10 @@ def main(): if b_names: logger.info("Baxter Kelly with wrong date format") logger.info(pformat(b_names)) + logger.info("-------------------------------------------") + logger.info(f"For week commencing: {WEEK_COMMENCING}") + logger.info(f"Submissions: {pformat(total_dict)}") + logger.info("-------------------------------------------") # Make a quick script that checks if the Pictures folder exists in a certain fail directory diff --git a/etl/development.py b/etl/development.py index 1e26f5f..d6a1b02 100644 --- a/etl/development.py +++ b/etl/development.py @@ -17,6 +17,7 @@ pdfReader2 = pdfReaderToText(DATA_LOC_2) doc1 = pdfReader2.get_reader() print(doc2.property_description.waterHeating) + # Transform def main(): @@ -33,6 +34,6 @@ if __name__ == "__main__": # - [x] Finish off scraping for section that I need to finish # - [in progress] Pydantic format for deemed report -# - [] Generate deemed report +# - [] Generate deemed report -> scopre of automations thats possible # - [] Docker compose to an sql database in docker compose (2 hours, then work on sql) # - [] Deploy via terraform to aws (1 day) \ No newline at end of file diff --git a/etl/pdfReader/sitenotes.py b/etl/pdfReader/sitenotes.py index a001d05..78ec346 100644 --- a/etl/pdfReader/sitenotes.py +++ b/etl/pdfReader/sitenotes.py @@ -3,7 +3,8 @@ from transform.types import ( CompanyInfo, SurverySummaryInfo, AssessorInfo, PropertyDescription, PropertyDetail, Dimension, Walls, Roofs, Floors, Door, VentilationAndCooling, - Lighting, WaterHeating, HotWaterCylinder, + Lighting, WaterHeating, HotWaterCylinder, SolarWaterHeating, + ShowerAndBaths, FlueGasHeatRecoverySystem, ) from datetime import datetime @@ -48,9 +49,6 @@ class QuidosSiteNotesExtractor(SiteNotesExtractor): # These one are quick fixes can be done on train or one at a time - # self.get_section_16() - # self.get_section_17() - # self.get_section_18() # self.get_section_19() # self.get_section_20() # self.get_section_21() @@ -221,6 +219,18 @@ class QuidosSiteNotesExtractor(SiteNotesExtractor): # Section 15.1 Hot Water Cylinder hotWaterCylinder = self.get_hot_water_cylinder() + # Section 16 Solar Water Heating + solarWaterHeating = self.get_solar_water_heating() + + # Section 17.0 + # ignored as it has nothing in the copy i'm using. Future todo clarity + + # Section 18.0 + showerAndBaths = self.get_shower_and_baths() + + # Section 19.0 + fghrs = self.get_fghrs() + self.property_description = PropertyDescription( built_form = get_value("Built Form"), detachment_or_position = get_value("Detachment/Position"), @@ -269,6 +279,9 @@ class QuidosSiteNotesExtractor(SiteNotesExtractor): lighting=lighting, waterHeating=waterHeating, hotWaterCylinder=hotWaterCylinder, + solarWaterHeating=solarWaterHeating, + showerAndBaths=showerAndBaths, + flueGasHeatRecoverySystem=fghrs, ) @@ -407,8 +420,6 @@ class QuidosSiteNotesExtractor(SiteNotesExtractor): return walls - - def get_roof(self): data = self.raw_data[self.raw_data.index('8.0 Roofs'): self.raw_data.index('9.0 Floors')] sub_titles = [ @@ -725,6 +736,7 @@ class QuidosSiteNotesExtractor(SiteNotesExtractor): "15.1 Hot Water Cylinder" ] dict_ = self.two_columns_processor(data, sub_tites, avoid) + return HotWaterCylinder( volume=dict_.get("volume", ""), insulation_type=dict_.get("insulation_type", ""), @@ -732,7 +744,7 @@ class QuidosSiteNotesExtractor(SiteNotesExtractor): thermostat=True if dict_.get("thermostat", "NO").upper() == "YES" else False, ) - def get_section_16(self): + def get_solar_water_heating(self): data = self.raw_data[self.raw_data.index("16.0 Solar Water Heating"):self.raw_data.index("17.0 Waste Water Heat Recovery System")] avoid = [ "16.0 Solar Water Heating", @@ -742,12 +754,19 @@ class QuidosSiteNotesExtractor(SiteNotesExtractor): "Solar Water Heating Details Known?" ] - self.two_columns_processor(data, sub_titles, avoid, 16.0) + dict_ = self.two_columns_processor(data, sub_titles, avoid) + + return SolarWaterHeating( + solar_water_heating_details_known=True if dict_.get("solar_water_heating_details_known", "NO").upper() == "YES" else False + ) def get_section_17(self): - pass + """ + Furture todo + """ + raise NotImplemented("Please contact Jun-te Kim to implement this") - def get_section_18(self): + def get_shower_and_baths(self): data = self.get_data_between("18.0 Showers And Baths", "19.0 Flue Gas Heat Recovery System") sub_titles = [ "Number of Rooms with Bath and/or Shower", @@ -757,7 +776,8 @@ class QuidosSiteNotesExtractor(SiteNotesExtractor): "19.0 Flue Gas Heat Recovery System", ] - self.two_columns_processor(data, sub_titles, avoid, 18.0) + dict_1 = self.two_columns_processor(data, sub_titles, avoid) + avoid = [ "18.0 Showers And Baths", "19.0 Flue Gas Heat Recovery System", @@ -766,9 +786,17 @@ class QuidosSiteNotesExtractor(SiteNotesExtractor): "Number of Rooms with Mixer Shower and no", # Number of Rooms with Mixer Shower and no Bath "Number of Rooms with Mixer Shower and", # Number of Rooms with Mixer Shower and Bath ] - self.two_columns_processor(data, sub_titles, avoid, 18.0, 2) + dict_2 = self.two_columns_processor(data, sub_titles, avoid, 2) + + return ShowerAndBaths( + no_of_rooms_with_baths_and_or_shower=int(dict_1.get("number_of_rooms_with_bath_and/or_shower", -1)), + no_of_rooms_with_mixer_shower_and_no_baths=int(dict_2.get("number_of_rooms_with_mixer_shower_and_no", -1)), + no_of_rooms_with_mixer_shower_and_baths=int(dict_2.get("number_of_rooms_with_mixer_shower_and", -1)), + ) + + - def get_section_19(self): + def get_fghrs(self): data = self.get_data_between("19.0 Flue Gas Heat Recovery System","20.0 Photovoltaic Panel") sub_titles = [ "FGHRS Present", @@ -778,7 +806,12 @@ class QuidosSiteNotesExtractor(SiteNotesExtractor): "20.0 Photovoltaic Panel", ] - self.two_columns_processor(data, sub_titles, avoid, 19) + dict_ = self.two_columns_processor(data, sub_titles, avoid) + + return FlueGasHeatRecoverySystem( + fghrs_present=True if dict_.get("fghrs_present", "NO").upper() == "YES" else False + ) + def get_section_20(self): data = self.get_data_between("20.0 Photovoltaic Panel","21.0 Wind Turbine") diff --git a/etl/scraper/scraper.py b/etl/scraper/scraper.py index de7a9ac..0fb83a8 100644 --- a/etl/scraper/scraper.py +++ b/etl/scraper/scraper.py @@ -9,7 +9,7 @@ from functools import wraps import re from etl.validator.validator import DomnaSharePointValidator -WEEK_COMMENCING = os.getenv("WEEK_COMMENCING", "W.C. 03.03.2025") +WEEK_COMMENCING = os.getenv("WEEK_COMMENCING", "W.C. 10.03.2025") class SharePointInstaller(Enum): SOUTH_COAST_INSULATION = os.getenv("SOUTH_COAST_INSULATION_SERVICE_SHAREPOINT_ID", None) diff --git a/etl/transform/types.py b/etl/transform/types.py index f83d9b4..c48db71 100644 --- a/etl/transform/types.py +++ b/etl/transform/types.py @@ -120,6 +120,18 @@ class HotWaterCylinder(BaseModel): insulation_thickness: str thermostat: bool +class SolarWaterHeating(BaseModel): + solar_water_heating_details_known: bool + +class ShowerAndBaths(BaseModel): + no_of_rooms_with_baths_and_or_shower: int + no_of_rooms_with_mixer_shower_and_no_baths: int + no_of_rooms_with_mixer_shower_and_baths: int + +class FlueGasHeatRecoverySystem(BaseModel): + fghrs_present: bool + + class PropertyDetail(BaseModel): age_band: str dimensions: List[Dimension] = [] @@ -151,4 +163,7 @@ class PropertyDescription(BaseModel): ventilationAndCooling: Optional[VentilationAndCooling] lighting: Optional[Lighting] waterHeating: Optional[WaterHeating] - hotWaterCylinder: Optional[HotWaterCylinder] \ No newline at end of file + hotWaterCylinder: Optional[HotWaterCylinder] + solarWaterHeating: Optional[SolarWaterHeating] + showerAndBaths: Optional[ShowerAndBaths] + flueGasHeatRecoverySystem: Optional[FlueGasHeatRecoverySystem]