Merge pull request #13 from Hestia-Homes/feature/sharepoint_integration

Feature/sharepoint integration

etl/daily_script.py

@@ -1,10 +1,12 @@
 import os
 from pdfReader.pdfReaderToText import pdfReaderToText
-from etl.scraper.scraper import SharePointScraper, SharePointInstaller
+from etl.scraper.scraper import SharePointScraper, SharePointInstaller, WEEK_COMMENCING
 from pprint import pprint, pformat
 import logging
 from etl.utils.logger import Logger
 from etl.validator.validator import DomnaSharePointValidator
+from collections import Counter
+
 
 logger = Logger(name="main.py", level=logging.INFO).get_logger()
 
@@ -16,17 +18,23 @@ def main():
     # Correct dates format checker
     south_coast_scraper = SharePointScraper(SharePointInstaller.SOUTH_COAST_INSULATION)
     south_coast_names = south_coast_scraper.list_of_names_that_has_the_wrong_date_format()
+    south_coast_submissions = south_coast_scraper.get_number_of_surverys_completed()
 
     jjc_scraper = SharePointScraper(SharePointInstaller.JJC)
     jjc_names = jjc_scraper.list_of_names_that_has_the_wrong_date_format()
+    jjc_coast_submission = jjc_scraper.get_number_of_surverys_completed()
 
     SGEC = SharePointScraper(SharePointInstaller.SGEC)
     sgec_names = SGEC.list_of_names_that_has_the_wrong_date_format()
+    sgec_submission = SGEC.get_number_of_surverys_completed()
 
     BAXTER_KELLY = SharePointScraper(SharePointInstaller.BAXTER_KELLY)
     b_names = BAXTER_KELLY.list_of_names_that_has_the_wrong_date_format()
+    BAXTER_KELLY_submissions = BAXTER_KELLY.get_number_of_surverys_completed()
 
-    
+    total_dict = dict(Counter(south_coast_submissions) + Counter(jjc_coast_submission) + Counter(sgec_submission) + Counter(BAXTER_KELLY_submissions))
+
+    logger.info("-------------------------------------------")
     logger.info("Good morning Cyrus")
     if south_coast_names:
         logger.info("South Coast with wrong date format:")
@@ -44,6 +52,10 @@ def main():
     if b_names:
         logger.info("Baxter Kelly with wrong date format")
         logger.info(pformat(b_names))
+    logger.info("-------------------------------------------")
+    logger.info(f"For week commencing: {WEEK_COMMENCING}")
+    logger.info(f"Submissions: {pformat(total_dict)}")
+    logger.info("-------------------------------------------")
     # Make a quick script that checks if the Pictures folder exists in a certain fail directory
 
 

etl/development.py

@@ -17,6 +17,7 @@ pdfReader2 = pdfReaderToText(DATA_LOC_2)
 doc1 = pdfReader2.get_reader()
 print(doc2.property_description.waterHeating)
 
+
 # Transform
 
 def main():
@@ -33,6 +34,6 @@ if __name__ == "__main__":
 
 # - [x] Finish off scraping for section that I need to finish
 # - [in progress] Pydantic format for deemed report
-# - [] Generate deemed report
+# - [] Generate deemed report -> scopre of automations thats possible
 # - [] Docker compose to an sql database in docker compose (2 hours, then work on sql)
 # - [] Deploy via terraform to aws (1 day)

etl/pdfReader/sitenotes.py

@@ -3,7 +3,8 @@ from transform.types import (
     CompanyInfo, SurverySummaryInfo, AssessorInfo,
     PropertyDescription, PropertyDetail, Dimension,
     Walls, Roofs, Floors, Door, VentilationAndCooling,
-    Lighting, WaterHeating, HotWaterCylinder,
+    Lighting, WaterHeating, HotWaterCylinder, SolarWaterHeating,
+    ShowerAndBaths, FlueGasHeatRecoverySystem,
 )
 from datetime import datetime
 
@@ -48,9 +49,6 @@ class QuidosSiteNotesExtractor(SiteNotesExtractor):
 
 
         # These one are quick fixes can be done on train or one at a time
-        # self.get_section_16()
-        # self.get_section_17()
-        # self.get_section_18()
         # self.get_section_19()
         # self.get_section_20()
         # self.get_section_21()
@@ -221,6 +219,18 @@ class QuidosSiteNotesExtractor(SiteNotesExtractor):
         # Section 15.1 Hot Water Cylinder
         hotWaterCylinder = self.get_hot_water_cylinder()
 
+        # Section 16 Solar Water Heating
+        solarWaterHeating = self.get_solar_water_heating()
+
+        # Section 17.0
+        # ignored as it has nothing in the copy i'm using. Future todo clarity
+
+        # Section 18.0
+        showerAndBaths = self.get_shower_and_baths()
+
+        # Section 19.0
+        fghrs = self.get_fghrs()
+
         self.property_description = PropertyDescription(
             built_form = get_value("Built Form"),
             detachment_or_position = get_value("Detachment/Position"),
@@ -269,6 +279,9 @@ class QuidosSiteNotesExtractor(SiteNotesExtractor):
             lighting=lighting,
             waterHeating=waterHeating,
             hotWaterCylinder=hotWaterCylinder,
+            solarWaterHeating=solarWaterHeating,
+            showerAndBaths=showerAndBaths,
+            flueGasHeatRecoverySystem=fghrs,
         )
 
 
@@ -407,8 +420,6 @@ class QuidosSiteNotesExtractor(SiteNotesExtractor):
         return walls
 
 
-
-
     def get_roof(self):
         data = self.raw_data[self.raw_data.index('8.0 Roofs'): self.raw_data.index('9.0 Floors')]
         sub_titles = [
@@ -725,6 +736,7 @@ class QuidosSiteNotesExtractor(SiteNotesExtractor):
             "15.1 Hot Water Cylinder"
         ]
         dict_ = self.two_columns_processor(data, sub_tites, avoid)
+
         return HotWaterCylinder(
             volume=dict_.get("volume", ""),
             insulation_type=dict_.get("insulation_type", ""),
@@ -732,7 +744,7 @@ class QuidosSiteNotesExtractor(SiteNotesExtractor):
             thermostat=True if dict_.get("thermostat", "NO").upper() == "YES" else False,
         )
 
-    def get_section_16(self):
+    def get_solar_water_heating(self):
         data = self.raw_data[self.raw_data.index("16.0 Solar Water Heating"):self.raw_data.index("17.0 Waste Water Heat Recovery System")]
         avoid = [
             "16.0 Solar Water Heating",
@@ -742,12 +754,19 @@ class QuidosSiteNotesExtractor(SiteNotesExtractor):
             "Solar Water Heating Details Known?"
         ]
 
-        self.two_columns_processor(data, sub_titles, avoid, 16.0)
+        dict_ = self.two_columns_processor(data, sub_titles, avoid)
+        
+        return SolarWaterHeating(
+            solar_water_heating_details_known=True if dict_.get("solar_water_heating_details_known", "NO").upper() == "YES" else False
+        )
     
     def get_section_17(self):
-        pass
+        """
+        Furture todo
+        """
+        raise NotImplemented("Please contact Jun-te Kim to implement this")
 
-    def get_section_18(self):
+    def get_shower_and_baths(self):
         data = self.get_data_between("18.0 Showers And Baths", "19.0 Flue Gas Heat Recovery System")
         sub_titles = [
             "Number of Rooms with Bath and/or Shower",
@@ -757,7 +776,8 @@ class QuidosSiteNotesExtractor(SiteNotesExtractor):
             "19.0 Flue Gas Heat Recovery System",
         ]
 
-        self.two_columns_processor(data, sub_titles, avoid, 18.0)
+        dict_1 = self.two_columns_processor(data, sub_titles, avoid)
+
         avoid = [
             "18.0 Showers And Baths",
             "19.0 Flue Gas Heat Recovery System",
@@ -766,9 +786,17 @@ class QuidosSiteNotesExtractor(SiteNotesExtractor):
             "Number of Rooms with Mixer Shower and no", # Number of Rooms with Mixer Shower and no Bath
             "Number of Rooms with Mixer Shower and", # Number of Rooms with Mixer Shower and Bath
         ]
-        self.two_columns_processor(data, sub_titles, avoid, 18.0, 2)
+        dict_2 = self.two_columns_processor(data, sub_titles, avoid, 2)
+
+        return ShowerAndBaths(
+            no_of_rooms_with_baths_and_or_shower=int(dict_1.get("number_of_rooms_with_bath_and/or_shower", -1)),
+            no_of_rooms_with_mixer_shower_and_no_baths=int(dict_2.get("number_of_rooms_with_mixer_shower_and_no", -1)),
+            no_of_rooms_with_mixer_shower_and_baths=int(dict_2.get("number_of_rooms_with_mixer_shower_and", -1)),
+        )
+
+
     
-    def get_section_19(self):
+    def get_fghrs(self):
         data = self.get_data_between("19.0 Flue Gas Heat Recovery System","20.0 Photovoltaic Panel")
         sub_titles = [
             "FGHRS Present",
@@ -778,7 +806,12 @@ class QuidosSiteNotesExtractor(SiteNotesExtractor):
             "20.0 Photovoltaic Panel",
         ]
 
-        self.two_columns_processor(data, sub_titles, avoid, 19)
+        dict_ = self.two_columns_processor(data, sub_titles, avoid)
+
+        return FlueGasHeatRecoverySystem(
+            fghrs_present=True if dict_.get("fghrs_present", "NO").upper() == "YES" else False
+        )
+
 
     def get_section_20(self):
         data = self.get_data_between("20.0 Photovoltaic Panel","21.0 Wind Turbine")

etl/scraper/scraper.py

@@ -9,7 +9,7 @@ from functools import wraps
 import re
 from etl.validator.validator import DomnaSharePointValidator
 
-WEEK_COMMENCING = os.getenv("WEEK_COMMENCING", "W.C. 03.03.2025")
+WEEK_COMMENCING = os.getenv("WEEK_COMMENCING", "W.C. 10.03.2025")
 
 class SharePointInstaller(Enum):
     SOUTH_COAST_INSULATION = os.getenv("SOUTH_COAST_INSULATION_SERVICE_SHAREPOINT_ID", None)

etl/transform/types.py

@@ -120,6 +120,18 @@ class HotWaterCylinder(BaseModel):
     insulation_thickness: str
     thermostat: bool
 
+class SolarWaterHeating(BaseModel):
+    solar_water_heating_details_known: bool
+
+class ShowerAndBaths(BaseModel):
+    no_of_rooms_with_baths_and_or_shower: int
+    no_of_rooms_with_mixer_shower_and_no_baths: int
+    no_of_rooms_with_mixer_shower_and_baths: int
+
+class FlueGasHeatRecoverySystem(BaseModel):
+    fghrs_present: bool
+
+
 class PropertyDetail(BaseModel):
     age_band: str
     dimensions: List[Dimension] = []
@@ -151,4 +163,7 @@ class PropertyDescription(BaseModel):
     ventilationAndCooling: Optional[VentilationAndCooling]
     lighting: Optional[Lighting]
     waterHeating: Optional[WaterHeating]
-    hotWaterCylinder: Optional[HotWaterCylinder]
+    hotWaterCylinder: Optional[HotWaterCylinder]
+    solarWaterHeating: Optional[SolarWaterHeating]
+    showerAndBaths: Optional[ShowerAndBaths]
+    flueGasHeatRecoverySystem: Optional[FlueGasHeatRecoverySystem]