diff --git a/etl/development.py b/etl/development.py index d6a1b02..58bfd6b 100644 --- a/etl/development.py +++ b/etl/development.py @@ -35,5 +35,8 @@ if __name__ == "__main__": # - [x] Finish off scraping for section that I need to finish # - [in progress] Pydantic format for deemed report # - [] Generate deemed report -> scopre of automations thats possible +# - [] Ensure lewis has sent me this week deemed score (invoice price for each job) +# - [] Ask lewis to put his rate card in sharepoint ( ask nick where the most logical place this will be) +# [] Ask kieran how some information is recieved from survery to submissions # - [] Docker compose to an sql database in docker compose (2 hours, then work on sql) # - [] Deploy via terraform to aws (1 day) \ No newline at end of file diff --git a/etl/pdfReader/sitenotes.py b/etl/pdfReader/sitenotes.py index 78ec346..d53764f 100644 --- a/etl/pdfReader/sitenotes.py +++ b/etl/pdfReader/sitenotes.py @@ -4,7 +4,9 @@ from transform.types import ( PropertyDescription, PropertyDetail, Dimension, Walls, Roofs, Floors, Door, VentilationAndCooling, Lighting, WaterHeating, HotWaterCylinder, SolarWaterHeating, - ShowerAndBaths, FlueGasHeatRecoverySystem, + ShowerAndBaths, FlueGasHeatRecoverySystem, PhotovoltaicPanel, + WindTurbine, OtherDetails, Windows, Heating, HeatingSystemControls, + HeatingType ) from datetime import datetime @@ -39,21 +41,12 @@ class QuidosSiteNotesExtractor(SiteNotesExtractor): self.transform_summary_information() self.transform_sections() - # Saves windows till the end as that requires thought - # self.get_section_11() - # Heaters require thought so will complete later # self.get_section_14() # self.get_section_14_1() # self.get_section_14_2() - # These one are quick fixes can be done on train or one at a time - # self.get_section_19() - # self.get_section_20() - # self.get_section_21() - # self.get_section_22() - def transform_summary_information(self): # Summary Information avoid = [ @@ -207,12 +200,24 @@ class QuidosSiteNotesExtractor(SiteNotesExtractor): # Section 10 door = self.get_door() + windows = self.get_windows() + print(windows["main_property"]) + # Section 12 ventilationAndCooling = self.get_ventilation_and_cooling() # Section 13 lighting = self.get_lighting() + # Section 14.0 + main_heating = self.get_main_heating() + + # Section 14.1 + secondary_heating = self.get_secondary_heating() + + # Section 14.2 + secondary_heating_type = self.get_secondary_heating_type() + # Section 15.0 Water Heating waterHeating = self.get_water_heating() @@ -231,6 +236,15 @@ class QuidosSiteNotesExtractor(SiteNotesExtractor): # Section 19.0 fghrs = self.get_fghrs() + # Section 20.0 + photovoltaicPanel = self.get_photovoltaic_panel() + + # Section 21.0 + windTurbine = self.get_wind_turbine() + + # Section 22.0 + otherDetails = self.get_other_details() + self.property_description = PropertyDescription( built_form = get_value("Built Form"), detachment_or_position = get_value("Detachment/Position"), @@ -251,6 +265,7 @@ class QuidosSiteNotesExtractor(SiteNotesExtractor): wall=walls[0], roof=roofs[0], floor=floors[0], + windows=windows.get("main_property", []), )if no_of_main_property > 0 else None, ex1_property=PropertyDetail( age_band= age_bands[1], @@ -258,6 +273,7 @@ class QuidosSiteNotesExtractor(SiteNotesExtractor): wall=walls[1], roof=roofs[1], floor=floors[1], + windows=windows.get("extension_1", []), )if no_of_extension_1 > 0 else None, ex2_property=PropertyDetail( age_band= age_bands[2], @@ -265,6 +281,7 @@ class QuidosSiteNotesExtractor(SiteNotesExtractor): wall=walls[2], roof=roofs[2], floor=floors[2], + windows=windows.get("extension_2", []), )if no_of_extension_2 > 0 else None, ex3_property=PropertyDetail( age_band= age_bands[3], @@ -272,6 +289,7 @@ class QuidosSiteNotesExtractor(SiteNotesExtractor): wall=walls[3], roof=roofs[3], floor=floors[3], + windows=windows.get("extension_3", []), )if no_of_extension_4 > 0 else None, conservatory=conservatory, door=door, @@ -282,6 +300,12 @@ class QuidosSiteNotesExtractor(SiteNotesExtractor): solarWaterHeating=solarWaterHeating, showerAndBaths=showerAndBaths, flueGasHeatRecoverySystem=fghrs, + photovoltaicPanel=photovoltaicPanel, + windTurbine=windTurbine, + otherDetails=otherDetails, + mainHeating=main_heating, + secondaryHeatingType=secondary_heating_type, + mainHeating2=secondary_heating, ) @@ -551,7 +575,7 @@ class QuidosSiteNotesExtractor(SiteNotesExtractor): dict_.update({f"{items.lower().replace('-', '_').replace(' ','_')}":get_value(items)}) return dict_ - def get_section_11(self): + def get_windows(self): data = self.get_data_between("Window Location", "12.0 Ventilation & Cooling") headers = data[:8] data_entries = data[8:] @@ -593,25 +617,26 @@ class QuidosSiteNotesExtractor(SiteNotesExtractor): title = None until = 0 + dict_to_return = {} for i, items in enumerate(data_entries): if data_entries[i] in subtitles: title = data_entries[i].lower().replace(" ", "_").replace("-", "_") - setattr(self, f"section_11_{title}_window", []) + dict_to_return.update({f"{title}":[]}) if title and until == i: entry = data_entries[i:] index = find_compose_index(entry,orientation) new_entry = entry[index-3:index+3] - dict_ = { - "glazing type": new_entry[0], - "Area (m2)": new_entry[1], - "Roof Window": new_entry[2], - "Orientation": new_entry[3], - "U-value (W/m²K)": new_entry[4], - "g-value": new_entry[5], - } - lst = getattr(self, f"section_11_{title}_window") - lst.append(dict_) + window = Windows( + glazing_type= new_entry[0], + area_m2=float(new_entry[1]), + roof_window=True if new_entry[2].upper() == "YES" else False, + orientation=new_entry[3].upper(), + u_value_w_m2_k=int(new_entry[4]), + g_value=int(new_entry[5]), + ) + dict_to_return[f"{title}"].append(window) until = index + 3 + i + return dict_to_return def get_ventilation_and_cooling(self): data = self.raw_data[self.raw_data.index('12.0 Ventilation & Cooling'): self.raw_data.index('13.0 Lighting')] @@ -643,13 +668,12 @@ class QuidosSiteNotesExtractor(SiteNotesExtractor): "Total number of L.E.L. fittings", ] dict_ = self.two_columns_processor(data, sub_titles, avoid = avoid) - print(dict_) return Lighting( total_no_of_light_fittings=int(dict_["total_number_of_light_fittings"]), total_no_of_lel_fittings=int(dict_["total_number_of_l.e.l._fittings"]), ) - def get_section_14(self): + def get_main_heating(self): data = self.raw_data[self.raw_data.index('14.0 Main Heating1'): self.raw_data.index('14.1 Main Heating2')] main_titles = [ "Main Heating Type", @@ -671,9 +695,27 @@ class QuidosSiteNotesExtractor(SiteNotesExtractor): "Mains Gas Available", ] - self.two_column_with_extension_processor(data, sub_titles, main_titles, 14.0) + lst_ = self.two_column_with_extension_processor(data, sub_titles, main_titles) + dict_ = lst_[0] + return Heating( + type="main", + heating_source=dict_.get("heating_source", ""), + efficiency_source=dict_.get("efficiency_source", ""), + heating_fuel=dict_.get("heating_fuel", ""), + brand_name=dict_.get("brand_name", ""), + model_name=dict_.get("model_name", ""), + model_qualifer=dict_.get("model_qualifier", ""), + controls=HeatingSystemControls( + control_type=dict_.get("control_type", ""), + flue_type=dict_.get("flue_type",""), + fan_assisted_flue=True if dict_.get("fan_assisted_flue", "NO").upper() == "YES" else False, + heat_emitter_type=dict_.get("heat_emitter_type", ""), + electricity_meter_type=dict_.get("electricity_meter_type", ""), + mains_gas_available=True if dict_.get("mains_gas_available", "NO").upper() == "YES" else False, + ) + ) - def get_section_14_1(self): + def get_secondary_heating(self): data = self.raw_data[self.raw_data.index("14.1 Main Heating2"):self.raw_data.index("14.2 Secondary Heating Type")] main_titles = [ "Second Main Heating Type", @@ -692,9 +734,29 @@ class QuidosSiteNotesExtractor(SiteNotesExtractor): "Fan Assisted Flue", "Heat Emitter Type", ] - self.two_column_with_extension_processor(data, sub_titles, main_titles, 14.1) + list = self.two_column_with_extension_processor(data, sub_titles, main_titles) + dict_ = list[0] + + return Heating( + type="secondary", + percentage_of_heated_floor_area_served=dict_.get("percentage_of_heated_floor_area_served_(%)", ""), + heating_source=dict_.get("heating_source", "") if dict_["heating_source"] is not None else "", + efficiency_source=dict_.get("efficiency_source", "") if dict_["efficiency_source"] is not None else "", + heating_fuel=dict_.get("heating_fuel", "") if dict_.get("heating_fuel") is not None else "", + brand_name=dict_.get("brand_name", ""), + model_name=dict_.get("model_name", ""), + model_qualifer=dict_.get("model_qualifier", ""), + controls=HeatingSystemControls( + control_type=dict_.get("control_type", ""), + flue_type=dict_.get("flue_type",""), + fan_assisted_flue=True if dict_.get("fan_assisted_flue", "NO").upper() == "YES" else False, + heat_emitter_type=dict_.get("heat_emitter_type", ""), + electricity_meter_type=dict_.get("electricity_meter_type", ""), + mains_gas_available=True if dict_.get("mains_gas_available", "NO").upper() == "YES" else False, + ) + ) - def get_section_14_2(self): + def get_secondary_heating_type(self): data = self.raw_data[self.raw_data.index("14.2 Secondary Heating Type"):self.raw_data.index("15.0 Water Heating")] avoid = [ "14.2 Secondary Heating Type", @@ -704,7 +766,11 @@ class QuidosSiteNotesExtractor(SiteNotesExtractor): "Heating Type", "Fuel Type", ] - self.two_columns_processor(data, sub_titles, avoid, 14.2) + dict_ = self.two_columns_processor(data, sub_titles, avoid) + return HeatingType( + heating_type=dict_.get("heating_type", ""), + fuel_type=dict_.get("fuel_type", ""), + ) def get_water_heating(self): data = self.raw_data[self.raw_data.index("15.0 Water Heating"):self.raw_data.index("15.1 Hot Water Cylinder")] @@ -813,7 +879,7 @@ class QuidosSiteNotesExtractor(SiteNotesExtractor): ) - def get_section_20(self): + def get_photovoltaic_panel(self): data = self.get_data_between("20.0 Photovoltaic Panel","21.0 Wind Turbine") sub_titles = [ "Percentage of External Roof Area with PVs" @@ -822,14 +888,18 @@ class QuidosSiteNotesExtractor(SiteNotesExtractor): "20.0 Photovoltaic Panel", "21.0 Wind Turbine", ] - self.two_columns_processor(data, sub_titles, avoid, 20) + dict_1 = self.two_columns_processor(data, sub_titles, avoid) sub_titles = [ "PVs are connected to dwelling electricity" # PVs are connected to dwelling electricity meter ] - self.two_columns_processor(data, sub_titles, avoid, 20, 2) + dict_2 = self.two_columns_processor(data, sub_titles, avoid, 2) + return PhotovoltaicPanel( + pvs_are_connected_to_dwelling_electricity_meter=True if dict_2.get("pvs_are_connected_to_dwelling_electricity", "NO").upper() == "YES" else False, + percentage_of_external_roof_area_with_pvs=dict_1.get("percentage_of_external_roof_area_with_pvs") + ) - def get_section_21(self): + def get_wind_turbine(self): data = self.get_data_between("21.0 Wind Turbine","22.0 Other Details") sub_titles = [ "Wind Turbine", @@ -838,9 +908,12 @@ class QuidosSiteNotesExtractor(SiteNotesExtractor): "21.0 Wind Turbine", "22.0 Other Details", ] - self.two_columns_processor(data, sub_titles, avoid, 21) + dict_ = self.two_columns_processor(data, sub_titles, avoid) + return WindTurbine( + wind_turbine=True if dict_.get("wind_turbine", "NO").upper()=="YES" else False + ) - def get_section_22(self): + def get_other_details(self): data = self.get_data_between("22.0 Other Details","Recommendations (Carbon Saving Figures Are For Guidance Only)") sub_titles = [ "Electricity Meter Type", @@ -851,10 +924,11 @@ class QuidosSiteNotesExtractor(SiteNotesExtractor): "Recommendations (Carbon Saving Figures Are For Guidance Only)", ] - self.two_columns_processor(data, sub_titles, avoid, 22) + dict_ = self.two_columns_processor(data, sub_titles, avoid) + return OtherDetails( + electricity_meter_type=dict_.get("electricity_meter_type", ""), + main_gas_avalible=True if dict_.get("main_gas_available", "NO").upper() == "YES" else False, + ) -# Section and 11 -# Extract -# Transform ( wiht validation pydantnic) -# Load \ No newline at end of file + \ No newline at end of file diff --git a/etl/scraper/scraper.py b/etl/scraper/scraper.py index 0fb83a8..06dc26e 100644 --- a/etl/scraper/scraper.py +++ b/etl/scraper/scraper.py @@ -9,7 +9,14 @@ from functools import wraps import re from etl.validator.validator import DomnaSharePointValidator -WEEK_COMMENCING = os.getenv("WEEK_COMMENCING", "W.C. 10.03.2025") +from datetime import datetime, timedelta + +def previous_monday(): + today = datetime.today() + last_monday = today - timedelta(days=today.weekday() + 7) # Go back to last week's Monday + return f"W.C. {last_monday.strftime('%d.%m.%Y')}" + +WEEK_COMMENCING = os.getenv("WEEK_COMMENCING", previous_monday()) class SharePointInstaller(Enum): SOUTH_COAST_INSULATION = os.getenv("SOUTH_COAST_INSULATION_SERVICE_SHAREPOINT_ID", None) diff --git a/etl/transform/types.py b/etl/transform/types.py index c48db71..cefb894 100644 --- a/etl/transform/types.py +++ b/etl/transform/types.py @@ -76,14 +76,6 @@ class AssessorInfo(BaseModel): phone_number: Optional[str] = None email_address: Optional[EmailStr] = None -class Windows(BaseModel): - glazing_type: str - area_m2: float - roof_window: bool - orientation: str - u_value_w_m2_k: int - g_value: int - class VentilationAndCooling(BaseModel): no_of_open_fireplaces: int ventilation_type: str @@ -93,22 +85,29 @@ class Lighting(BaseModel): total_no_of_light_fittings: int total_no_of_lel_fittings: int -class MainHeatingSystemControls(BaseModel): +class HeatingSystemControls(BaseModel): control_type: str flue_type: str - fan_assisted_flue: str + fan_assisted_flue: bool heat_emitter_type: str - electricity_meter_type: Optional[str] - mains_gas_available: Optional[bool] + electricity_meter_type: Optional[str] = "" + mains_gas_available: Optional[bool] = False -class MainHeating(BaseModel): +class Heating(BaseModel): + type: str heating_source: str efficiency_source: str heating_fuel: str brand_name: str model_name: str model_qualifer: str - controls: MainHeatingSystemControls + sap_2009_table: Optional[str] = "" + percentage_of_heated_floor_area_served: Optional[str] = "" + controls: HeatingSystemControls + +class HeatingType(BaseModel): + heating_type: str + fuel_type: str class WaterHeating(BaseModel): heating_type: str @@ -131,6 +130,24 @@ class ShowerAndBaths(BaseModel): class FlueGasHeatRecoverySystem(BaseModel): fghrs_present: bool +class PhotovoltaicPanel(BaseModel): + pvs_are_connected_to_dwelling_electricity_meter: bool + percentage_of_external_roof_area_with_pvs: str + +class WindTurbine(BaseModel): + wind_turbine: bool + +class OtherDetails(BaseModel): + electricity_meter_type: str + main_gas_avalible: bool + +class Windows(BaseModel): + glazing_type: str + area_m2: float + roof_window: bool + orientation: str + u_value_w_m2_k: int + g_value: int class PropertyDetail(BaseModel): age_band: str @@ -138,6 +155,8 @@ class PropertyDetail(BaseModel): wall: Optional[Walls] = None roof: Optional[Roofs] = None floor: Optional[Floors] = None + windows: Optional[List[Windows]] = [] + class PropertyDescription(BaseModel): built_form: str @@ -167,3 +186,9 @@ class PropertyDescription(BaseModel): solarWaterHeating: Optional[SolarWaterHeating] showerAndBaths: Optional[ShowerAndBaths] flueGasHeatRecoverySystem: Optional[FlueGasHeatRecoverySystem] + photovoltaicPanel: Optional[PhotovoltaicPanel] + windTurbine: Optional[WindTurbine] + otherDetails: Optional[OtherDetails] + mainHeating: Optional[Heating] + mainHeating2: Optional[Heating] + secondaryHeatingType: Optional[HeatingType]