diff --git a/.idea/Model.iml b/.idea/Model.iml index 4413bb06..b0f9c00d 100644 --- a/.idea/Model.iml +++ b/.idea/Model.iml @@ -7,7 +7,7 @@ - + diff --git a/.idea/misc.xml b/.idea/misc.xml index 6f308057..1122b380 100644 --- a/.idea/misc.xml +++ b/.idea/misc.xml @@ -3,7 +3,7 @@ - + diff --git a/backend/apis/GoogleSolarApi.py b/backend/apis/GoogleSolarApi.py new file mode 100644 index 00000000..86324c58 --- /dev/null +++ b/backend/apis/GoogleSolarApi.py @@ -0,0 +1,334 @@ +from backend.Property import Property +from backend.SearchEpc import SearchEpc +from etl.epc.Record import EPCRecord +from dotenv import load_dotenv +from utils.s3 import read_dataframe_from_s3_parquet +import os +import requests + +load_dotenv(dotenv_path="backend/.env") +EPC_AUTH_TOKEN = os.getenv("EPC_AUTH_TOKEN") + +# This is for 6 Laura Close, Tintagel, PL34 0EB (same property that Cotswolrd energy used) +uprn = 100040099104 + +cleaning_data = read_dataframe_from_s3_parquet( + bucket_name="retrofit-data-dev", file_key="sap_change_model/cleaning_dataset.parquet", +) + +searcher = SearchEpc(address1="6 Laura Close", postcode="PL34 0EB", uprn=uprn, auth_token=EPC_AUTH_TOKEN, os_api_key="") + +searcher.find_property(skip_os=True) + +epc_records = { + 'original_epc': searcher.newest_epc.copy(), + 'full_sap_epc': searcher.full_sap_epc.copy(), + 'old_data': searcher.older_epcs.copy(), +} + +epc = EPCRecord( + epc_records=epc_records, + run_mode="newdata", + cleaning_data=cleaning_data +) + +uprn_filenames = read_dataframe_from_s3_parquet( + bucket_name="retrofit-data-dev", file_key="spatial/filename_meta.parquet" +) + +p = Property( + id=0, + address=searcher.address_clean, + postcode=searcher.postcode_clean, + epc_record=epc, + already_installed={}, + non_invasive_recommendations={}, +) + +p.get_spatial_data(uprn_filenames) + +longitude = p.spatial["longitude"] +latitude = p.spatial["latitude"] + +api_key = "AIzaSyCIz8Psu5h-1txuDX0rQpUTgkvdj8yohqU" +url = 'https://solar.googleapis.com/v1/solarPotential' +params = { + 'location.latitude': f'{latitude:.5f}', + 'location.longitude': f'{longitude:.5f}', + 'requiredQuality': "MEDIUM", + 'key': api_key +} + +insights_url = 'https://solar.googleapis.com/v1/buildingInsights:findClosest' + +# Make the GET request to the Solar API +insights_response = requests.get(insights_url, params=params) +insights_data = insights_response.json() + +solar_potential = insights_data["solarPotential"] + +from pprint import pprint + +pprint(solar_potential) + +# This is the size of the panels used in the calculation - 400 watt +solar_potential["panelCapacityWatts"] +# Height of the panels used +solar_potential["panelHeightMeters"] +# Width of the panels used +solar_potential["panelWidthMeters"] + +solar_potential["wholeRoofStats"] + +# Copy of response for testing: +# {'name': 'buildings/ChIJ2yC6t4KEa0gRh2TIssogI7k', 'center': {'latitude': 50.667375, 'longitude': -4.7416833}, +# 'imageryDate': {'year': 2021, 'month': 7, 'day': 19}, 'regionCode': 'GB', 'solarPotential': {'maxArrayPanelsCount': +# 39, 'maxArrayAreaMeters2': 76.578636, 'maxSunshineHoursPerYear': 1172.0627, 'carbonOffsetFactorKgPerMwh': +# 478.99942, 'wholeRoofStats': {'areaMeters2': 129.65686, 'sunshineQuantiles': [537, 738.3836, 805.62445, 842.6802, +# 909.8431, 972.15234, 1036.1013, 1092.051, 1135.8192, 1163.1444, 1193.6012], 'groundAreaMeters2': 112.33}, +# 'roofSegmentStats': [{'pitchDegrees': 31.443022, 'azimuthDegrees': 218.25331, 'stats': {'areaMeters2': 44.08321, +# 'sunshineQuantiles': [614, 940.86975, 982.39124, 1057.0664, 1109.6869, 1137.5837, 1152.9211, 1163.1106, 1168.2212, +# 1170.8883, 1193.6012], 'groundAreaMeters2': 37.61}, 'center': {'latitude': 50.6673664, 'longitude': +# -4.741714099999999}, 'boundingBox': {'sw': {'latitude': 50.6673354, 'longitude': -4.741777}, 'ne': {'latitude': +# 50.6674029, 'longitude': -4.7416472}}, 'planeHeightAtCenterMeters': 93.0221}, {'pitchDegrees': 34.39779, +# 'azimuthDegrees': 31.74401, 'stats': {'areaMeters2': 44.622986, 'sunshineQuantiles': [537, 671.49774, 733.84985, +# 780.82733, 801.4026, 814.0189, 824.0077, 847.77484, 895.08295, 950.1469, 1123.3503], 'groundAreaMeters2': 36.82}, +# 'center': {'latitude': 50.6673966, 'longitude': -4.7416813}, 'boundingBox': {'sw': {'latitude': 50.667361, +# 'longitude': -4.7417497}, 'ne': {'latitude': 50.6674303, 'longitude': -4.741615599999999}}, +# 'planeHeightAtCenterMeters': 92.87593}, {'pitchDegrees': 3.0681775, 'azimuthDegrees': 301.1099, 'stats': { +# 'areaMeters2': 17.074476, 'sunshineQuantiles': [644.71136, 731.0546, 782.89813, 842.7107, 908.55585, 966.6212, +# 1010.6367, 1038.2543, 1053.2788, 1090.6831, 1128.0178], 'groundAreaMeters2': 17.050001}, 'center': {'latitude': +# 50.66740850000001, 'longitude': -4.7416025}, 'boundingBox': {'sw': {'latitude': 50.6673895, 'longitude': +# -4.7416436}, 'ne': {'latitude': 50.667431199999996, 'longitude': -4.7415572}}, 'planeHeightAtCenterMeters': +# 90.630356}, {'pitchDegrees': 27.093596, 'azimuthDegrees': 132.60162, 'stats': {'areaMeters2': 13.501617, +# 'sunshineQuantiles': [749, 976.85345, 1059.0062, 1081.6173, 1097.4441, 1110.3171, 1128.2186, 1133.9421, 1142.068, +# 1148.2168, 1157.632], 'groundAreaMeters2': 12.02}, 'center': {'latitude': 50.667315699999996, 'longitude': +# -4.741675400000001}, 'boundingBox': {'sw': {'latitude': 50.667291399999996, 'longitude': -4.7417066}, +# 'ne': {'latitude': 50.6673372, 'longitude': -4.741648400000001}}, 'planeHeightAtCenterMeters': 92.36334}, +# {'pitchDegrees': 31.666294, 'azimuthDegrees': 308.42334, 'stats': {'areaMeters2': 10.374564, 'sunshineQuantiles': [ +# 617.9507, 752.2504, 847.66315, 872.0505, 881.26227, 900.9639, 933.3188, 967.4747, 1000.8129, 1038.3002, 1105.545], +# 'groundAreaMeters2': 8.83}, 'center': {'latitude': 50.6673295, 'longitude': -4.7417128}, 'boundingBox': {'sw': { +# 'latitude': 50.6673134, 'longitude': -4.7417422}, 'ne': {'latitude': 50.6673413, 'longitude': -4.7416775}}, +# 'planeHeightAtCenterMeters': 92.31146}], 'solarPanelConfigs': [{'panelsCount': 4, 'yearlyEnergyDcKwh': 1867.1516, +# 'roofSegmentSummaries': [{'pitchDegrees': 31.443022, 'azimuthDegrees': 218.25331, 'panelsCount': 4, +# 'yearlyEnergyDcKwh': 1867.1515, 'segmentIndex': 0}]}, {'panelsCount': 5, 'yearlyEnergyDcKwh': 2335.0068, +# 'roofSegmentSummaries': [{'pitchDegrees': 31.443022, 'azimuthDegrees': 218.25331, 'panelsCount': 5, +# 'yearlyEnergyDcKwh': 2335.0068, 'segmentIndex': 0}]}, {'panelsCount': 6, 'yearlyEnergyDcKwh': 2799.8508, +# 'roofSegmentSummaries': [{'pitchDegrees': 31.443022, 'azimuthDegrees': 218.25331, 'panelsCount': 6, +# 'yearlyEnergyDcKwh': 2799.8508, 'segmentIndex': 0}]}, {'panelsCount': 7, 'yearlyEnergyDcKwh': 3264.6506, +# 'roofSegmentSummaries': [{'pitchDegrees': 31.443022, 'azimuthDegrees': 218.25331, 'panelsCount': 7, +# 'yearlyEnergyDcKwh': 3264.6506, 'segmentIndex': 0}]}, {'panelsCount': 8, 'yearlyEnergyDcKwh': 3726.2405, +# 'roofSegmentSummaries': [{'pitchDegrees': 31.443022, 'azimuthDegrees': 218.25331, 'panelsCount': 8, +# 'yearlyEnergyDcKwh': 3726.2405, 'segmentIndex': 0}]}, {'panelsCount': 9, 'yearlyEnergyDcKwh': 4187.721, +# 'roofSegmentSummaries': [{'pitchDegrees': 31.443022, 'azimuthDegrees': 218.25331, 'panelsCount': 9, +# 'yearlyEnergyDcKwh': 4187.721, 'segmentIndex': 0}]}, {'panelsCount': 10, 'yearlyEnergyDcKwh': 4646.094, +# 'roofSegmentSummaries': [{'pitchDegrees': 31.443022, 'azimuthDegrees': 218.25331, 'panelsCount': 10, +# 'yearlyEnergyDcKwh': 4646.094, 'segmentIndex': 0}]}, {'panelsCount': 11, 'yearlyEnergyDcKwh': 5103.777, +# 'roofSegmentSummaries': [{'pitchDegrees': 31.443022, 'azimuthDegrees': 218.25331, 'panelsCount': 10, +# 'yearlyEnergyDcKwh': 4646.094, 'segmentIndex': 0}, {'pitchDegrees': 27.093596, 'azimuthDegrees': 132.60162, +# 'panelsCount': 1, 'yearlyEnergyDcKwh': 457.68268, 'segmentIndex': 3}]}, {'panelsCount': 12, 'yearlyEnergyDcKwh': +# 5559.845, 'roofSegmentSummaries': [{'pitchDegrees': 31.443022, 'azimuthDegrees': 218.25331, 'panelsCount': 10, +# 'yearlyEnergyDcKwh': 4646.094, 'segmentIndex': 0}, {'pitchDegrees': 27.093596, 'azimuthDegrees': 132.60162, +# 'panelsCount': 2, 'yearlyEnergyDcKwh': 913.7509, 'segmentIndex': 3}]}, {'panelsCount': 13, 'yearlyEnergyDcKwh': +# 6013.053, 'roofSegmentSummaries': [{'pitchDegrees': 31.443022, 'azimuthDegrees': 218.25331, 'panelsCount': 11, +# 'yearlyEnergyDcKwh': 5099.302, 'segmentIndex': 0}, {'pitchDegrees': 27.093596, 'azimuthDegrees': 132.60162, +# 'panelsCount': 2, 'yearlyEnergyDcKwh': 913.7509, 'segmentIndex': 3}]}, {'panelsCount': 14, 'yearlyEnergyDcKwh': +# 6461.664, 'roofSegmentSummaries': [{'pitchDegrees': 31.443022, 'azimuthDegrees': 218.25331, 'panelsCount': 12, +# 'yearlyEnergyDcKwh': 5547.9126, 'segmentIndex': 0}, {'pitchDegrees': 27.093596, 'azimuthDegrees': 132.60162, +# 'panelsCount': 2, 'yearlyEnergyDcKwh': 913.7509, 'segmentIndex': 3}]}, {'panelsCount': 15, 'yearlyEnergyDcKwh': +# 6902.33, 'roofSegmentSummaries': [{'pitchDegrees': 31.443022, 'azimuthDegrees': 218.25331, 'panelsCount': 12, +# 'yearlyEnergyDcKwh': 5547.9126, 'segmentIndex': 0}, {'pitchDegrees': 27.093596, 'azimuthDegrees': 132.60162, +# 'panelsCount': 3, 'yearlyEnergyDcKwh': 1354.4171, 'segmentIndex': 3}]}, {'panelsCount': 16, 'yearlyEnergyDcKwh': +# 7321.6436, 'roofSegmentSummaries': [{'pitchDegrees': 31.443022, 'azimuthDegrees': 218.25331, 'panelsCount': 12, +# 'yearlyEnergyDcKwh': 5547.9126, 'segmentIndex': 0}, {'pitchDegrees': 3.0681775, 'azimuthDegrees': 301.1099, +# 'panelsCount': 1, 'yearlyEnergyDcKwh': 419.31348, 'segmentIndex': 2}, {'pitchDegrees': 27.093596, 'azimuthDegrees': +# 132.60162, 'panelsCount': 3, 'yearlyEnergyDcKwh': 1354.4171, 'segmentIndex': 3}]}, {'panelsCount': 17, +# 'yearlyEnergyDcKwh': 7740.388, 'roofSegmentSummaries': [{'pitchDegrees': 31.443022, 'azimuthDegrees': 218.25331, +# 'panelsCount': 12, 'yearlyEnergyDcKwh': 5547.9126, 'segmentIndex': 0}, {'pitchDegrees': 3.0681775, +# 'azimuthDegrees': 301.1099, 'panelsCount': 2, 'yearlyEnergyDcKwh': 838.0579, 'segmentIndex': 2}, {'pitchDegrees': +# 27.093596, 'azimuthDegrees': 132.60162, 'panelsCount': 3, 'yearlyEnergyDcKwh': 1354.4171, 'segmentIndex': 3}]}, +# {'panelsCount': 18, 'yearlyEnergyDcKwh': 8154.265, 'roofSegmentSummaries': [{'pitchDegrees': 31.443022, +# 'azimuthDegrees': 218.25331, 'panelsCount': 13, 'yearlyEnergyDcKwh': 5961.7896, 'segmentIndex': 0}, +# {'pitchDegrees': 3.0681775, 'azimuthDegrees': 301.1099, 'panelsCount': 2, 'yearlyEnergyDcKwh': 838.0579, +# 'segmentIndex': 2}, {'pitchDegrees': 27.093596, 'azimuthDegrees': 132.60162, 'panelsCount': 3, 'yearlyEnergyDcKwh': +# 1354.4171, 'segmentIndex': 3}]}, {'panelsCount': 19, 'yearlyEnergyDcKwh': 8566.032, 'roofSegmentSummaries': [{ +# 'pitchDegrees': 31.443022, 'azimuthDegrees': 218.25331, 'panelsCount': 14, 'yearlyEnergyDcKwh': 6373.556, +# 'segmentIndex': 0}, {'pitchDegrees': 3.0681775, 'azimuthDegrees': 301.1099, 'panelsCount': 2, 'yearlyEnergyDcKwh': +# 838.0579, 'segmentIndex': 2}, {'pitchDegrees': 27.093596, 'azimuthDegrees': 132.60162, 'panelsCount': 3, +# 'yearlyEnergyDcKwh': 1354.4171, 'segmentIndex': 3}]}, {'panelsCount': 20, 'yearlyEnergyDcKwh': 8976.624, +# 'roofSegmentSummaries': [{'pitchDegrees': 31.443022, 'azimuthDegrees': 218.25331, 'panelsCount': 15, +# 'yearlyEnergyDcKwh': 6784.1484, 'segmentIndex': 0}, {'pitchDegrees': 3.0681775, 'azimuthDegrees': 301.1099, +# 'panelsCount': 2, 'yearlyEnergyDcKwh': 838.0579, 'segmentIndex': 2}, {'pitchDegrees': 27.093596, 'azimuthDegrees': +# 132.60162, 'panelsCount': 3, 'yearlyEnergyDcKwh': 1354.4171, 'segmentIndex': 3}]}, {'panelsCount': 21, +# 'yearlyEnergyDcKwh': 9380.78, 'roofSegmentSummaries': [{'pitchDegrees': 31.443022, 'azimuthDegrees': 218.25331, +# 'panelsCount': 15, 'yearlyEnergyDcKwh': 6784.1484, 'segmentIndex': 0}, {'pitchDegrees': 3.0681775, +# 'azimuthDegrees': 301.1099, 'panelsCount': 3, 'yearlyEnergyDcKwh': 1242.214, 'segmentIndex': 2}, {'pitchDegrees': +# 27.093596, 'azimuthDegrees': 132.60162, 'panelsCount': 3, 'yearlyEnergyDcKwh': 1354.4171, 'segmentIndex': 3}]}, +# {'panelsCount': 22, 'yearlyEnergyDcKwh': 9784.078, 'roofSegmentSummaries': [{'pitchDegrees': 31.443022, +# 'azimuthDegrees': 218.25331, 'panelsCount': 15, 'yearlyEnergyDcKwh': 6784.1484, 'segmentIndex': 0}, +# {'pitchDegrees': 3.0681775, 'azimuthDegrees': 301.1099, 'panelsCount': 4, 'yearlyEnergyDcKwh': 1645.5122, +# 'segmentIndex': 2}, {'pitchDegrees': 27.093596, 'azimuthDegrees': 132.60162, 'panelsCount': 3, 'yearlyEnergyDcKwh': +# 1354.4171, 'segmentIndex': 3}]}, {'panelsCount': 23, 'yearlyEnergyDcKwh': 10162.354, 'roofSegmentSummaries': [{ +# 'pitchDegrees': 31.443022, 'azimuthDegrees': 218.25331, 'panelsCount': 15, 'yearlyEnergyDcKwh': 6784.1484, +# 'segmentIndex': 0}, {'pitchDegrees': 3.0681775, 'azimuthDegrees': 301.1099, 'panelsCount': 4, 'yearlyEnergyDcKwh': +# 1645.5122, 'segmentIndex': 2}, {'pitchDegrees': 27.093596, 'azimuthDegrees': 132.60162, 'panelsCount': 3, +# 'yearlyEnergyDcKwh': 1354.4171, 'segmentIndex': 3}, {'pitchDegrees': 31.666294, 'azimuthDegrees': 308.42334, +# 'panelsCount': 1, 'yearlyEnergyDcKwh': 378.2754, 'segmentIndex': 4}]}, {'panelsCount': 24, 'yearlyEnergyDcKwh': +# 10535.894, 'roofSegmentSummaries': [{'pitchDegrees': 31.443022, 'azimuthDegrees': 218.25331, 'panelsCount': 15, +# 'yearlyEnergyDcKwh': 6784.1484, 'segmentIndex': 0}, {'pitchDegrees': 3.0681775, 'azimuthDegrees': 301.1099, +# 'panelsCount': 5, 'yearlyEnergyDcKwh': 2019.0519, 'segmentIndex': 2}, {'pitchDegrees': 27.093596, 'azimuthDegrees': +# 132.60162, 'panelsCount': 3, 'yearlyEnergyDcKwh': 1354.4171, 'segmentIndex': 3}, {'pitchDegrees': 31.666294, +# 'azimuthDegrees': 308.42334, 'panelsCount': 1, 'yearlyEnergyDcKwh': 378.2754, 'segmentIndex': 4}]}, {'panelsCount': +# 25, 'yearlyEnergyDcKwh': 10901.273, 'roofSegmentSummaries': [{'pitchDegrees': 31.443022, 'azimuthDegrees': +# 218.25331, 'panelsCount': 15, 'yearlyEnergyDcKwh': 6784.1484, 'segmentIndex': 0}, {'pitchDegrees': 3.0681775, +# 'azimuthDegrees': 301.1099, 'panelsCount': 5, 'yearlyEnergyDcKwh': 2019.0519, 'segmentIndex': 2}, {'pitchDegrees': +# 27.093596, 'azimuthDegrees': 132.60162, 'panelsCount': 3, 'yearlyEnergyDcKwh': 1354.4171, 'segmentIndex': 3}, +# {'pitchDegrees': 31.666294, 'azimuthDegrees': 308.42334, 'panelsCount': 2, 'yearlyEnergyDcKwh': 743.65497, +# 'segmentIndex': 4}]}, {'panelsCount': 26, 'yearlyEnergyDcKwh': 11242.756, 'roofSegmentSummaries': [{'pitchDegrees': +# 31.443022, 'azimuthDegrees': 218.25331, 'panelsCount': 15, 'yearlyEnergyDcKwh': 6784.1484, 'segmentIndex': 0}, +# {'pitchDegrees': 34.39779, 'azimuthDegrees': 31.74401, 'panelsCount': 1, 'yearlyEnergyDcKwh': 341.4827, +# 'segmentIndex': 1}, {'pitchDegrees': 3.0681775, 'azimuthDegrees': 301.1099, 'panelsCount': 5, 'yearlyEnergyDcKwh': +# 2019.0519, 'segmentIndex': 2}, {'pitchDegrees': 27.093596, 'azimuthDegrees': 132.60162, 'panelsCount': 3, +# 'yearlyEnergyDcKwh': 1354.4171, 'segmentIndex': 3}, {'pitchDegrees': 31.666294, 'azimuthDegrees': 308.42334, +# 'panelsCount': 2, 'yearlyEnergyDcKwh': 743.65497, 'segmentIndex': 4}]}, {'panelsCount': 27, 'yearlyEnergyDcKwh': +# 11579.401, 'roofSegmentSummaries': [{'pitchDegrees': 31.443022, 'azimuthDegrees': 218.25331, 'panelsCount': 15, +# 'yearlyEnergyDcKwh': 6784.1484, 'segmentIndex': 0}, {'pitchDegrees': 34.39779, 'azimuthDegrees': 31.74401, +# 'panelsCount': 2, 'yearlyEnergyDcKwh': 678.1277, 'segmentIndex': 1}, {'pitchDegrees': 3.0681775, 'azimuthDegrees': +# 301.1099, 'panelsCount': 5, 'yearlyEnergyDcKwh': 2019.0519, 'segmentIndex': 2}, {'pitchDegrees': 27.093596, +# 'azimuthDegrees': 132.60162, 'panelsCount': 3, 'yearlyEnergyDcKwh': 1354.4171, 'segmentIndex': 3}, {'pitchDegrees': +# 31.666294, 'azimuthDegrees': 308.42334, 'panelsCount': 2, 'yearlyEnergyDcKwh': 743.65497, 'segmentIndex': 4}]}, +# {'panelsCount': 28, 'yearlyEnergyDcKwh': 11919.106, 'roofSegmentSummaries': [{'pitchDegrees': 31.443022, +# 'azimuthDegrees': 218.25331, 'panelsCount': 15, 'yearlyEnergyDcKwh': 6784.1484, 'segmentIndex': 0}, +# {'pitchDegrees': 34.39779, 'azimuthDegrees': 31.74401, 'panelsCount': 3, 'yearlyEnergyDcKwh': 1017.83356, +# 'segmentIndex': 1}, {'pitchDegrees': 3.0681775, 'azimuthDegrees': 301.1099, 'panelsCount': 5, 'yearlyEnergyDcKwh': +# 2019.0519, 'segmentIndex': 2}, {'pitchDegrees': 27.093596, 'azimuthDegrees': 132.60162, 'panelsCount': 3, +# 'yearlyEnergyDcKwh': 1354.4171, 'segmentIndex': 3}, {'pitchDegrees': 31.666294, 'azimuthDegrees': 308.42334, +# 'panelsCount': 2, 'yearlyEnergyDcKwh': 743.65497, 'segmentIndex': 4}]}, {'panelsCount': 29, 'yearlyEnergyDcKwh': +# 12255.358, 'roofSegmentSummaries': [{'pitchDegrees': 31.443022, 'azimuthDegrees': 218.25331, 'panelsCount': 15, +# 'yearlyEnergyDcKwh': 6784.1484, 'segmentIndex': 0}, {'pitchDegrees': 34.39779, 'azimuthDegrees': 31.74401, +# 'panelsCount': 4, 'yearlyEnergyDcKwh': 1354.0854, 'segmentIndex': 1}, {'pitchDegrees': 3.0681775, 'azimuthDegrees': +# 301.1099, 'panelsCount': 5, 'yearlyEnergyDcKwh': 2019.0519, 'segmentIndex': 2}, {'pitchDegrees': 27.093596, +# 'azimuthDegrees': 132.60162, 'panelsCount': 3, 'yearlyEnergyDcKwh': 1354.4171, 'segmentIndex': 3}, {'pitchDegrees': +# 31.666294, 'azimuthDegrees': 308.42334, 'panelsCount': 2, 'yearlyEnergyDcKwh': 743.65497, 'segmentIndex': 4}]}, +# {'panelsCount': 30, 'yearlyEnergyDcKwh': 12586.448, 'roofSegmentSummaries': [{'pitchDegrees': 31.443022, +# 'azimuthDegrees': 218.25331, 'panelsCount': 15, 'yearlyEnergyDcKwh': 6784.1484, 'segmentIndex': 0}, +# {'pitchDegrees': 34.39779, 'azimuthDegrees': 31.74401, 'panelsCount': 5, 'yearlyEnergyDcKwh': 1685.1748, +# 'segmentIndex': 1}, {'pitchDegrees': 3.0681775, 'azimuthDegrees': 301.1099, 'panelsCount': 5, 'yearlyEnergyDcKwh': +# 2019.0519, 'segmentIndex': 2}, {'pitchDegrees': 27.093596, 'azimuthDegrees': 132.60162, 'panelsCount': 3, +# 'yearlyEnergyDcKwh': 1354.4171, 'segmentIndex': 3}, {'pitchDegrees': 31.666294, 'azimuthDegrees': 308.42334, +# 'panelsCount': 2, 'yearlyEnergyDcKwh': 743.65497, 'segmentIndex': 4}]}, {'panelsCount': 31, 'yearlyEnergyDcKwh': +# 12911.502, 'roofSegmentSummaries': [{'pitchDegrees': 31.443022, 'azimuthDegrees': 218.25331, 'panelsCount': 15, +# 'yearlyEnergyDcKwh': 6784.1484, 'segmentIndex': 0}, {'pitchDegrees': 34.39779, 'azimuthDegrees': 31.74401, +# 'panelsCount': 6, 'yearlyEnergyDcKwh': 2010.2289, 'segmentIndex': 1}, {'pitchDegrees': 3.0681775, 'azimuthDegrees': +# 301.1099, 'panelsCount': 5, 'yearlyEnergyDcKwh': 2019.0519, 'segmentIndex': 2}, {'pitchDegrees': 27.093596, +# 'azimuthDegrees': 132.60162, 'panelsCount': 3, 'yearlyEnergyDcKwh': 1354.4171, 'segmentIndex': 3}, {'pitchDegrees': +# 31.666294, 'azimuthDegrees': 308.42334, 'panelsCount': 2, 'yearlyEnergyDcKwh': 743.65497, 'segmentIndex': 4}]}, +# {'panelsCount': 32, 'yearlyEnergyDcKwh': 13233.139, 'roofSegmentSummaries': [{'pitchDegrees': 31.443022, +# 'azimuthDegrees': 218.25331, 'panelsCount': 15, 'yearlyEnergyDcKwh': 6784.1484, 'segmentIndex': 0}, +# {'pitchDegrees': 34.39779, 'azimuthDegrees': 31.74401, 'panelsCount': 7, 'yearlyEnergyDcKwh': 2331.8652, +# 'segmentIndex': 1}, {'pitchDegrees': 3.0681775, 'azimuthDegrees': 301.1099, 'panelsCount': 5, 'yearlyEnergyDcKwh': +# 2019.0519, 'segmentIndex': 2}, {'pitchDegrees': 27.093596, 'azimuthDegrees': 132.60162, 'panelsCount': 3, +# 'yearlyEnergyDcKwh': 1354.4171, 'segmentIndex': 3}, {'pitchDegrees': 31.666294, 'azimuthDegrees': 308.42334, +# 'panelsCount': 2, 'yearlyEnergyDcKwh': 743.65497, 'segmentIndex': 4}]}, {'panelsCount': 33, 'yearlyEnergyDcKwh': +# 13554.602, 'roofSegmentSummaries': [{'pitchDegrees': 31.443022, 'azimuthDegrees': 218.25331, 'panelsCount': 15, +# 'yearlyEnergyDcKwh': 6784.1484, 'segmentIndex': 0}, {'pitchDegrees': 34.39779, 'azimuthDegrees': 31.74401, +# 'panelsCount': 8, 'yearlyEnergyDcKwh': 2653.3286, 'segmentIndex': 1}, {'pitchDegrees': 3.0681775, 'azimuthDegrees': +# 301.1099, 'panelsCount': 5, 'yearlyEnergyDcKwh': 2019.0519, 'segmentIndex': 2}, {'pitchDegrees': 27.093596, +# 'azimuthDegrees': 132.60162, 'panelsCount': 3, 'yearlyEnergyDcKwh': 1354.4171, 'segmentIndex': 3}, {'pitchDegrees': +# 31.666294, 'azimuthDegrees': 308.42334, 'panelsCount': 2, 'yearlyEnergyDcKwh': 743.65497, 'segmentIndex': 4}]}, +# {'panelsCount': 34, 'yearlyEnergyDcKwh': 13893.903, 'roofSegmentSummaries': [{'pitchDegrees': 31.443022, +# 'azimuthDegrees': 218.25331, 'panelsCount': 15, 'yearlyEnergyDcKwh': 6784.1484, 'segmentIndex': 0}, +# {'pitchDegrees': 34.39779, 'azimuthDegrees': 31.74401, 'panelsCount': 9, 'yearlyEnergyDcKwh': 2992.6301, +# 'segmentIndex': 1}, {'pitchDegrees': 3.0681775, 'azimuthDegrees': 301.1099, 'panelsCount': 5, 'yearlyEnergyDcKwh': +# 2019.0519, 'segmentIndex': 2}, {'pitchDegrees': 27.093596, 'azimuthDegrees': 132.60162, 'panelsCount': 3, +# 'yearlyEnergyDcKwh': 1354.4171, 'segmentIndex': 3}, {'pitchDegrees': 31.666294, 'azimuthDegrees': 308.42334, +# 'panelsCount': 2, 'yearlyEnergyDcKwh': 743.65497, 'segmentIndex': 4}]}, {'panelsCount': 35, 'yearlyEnergyDcKwh': +# 14221.166, 'roofSegmentSummaries': [{'pitchDegrees': 31.443022, 'azimuthDegrees': 218.25331, 'panelsCount': 15, +# 'yearlyEnergyDcKwh': 6784.1484, 'segmentIndex': 0}, {'pitchDegrees': 34.39779, 'azimuthDegrees': 31.74401, +# 'panelsCount': 10, 'yearlyEnergyDcKwh': 3319.893, 'segmentIndex': 1}, {'pitchDegrees': 3.0681775, 'azimuthDegrees': +# 301.1099, 'panelsCount': 5, 'yearlyEnergyDcKwh': 2019.0519, 'segmentIndex': 2}, {'pitchDegrees': 27.093596, +# 'azimuthDegrees': 132.60162, 'panelsCount': 3, 'yearlyEnergyDcKwh': 1354.4171, 'segmentIndex': 3}, {'pitchDegrees': +# 31.666294, 'azimuthDegrees': 308.42334, 'panelsCount': 2, 'yearlyEnergyDcKwh': 743.65497, 'segmentIndex': 4}]}, +# {'panelsCount': 36, 'yearlyEnergyDcKwh': 14536.154, 'roofSegmentSummaries': [{'pitchDegrees': 31.443022, +# 'azimuthDegrees': 218.25331, 'panelsCount': 15, 'yearlyEnergyDcKwh': 6784.1484, 'segmentIndex': 0}, +# {'pitchDegrees': 34.39779, 'azimuthDegrees': 31.74401, 'panelsCount': 11, 'yearlyEnergyDcKwh': 3634.8809, +# 'segmentIndex': 1}, {'pitchDegrees': 3.0681775, 'azimuthDegrees': 301.1099, 'panelsCount': 5, 'yearlyEnergyDcKwh': +# 2019.0519, 'segmentIndex': 2}, {'pitchDegrees': 27.093596, 'azimuthDegrees': 132.60162, 'panelsCount': 3, +# 'yearlyEnergyDcKwh': 1354.4171, 'segmentIndex': 3}, {'pitchDegrees': 31.666294, 'azimuthDegrees': 308.42334, +# 'panelsCount': 2, 'yearlyEnergyDcKwh': 743.65497, 'segmentIndex': 4}]}, {'panelsCount': 37, 'yearlyEnergyDcKwh': +# 14850.317, 'roofSegmentSummaries': [{'pitchDegrees': 31.443022, 'azimuthDegrees': 218.25331, 'panelsCount': 15, +# 'yearlyEnergyDcKwh': 6784.1484, 'segmentIndex': 0}, {'pitchDegrees': 34.39779, 'azimuthDegrees': 31.74401, +# 'panelsCount': 12, 'yearlyEnergyDcKwh': 3949.0444, 'segmentIndex': 1}, {'pitchDegrees': 3.0681775, +# 'azimuthDegrees': 301.1099, 'panelsCount': 5, 'yearlyEnergyDcKwh': 2019.0519, 'segmentIndex': 2}, {'pitchDegrees': +# 27.093596, 'azimuthDegrees': 132.60162, 'panelsCount': 3, 'yearlyEnergyDcKwh': 1354.4171, 'segmentIndex': 3}, +# {'pitchDegrees': 31.666294, 'azimuthDegrees': 308.42334, 'panelsCount': 2, 'yearlyEnergyDcKwh': 743.65497, +# 'segmentIndex': 4}]}, {'panelsCount': 38, 'yearlyEnergyDcKwh': 15160.658, 'roofSegmentSummaries': [{'pitchDegrees': +# 31.443022, 'azimuthDegrees': 218.25331, 'panelsCount': 15, 'yearlyEnergyDcKwh': 6784.1484, 'segmentIndex': 0}, +# {'pitchDegrees': 34.39779, 'azimuthDegrees': 31.74401, 'panelsCount': 13, 'yearlyEnergyDcKwh': 4259.385, +# 'segmentIndex': 1}, {'pitchDegrees': 3.0681775, 'azimuthDegrees': 301.1099, 'panelsCount': 5, 'yearlyEnergyDcKwh': +# 2019.0519, 'segmentIndex': 2}, {'pitchDegrees': 27.093596, 'azimuthDegrees': 132.60162, 'panelsCount': 3, +# 'yearlyEnergyDcKwh': 1354.4171, 'segmentIndex': 3}, {'pitchDegrees': 31.666294, 'azimuthDegrees': 308.42334, +# 'panelsCount': 2, 'yearlyEnergyDcKwh': 743.65497, 'segmentIndex': 4}]}, {'panelsCount': 39, 'yearlyEnergyDcKwh': +# 15438.986, 'roofSegmentSummaries': [{'pitchDegrees': 31.443022, 'azimuthDegrees': 218.25331, 'panelsCount': 15, +# 'yearlyEnergyDcKwh': 6784.1484, 'segmentIndex': 0}, {'pitchDegrees': 34.39779, 'azimuthDegrees': 31.74401, +# 'panelsCount': 14, 'yearlyEnergyDcKwh': 4537.713, 'segmentIndex': 1}, {'pitchDegrees': 3.0681775, 'azimuthDegrees': +# 301.1099, 'panelsCount': 5, 'yearlyEnergyDcKwh': 2019.0519, 'segmentIndex': 2}, {'pitchDegrees': 27.093596, +# 'azimuthDegrees': 132.60162, 'panelsCount': 3, 'yearlyEnergyDcKwh': 1354.4171, 'segmentIndex': 3}, {'pitchDegrees': +# 31.666294, 'azimuthDegrees': 308.42334, 'panelsCount': 2, 'yearlyEnergyDcKwh': 743.65497, 'segmentIndex': 4}]}], +# 'panelCapacityWatts': 400, 'panelHeightMeters': 1.879, 'panelWidthMeters': 1.045, 'panelLifetimeYears': 20, +# 'buildingStats': {'areaMeters2': 138.38115, 'sunshineQuantiles': [537, 728.5604, 799.23975, 833.99713, 900.88086, +# 959.65875, 1024.2743, 1086.1285, 1132.8774, 1162.1904, 1193.6012], 'groundAreaMeters2': 117.16}, 'solarPanels': [{ +# 'center': {'latitude': 50.667371499999994, 'longitude': -4.7417235}, 'orientation': 'LANDSCAPE', +# 'yearlyEnergyDcKwh': 468.5037, 'segmentIndex': 0}, {'center': {'latitude': 50.6673614, 'longitude': -4.7417023}, +# 'orientation': 'LANDSCAPE', 'yearlyEnergyDcKwh': 467.61072, 'segmentIndex': 0}, {'center': {'latitude': +# 50.667365100000005, 'longitude': -4.7417311}, 'orientation': 'LANDSCAPE', 'yearlyEnergyDcKwh': 465.55005, +# 'segmentIndex': 0}, {'center': {'latitude': 50.6673512, 'longitude': -4.741681000000001}, 'orientation': +# 'LANDSCAPE', 'yearlyEnergyDcKwh': 465.48712, 'segmentIndex': 0}, {'center': {'latitude': 50.667357599999995, +# 'longitude': -4.7416734}, 'orientation': 'LANDSCAPE', 'yearlyEnergyDcKwh': 467.8553, 'segmentIndex': 0}, +# {'center': {'latitude': 50.6673779, 'longitude': -4.741715999999999}, 'orientation': 'LANDSCAPE', +# 'yearlyEnergyDcKwh': 464.84396, 'segmentIndex': 0}, {'center': {'latitude': 50.6673678, 'longitude': -4.7416947}, +# 'orientation': 'LANDSCAPE', 'yearlyEnergyDcKwh': 464.79984, 'segmentIndex': 0}, {'center': {'latitude': 50.6673549, +# 'longitude': -4.7417098}, 'orientation': 'LANDSCAPE', 'yearlyEnergyDcKwh': 461.58975, 'segmentIndex': 0}, +# {'center': {'latitude': 50.6673816, 'longitude': -4.7417448}, 'orientation': 'LANDSCAPE', 'yearlyEnergyDcKwh': +# 461.48065, 'segmentIndex': 0}, {'center': {'latitude': 50.6673881, 'longitude': -4.7417372}, 'orientation': +# 'LANDSCAPE', 'yearlyEnergyDcKwh': 458.3733, 'segmentIndex': 0}, {'center': {'latitude': 50.6673149, 'longitude': +# -4.7416768}, 'orientation': 'LANDSCAPE', 'yearlyEnergyDcKwh': 457.68268, 'segmentIndex': 3}, {'center': { +# 'latitude': 50.6673204, 'longitude': -4.7416867}, 'orientation': 'LANDSCAPE', 'yearlyEnergyDcKwh': 456.06827, +# 'segmentIndex': 3}, {'center': {'latitude': 50.667375199999995, 'longitude': -4.7417524}, 'orientation': +# 'LANDSCAPE', 'yearlyEnergyDcKwh': 453.20776, 'segmentIndex': 0}, {'center': {'latitude': 50.667364, 'longitude': +# -4.7416659}, 'orientation': 'LANDSCAPE', 'yearlyEnergyDcKwh': 448.61087, 'segmentIndex': 0}, {'center': { +# 'latitude': 50.6673094, 'longitude': -4.741666899999999}, 'orientation': 'LANDSCAPE', 'yearlyEnergyDcKwh': +# 440.66626, 'segmentIndex': 3}, {'center': {'latitude': 50.667403799999995, 'longitude': -4.741588900000001}, +# 'orientation': 'LANDSCAPE', 'yearlyEnergyDcKwh': 419.31348, 'segmentIndex': 2}, {'center': {'latitude': +# 50.66740850000001, 'longitude': -4.7416016999999995}, 'orientation': 'LANDSCAPE', 'yearlyEnergyDcKwh': 418.74448, +# 'segmentIndex': 2}, {'center': {'latitude': 50.6673688, 'longitude': -4.7417599}, 'orientation': 'LANDSCAPE', +# 'yearlyEnergyDcKwh': 413.877, 'segmentIndex': 0}, {'center': {'latitude': 50.667348499999996, 'longitude': +# -4.7417174}, 'orientation': 'LANDSCAPE', 'yearlyEnergyDcKwh': 411.76657, 'segmentIndex': 0}, {'center': { +# 'latitude': 50.6673587, 'longitude': -4.7417387}, 'orientation': 'LANDSCAPE', 'yearlyEnergyDcKwh': 410.5925, +# 'segmentIndex': 0}, {'center': {'latitude': 50.6673992, 'longitude': -4.7415761}, 'orientation': 'LANDSCAPE', +# 'yearlyEnergyDcKwh': 404.15607, 'segmentIndex': 2}, {'center': {'latitude': 50.6674132, 'longitude': -4.7416145}, +# 'orientation': 'LANDSCAPE', 'yearlyEnergyDcKwh': 403.29822, 'segmentIndex': 2}, {'center': {'latitude': 50.6673324, +# 'longitude': -4.7417015}, 'orientation': 'PORTRAIT', 'yearlyEnergyDcKwh': 378.2754, 'segmentIndex': 4}, {'center': +# {'latitude': 50.667417799999996, 'longitude': -4.7416273}, 'orientation': 'LANDSCAPE', 'yearlyEnergyDcKwh': +# 373.53967, 'segmentIndex': 2}, {'center': {'latitude': 50.667324900000004, 'longitude': -4.7417104}, 'orientation': +# 'PORTRAIT', 'yearlyEnergyDcKwh': 365.37958, 'segmentIndex': 4}, {'center': {'latitude': 50.6674043, 'longitude': +# -4.741680800000001}, 'orientation': 'PORTRAIT', 'yearlyEnergyDcKwh': 341.4827, 'segmentIndex': 1}, {'center': { +# 'latitude': 50.667392299999996, 'longitude': -4.7416919}, 'orientation': 'PORTRAIT', 'yearlyEnergyDcKwh': +# 336.64502, 'segmentIndex': 1}, {'center': {'latitude': 50.667397, 'longitude': -4.741704599999999}, 'orientation': +# 'PORTRAIT', 'yearlyEnergyDcKwh': 339.7059, 'segmentIndex': 1}, {'center': {'latitude': 50.6674018, 'longitude': +# -4.7417174}, 'orientation': 'PORTRAIT', 'yearlyEnergyDcKwh': 336.25195, 'segmentIndex': 1}, {'center': {'latitude': +# 50.6673875, 'longitude': -4.7416791}, 'orientation': 'PORTRAIT', 'yearlyEnergyDcKwh': 331.08936, 'segmentIndex': +# 1}, {'center': {'latitude': 50.6674065, 'longitude': -4.7417301}, 'orientation': 'PORTRAIT', 'yearlyEnergyDcKwh': +# 325.05405, 'segmentIndex': 1}, {'center': {'latitude': 50.6673828, 'longitude': -4.7416664}, 'orientation': +# 'PORTRAIT', 'yearlyEnergyDcKwh': 321.63647, 'segmentIndex': 1}, {'center': {'latitude': 50.667378, 'longitude': +# -4.741653599999999}, 'orientation': 'PORTRAIT', 'yearlyEnergyDcKwh': 321.46332, 'segmentIndex': 1}, {'center': { +# 'latitude': 50.667373299999994, 'longitude': -4.7416409}, 'orientation': 'PORTRAIT', 'yearlyEnergyDcKwh': 339.3016, +# 'segmentIndex': 1}, {'center': {'latitude': 50.6673853, 'longitude': -4.7416298}, 'orientation': 'PORTRAIT', +# 'yearlyEnergyDcKwh': 327.26282, 'segmentIndex': 1}, {'center': {'latitude': 50.667399499999995, 'longitude': +# -4.741668}, 'orientation': 'PORTRAIT', 'yearlyEnergyDcKwh': 314.9878, 'segmentIndex': 1}, {'center': {'latitude': +# 50.6673948, 'longitude': -4.7416553}, 'orientation': 'PORTRAIT', 'yearlyEnergyDcKwh': 314.16364, 'segmentIndex': +# 1}, {'center': {'latitude': 50.667390000000005, 'longitude': -4.7416425}, 'orientation': 'PORTRAIT', +# 'yearlyEnergyDcKwh': 310.3404, 'segmentIndex': 1}, {'center': {'latitude': 50.6674186, 'longitude': -4.7417191}, +# 'orientation': 'PORTRAIT', 'yearlyEnergyDcKwh': 278.3281, 'segmentIndex': 1}]}, 'boundingBox': {'sw': {'latitude': +# 50.6672904, 'longitude': -4.741778}, 'ne': {'latitude': 50.667431199999996, 'longitude': -4.7415536}}, +# 'imageryQuality': 'MEDIUM', 'imageryProcessedDate': {'year': 2024, 'month': 4, 'day': 18}} diff --git a/etl/customers/places_for_people/route_march.py b/etl/customers/places_for_people/route_march.py index c38c71d3..5da1c2f7 100644 --- a/etl/customers/places_for_people/route_march.py +++ b/etl/customers/places_for_people/route_march.py @@ -1,4 +1,5 @@ import os +import time import pandas as pd from tqdm import tqdm @@ -33,7 +34,7 @@ def app(): lst = [ pfp_property["ADDRESS"], pfp_property["ADDRESS.1"], - pfp_property["ADDRESS.2"], + # pfp_property["ADDRESS.2"], pfp_property["POSTCODE"] ] lst = [str(x).strip() for x in lst if not pd.isnull(x)] @@ -135,3 +136,165 @@ def app(): # Store as an excel filename = "Places For People EPC data.xlsx" asset_list.to_excel(filename, index=False) + + +# TODO: TEMP +# This script takes in a a list of properties +# Will be postcode and address + +import requests +import numpy as np +import pandas as pd +from bs4 import BeautifulSoup +from tqdm import tqdm + +SEARCH_POSTCODE_URL = ("https://find-energy-certificate.service.gov.uk/find-a-certificate/search-by-postcode?postcode" + "={postcode_input}") +BASE_ENERGY_URL = "https://find-energy-certificate.service.gov.uk" + + +def retrieve_find_my_epc_data(postcode: str, address: str): + """ + For a post code and address, we pull out all the required data from the find my epc website + """ + + headers = { + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) ' + 'Chrome/111.0.0.0 Safari/537.36'} + postcode_input = postcode.replace(" ", "+") + postcode_search = SEARCH_POSTCODE_URL.format(postcode_input=postcode_input) + postcode_response = requests.get(postcode_search, headers=headers) + + postcode_res = BeautifulSoup(postcode_response.text) + address_links_full = postcode_res.findAll('a', {'class': 'govuk-link', 'rel': 'nofollow'}) + address_links = {element.text.lstrip().rstrip(): BASE_ENERGY_URL + element['href'] for element in + address_links_full} + + # TODO: to check the logic works for all cases but seems to be good + index_of_address = [key.lower().startswith(address) for key in list(address_links.keys())] + chosen_epc = address_links[list(address_links.keys())[np.where(index_of_address)[0][0]]] + + epc_certificate = chosen_epc.split('/')[-1] + + address_response = requests.get(chosen_epc, headers=headers) + address_res = BeautifulSoup(address_response.text) + + # print("## Energy rating - current and potential") + ratings = address_res.find('desc', {'id': 'svg-desc'}).text + + # print('### Current EPC rating') + current_rating = ratings.split(".")[0] + # print("##### " + current_rating) + + # print('### Potential EPC rating') + potential_rating = ratings.split(".")[1] + # print("##### " + potential_rating) + + new_property_df = pd.DataFrame( + {'address': [address], + 'epc_certificate': [epc_certificate], + 'current_epc_rating': [current_rating.split(' ')[-6]], + 'current_epc_efficiency': [current_rating.split(' ')[-1]], + 'potential_epc_rating': [potential_rating.split(' ')[-6]], + "potential_epc_efficiency": [potential_rating.split(' ')[-1]]} + ) + + # print("Find assessor") + assessor_block = address_res.find('div', {'class': 'epc-contact-assessor'}) + assessor_fields = assessor_block.find_all('dd', {"class": 'govuk-summary-list__value govuk-!-width-one-half'}) + assessor_name = assessor_fields[0].text.strip() + assessor_number = assessor_fields[1].text.strip() + assessor_email = assessor_fields[2].text.strip() + + new_property_df['assessor_name'] = assessor_name + new_property_df['assessor_number'] = assessor_number + new_property_df['assessor_email'] = assessor_email + + return new_property_df + + # print('### Changes that can be made:') + # improvements = address_res.find('div', {"class": "govuk-body printable-area epb-recommended-improvements"}) + + # if improvements is None: + # print("No changes suggested") + # else: + # changes = improvements.find_all('h3') + # changes_impact = improvements.find_all('dl', {"class": 'govuk-summary-list'}) + + # for element in zip(changes, changes_impact): + # improvement_header = element[0].text + # print("#### " + improvement_header) + + # improvement_text = element[1].text + # print(improvement_text) + + # col_name = improvement_header.split(":")[1] + # cost = element[1].find('dd', {"class": "govuk-summary-list__value"}).text.lstrip().rstrip() + + # impact = element[1].find('text', {"class": "govuk-!-font-weight-bold"}).text.split(" ") + # impact_num = impact[0] + # impact_cat = impact[1] + # print(cost) + # new_property_df[col_name] = True + # # cost_column = col_name + '-cost' + # # new_property_df.assign(cost_column=cost) + # new_property_df[col_name + '-cost'] = cost + # new_property_df[col_name + '-impact_num'] = impact_num + # new_property_df[col_name + '-impact_cat'] = impact_cat + + # data = pd.concat([data, new_property_df]) + # data.to_csv('./portfolio.csv') + + +def main(): + """ + Main pipeline function to take in a predefined list of properties and extract names of contractors + """ + + # Load in list of properties + addresses_df = pd.read_excel("/Users/khalimconn-kowlessar/Downloads/Places For People EPC data.xlsx") + addresses_df["uprn"] = addresses_df["uprn"].astype("Int64").astype(str) + # 1256 + + find_my_epc_data_list = [] + for i, row in tqdm(addresses_df.iterrows(), total=addresses_df.shape[0]): + + if pd.isnull(row['Matched EPC Address']): + continue + # 10 second break every 50 iterations + if (i % 50 == 0) and (i != 0): + time.sleep(10) + time.sleep(1) + if row['Matched EPC Address'] == "6 CHURCHWOOD, CHURCH STREET, CRAMLINGTON": + address_data = retrieve_find_my_epc_data( + postcode=row['POSTCODE'], + address=" ".join([str(row["ADDRESS"]), row["ADDRESS.1"]]).lower() + ) + else: + address_data = retrieve_find_my_epc_data( + postcode=row['POSTCODE'], + address=", ".join(row['Matched EPC Address'].split(", ")[:-1]).lower() + ) + + address_data.insert(0, "uprn", row["uprn"]) + + find_my_epc_data_list.append(address_data) + + find_my_epc_data = pd.concat(find_my_epc_data_list) + + find_my_epc_data.to_csv('find_my_epc_data.csv') + + find_my_epc_data = find_my_epc_data.drop_duplicates("uprn") + + # Match back to addresses + addresses_df2 = addresses_df.merge( + find_my_epc_data, + how="left", + on="uprn" + ) + + addresses_df2.to_excel("Places For People EPC data with surveyor.xlsx", index=False) + + +if __name__ == "__main__": + main() diff --git a/etl/eligibility/ha_15_32/ha_analysis_batch_3.py b/etl/eligibility/ha_15_32/ha_analysis_batch_3.py index f99c7b1a..aca36584 100644 --- a/etl/eligibility/ha_15_32/ha_analysis_batch_3.py +++ b/etl/eligibility/ha_15_32/ha_analysis_batch_3.py @@ -24,9 +24,13 @@ from etl.epc_clean.epc_attributes.RoofAttributes import RoofAttributes from etl.epc.DataProcessor import EPCDataProcessor from datetime import datetime +import inspect + +src_file_path = inspect.getfile(lambda: None) + EPC_AUTH_TOKEN = os.getenv("EPC_AUTH_TOKEN") -ENV_FILE = Path(__file__).parent / "etl" / "eligibility" / "ha_15_32" / ".env" -DATA_FOLDER = Path(__file__).parent / "local_data" / "ha_data" +ENV_FILE = Path(src_file_path).parent / "etl" / "eligibility" / "ha_15_32" / ".env" +DATA_FOLDER = Path(src_file_path).parent / "local_data" / "ha_data" logger = setup_logger() load_dotenv(ENV_FILE) @@ -6127,6 +6131,9 @@ def classify_loft(x): def fml_analysis(loader): + # In the case of the optimistic scenario, we assume that the at-risk pipeline is still viable, just at a lower rate + optimistic_scenario_rate = 1500 + assumed_ciga_pass_rate = 0.731 has_bruh = [ "HA7", "HA14", "HA25", "HA39", "HA16", "HA28", "HA13", @@ -6224,6 +6231,7 @@ def fml_analysis(loader): if fuck_this.shape[0] != before_merge_shape: raise Exception("SOMETHING WENT WRONG") + # Automated archetype check if any(fuck_this["ECO Eligibility"].str.contains("subject to archetype")): # We perform the archetype test. If the property is a house, we it needs to be detached, semi-detached # or end terrace. If it's a bungalow, it must be attached @@ -6319,6 +6327,7 @@ def fml_analysis(loader): ] # Characterise no CIGA check needed + # !!!!!!!!!!!! AT RISK !!!!!!!!!!!! ciga_check_passed = had_survey[had_survey["ECO Eligibility"] == "eco4 - passed ciga"] # These should be treated the same as one that have passed their ciga checks, from a detection perspective ciga_check_passed_eligible = ciga_check_passed[ @@ -6392,6 +6401,12 @@ def fml_analysis(loader): identified_as_gbis_looks_like_eco4 ) + # This is the work that is at risk + eco4_work_at_risk = ( + passed_ciga_expectation + + ciga_check_expectation + ) + no_ciga_check_needed_actually_gbis = no_ciga_check_needed_eligible_gbis.shape[0] gbis_qualified = gbis_qualified.shape[0] @@ -6490,11 +6505,13 @@ def fml_analysis(loader): # "Of which sold": sales_since_nov, "EPC verified ECO4 Eligible - Remaining": int(total_eco4_expectation), "EPC verified GBIS Eligibile - Remaining": int(total_gbis_expectation), + # At risk work + "Work at risk due to audits": eco4_work_at_risk } ) results_df = pd.DataFrame(results) - results_df.to_csv("analysis - revised.csv") + results_df.to_csv("analysis - revised - audit update.csv") # results_df["Delta vs November"] = 100 * ( # results_df["Of which ECO4 Eligible - Remaining"] - results_df["Original ECO4 Estimate - Remaining"] @@ -6509,7 +6526,7 @@ def create_final_report(): This function will produce the final output for the HA analysis :return: """ - epc_validated_results = pd.read_csv("analysis - revised.csv") + epc_validated_results = pd.read_csv("analysis - revised - audit update.csv") pipeline_results = pd.read_csv("pipeline_remaining_raw.csv") #################################### @@ -6593,12 +6610,14 @@ def create_final_report(): [ "HA Name", "EPC verified ECO4 Eligible - Remaining", - "EPC verified GBIS Eligibile - Remaining" + "EPC verified GBIS Eligibile - Remaining", + "Work at risk due to audits" ] ].copy().rename( columns={ "EPC verified ECO4 Eligible - Remaining": "# ECO4 remaining - From EPC Database (post CIGA)", "EPC verified GBIS Eligibile - Remaining": "# GBIS remaining - From EPC Database (post CIGA)", + "Work at risk due to audits": "ECO4 remaining work at risk due to Audits", } ) @@ -6623,7 +6642,8 @@ def create_final_report(): '# ECO4 remaining - All HA Summary', '# ECO4 remaining - Postcode list (pre CIGA)', '# ECO4 remaining - Postcode list (post CIGA)', - '# ECO4 remaining - From EPC Database (post CIGA)' + '# ECO4 remaining - From EPC Database (post CIGA)', + 'ECO4 remaining work at risk due to Audits' ]: revenue[col] = revenue[col] * 1710 @@ -6688,8 +6708,8 @@ def create_final_report(): # "# GBIS remaining - Postcode list (post CIGA)"]] # Store final outputs - volumes.to_csv("HA Analysis Final - volumes.csv") - revenue.to_csv("HA Analysis Final - revenue.csv") + volumes.to_csv("HA Analysis - Audit Update - volumes.csv") + revenue.to_csv("HA Analysis - Audit Update - revenue.csv") def identify_eco_works(loader): @@ -7203,84 +7223,96 @@ def app(): loader.load() loader.ha_facts_and_figures() + # import pickle + # with open("ha_analysis_data_temp.pkl", "wb") as f: + # pickle.dump(loader, f) + # import pickle + # with open("ha_analysis_data_temp.pkl", "rb") as f: + # loader = pickle.load(f) + forecast_remaining_sales(loader) + # Functions to produce the final output lol... + # fml_data_pull(loader) # If we need to pull EPC data + fml_analysis(loader) + create_final_report() + # Adhoc - for HA16, get the properties that still need a CIGA check - asset_list_ha16 = loader.data["HA16"]["asset_list"].copy() - ha_16_need_ciga = asset_list_ha16[ - asset_list_ha16["ECO Eligibility"].str.contains("subject to ciga") - ] - completed_cigas = loader.data["HA16"]["ciga_list"].copy() - # Store the results - ha_16_need_ciga.to_csv("ha16_need_ciga.csv") - completed_cigas.to_csv("ha16_completed_cigas.csv") - - # Adhoc - look at the current pipeline and identify how many dormant, CIGA dependent properties there are for - # live projects - - # Read excel - orderbook_filepath = "local_data/ha_data/Warmfront HA client order book overview_20240129.xlsx" - orderbook_workbook = openpyxl.load_workbook(orderbook_filepath) - orderbook_sheet = orderbook_workbook["Contractual Info"] - orderbook_colnames = [cell.value for cell in orderbook_sheet[1]] - - rows = [] - for row in orderbook_sheet.iter_rows(min_row=2, values_only=False): - row_data = [cell.value for cell in row] # This will get you the cell values - rows.append(row_data) - - orderbook = pd.DataFrame(rows, columns=orderbook_colnames) - live_orderbook = orderbook[orderbook["Live, New, or Historic?"] == "LIVE"].copy() - live_orderbook['Redacted HA'] = live_orderbook['Redacted HA'].str.replace(" ", "") - - dormant_properties = [] - missed_has = [] - for _, customer in live_orderbook.iterrows(): - if customer['Redacted HA'] not in loader.data.keys(): - missed_has.append(customer['Redacted HA']) - continue - asset_list = loader.data[customer['Redacted HA']]["asset_list"].copy() - survey_list = loader.data[customer['Redacted HA']]["survey_list"].copy() - # Remove sold - if not survey_list.empty: - survey_list = survey_list[~pd.isnull(survey_list["asset_list_row_id"])] - asset_list = asset_list.merge( - survey_list[["asset_list_row_id", "installation_status"]], - how="left", - on="asset_list_row_id" - ) - # Anything that has an installation has gone to installation, and therefore is not remaining - asset_list = asset_list[pd.isnull(asset_list["installation_status"])] - asset_list = asset_list.drop(columns=["installation_status"]) - - # We pull out the properties that need a CIGA check - need_ciga = asset_list[asset_list["ECO Eligibility"] == "eco4 (subject to ciga)"] - need_archetype = asset_list[asset_list["ECO Eligibility"] == "eco4 (subject to archetype)"] - need_ciga_and_archetype = asset_list[ - asset_list["ECO Eligibility"] == "eco4 (subject to ciga) (subject to archetype)" - ] - - dormant_properties.append( - { - "HA Name": customer['Redacted HA'], - "Need CIGA": need_ciga.shape[0], - "Need Archetype": need_archetype.shape[0], - "Need CIGA and Archetype": need_ciga_and_archetype.shape[0] - } - ) - - dormant_properties = pd.DataFrame(dormant_properties) - totals = dormant_properties.sum() - totals["HA Name"] = "Total" - - dormant_properties = pd.concat([dormant_properties, totals.to_frame().T]) - dormant_properties.to_csv("dormant_properties.csv") - - loader.december_figures["ECO4 remaining"].sum() - december_figures = loader.december_figures.copy() - december_figures["ECO4 remaining"] = np.where( - december_figures["ECO4 remaining"] < 0, - 0, - december_figures["ECO4 remaining"] - ) - december_figures["ECO4 remaining"].sum() + # asset_list_ha16 = loader.data["HA16"]["asset_list"].copy() + # ha_16_need_ciga = asset_list_ha16[ + # asset_list_ha16["ECO Eligibility"].str.contains("subject to ciga") + # ] + # completed_cigas = loader.data["HA16"]["ciga_list"].copy() + # # Store the results + # ha_16_need_ciga.to_csv("ha16_need_ciga.csv") + # completed_cigas.to_csv("ha16_completed_cigas.csv") + # + # # Adhoc - look at the current pipeline and identify how many dormant, CIGA dependent properties there are for + # # live projects + # + # # Read excel + # orderbook_filepath = "local_data/ha_data/Warmfront HA client order book overview_20240129.xlsx" + # orderbook_workbook = openpyxl.load_workbook(orderbook_filepath) + # orderbook_sheet = orderbook_workbook["Contractual Info"] + # orderbook_colnames = [cell.value for cell in orderbook_sheet[1]] + # + # rows = [] + # for row in orderbook_sheet.iter_rows(min_row=2, values_only=False): + # row_data = [cell.value for cell in row] # This will get you the cell values + # rows.append(row_data) + # + # orderbook = pd.DataFrame(rows, columns=orderbook_colnames) + # live_orderbook = orderbook[orderbook["Live, New, or Historic?"] == "LIVE"].copy() + # live_orderbook['Redacted HA'] = live_orderbook['Redacted HA'].str.replace(" ", "") + # + # dormant_properties = [] + # missed_has = [] + # for _, customer in live_orderbook.iterrows(): + # if customer['Redacted HA'] not in loader.data.keys(): + # missed_has.append(customer['Redacted HA']) + # continue + # asset_list = loader.data[customer['Redacted HA']]["asset_list"].copy() + # survey_list = loader.data[customer['Redacted HA']]["survey_list"].copy() + # # Remove sold + # if not survey_list.empty: + # survey_list = survey_list[~pd.isnull(survey_list["asset_list_row_id"])] + # asset_list = asset_list.merge( + # survey_list[["asset_list_row_id", "installation_status"]], + # how="left", + # on="asset_list_row_id" + # ) + # # Anything that has an installation has gone to installation, and therefore is not remaining + # asset_list = asset_list[pd.isnull(asset_list["installation_status"])] + # asset_list = asset_list.drop(columns=["installation_status"]) + # + # # We pull out the properties that need a CIGA check + # need_ciga = asset_list[asset_list["ECO Eligibility"] == "eco4 (subject to ciga)"] + # need_archetype = asset_list[asset_list["ECO Eligibility"] == "eco4 (subject to archetype)"] + # need_ciga_and_archetype = asset_list[ + # asset_list["ECO Eligibility"] == "eco4 (subject to ciga) (subject to archetype)" + # ] + # + # dormant_properties.append( + # { + # "HA Name": customer['Redacted HA'], + # "Need CIGA": need_ciga.shape[0], + # "Need Archetype": need_archetype.shape[0], + # "Need CIGA and Archetype": need_ciga_and_archetype.shape[0] + # } + # ) + # + # dormant_properties = pd.DataFrame(dormant_properties) + # totals = dormant_properties.sum() + # totals["HA Name"] = "Total" + # + # dormant_properties = pd.concat([dormant_properties, totals.to_frame().T]) + # dormant_properties.to_csv("dormant_properties.csv") + # + # loader.december_figures["ECO4 remaining"].sum() + # december_figures = loader.december_figures.copy() + # december_figures["ECO4 remaining"] = np.where( + # december_figures["ECO4 remaining"] < 0, + # 0, + # december_figures["ECO4 remaining"] + # ) + # december_figures["ECO4 remaining"].sum()