junte/Model
1
0
Fork 0
mirror of https://github.com/Hestia-Homes/Model.git synced 2026-06-30 13:10:47 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

debugging data pull

Browse source
This commit is contained in:
Khalim Conn-Kowlessar 2024-11-05 14:29:54 +00:00
parent 557c0b5898
commit a7aecb2462
1 changed files with 6 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
etl/route_march/oo_prs_additional_units/oo_prs_additional_units.py
View file

@ -75,9 +75,12 @@ SOCIAL_TENURES = ["Rented (social)", "rental (social)"]
def process_postcode_epcs(postcode, client): def process_postcode_epcs(postcode, client):
params = {"postcode": postcode} params = {"postcode": postcode.rstrip().lstrip()}
url = os.path.join(client.domestic.host, "search") + "?" + urlencode({"size": 1000}) url = os.path.join(client.domestic.host, "search") + "?" + urlencode({"size": 1000})
response = client.domestic.call(method="get", url=url, params=params) response = client.domestic.call(method="get", url=url, params=params)
if "rows" not in response:
logger.warning("No EPCs found for postcode %s", postcode)
return pd.DataFrame()
postcode_epcs = pd.DataFrame(response["rows"]) postcode_epcs = pd.DataFrame(response["rows"])
# Processing code here # Processing code here
@ -191,6 +194,8 @@ def main():
epcs = [] epcs = []
for postcode in tqdm(postcodes): for postcode in tqdm(postcodes):
postcode_epcs = process_postcode_epcs(postcode, client) postcode_epcs = process_postcode_epcs(postcode, client)
if postcode_epcs.empty:
continue
epcs.append(postcode_epcs) epcs.append(postcode_epcs)
# Concatenate all postcodes' data and filter it # Concatenate all postcodes' data and filter it