junte/Model
1
0
Fork 0
mirror of https://github.com/Hestia-Homes/Model.git synced 2026-06-30 13:10:47 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

matched submissions

Browse source
This commit is contained in:
Khalim Conn-Kowlessar 2025-04-13 21:54:32 +01:00
parent 3cfe938e27
commit 83a1ac8cf3
2 changed files with 23 additions and 9 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

26
asset_list/AssetList.py
View file

@ -5,7 +5,6 @@ import tiktoken
from pprint import pprint from pprint import pprint
from datetime import datetime from datetime import datetime
from docutils.utils.math.tex2mathml_extern import blahtexml
from openai import OpenAI from openai import OpenAI
import numpy as np import numpy as np
import pandas as pd import pandas as pd
@ -379,9 +378,10 @@ class AssetList:
self.contact_details = None self.contact_details = None
self.contact_detail_fields = None self.contact_detail_fields = None
self.outcomes = None self.outcomes = None
self.outcomes_no_match = None self.outcomes_no_match = pd.DataFrame()
self.outcomes_for_output = pd.DataFrame() self.outcomes_for_output = pd.DataFrame()
self.master_surveyed = None self.master_surveyed = None
self.unmatched_submissions = pd.DataFrame()
# When this is True, we intend to break the programme into multiple phases. We may need to review # When this is True, we intend to break the programme into multiple phases. We may need to review
# how this is structured in the future, as depending on how we get future data, we may need to # how this is structured in the future, as depending on how we get future data, we may need to
@ -2249,6 +2249,7 @@ class AssetList:
logger.info("Getting masters and merging onto asset list") logger.info("Getting masters and merging onto asset list")
master_surveyed = [] master_surveyed = []
unmatched_submissions = []
for filepath in master_filepaths: for filepath in master_filepaths:
master_data = pd.read_csv(filepath) master_data = pd.read_csv(filepath)
# Strip columns # Strip columns
@ -2293,14 +2294,17 @@ class AssetList:
axis=1 axis=1
) )
postcode_col = "POSTCODE" if "POSTCODE" in master_data.columns else "Post Code"
house_no_col = 'NO.' if 'NO.' in master_data.columns else "NO"
# Otherwise, we need to match algorithmically # Otherwise, we need to match algorithmically
logger.info("Matching master data to asset list") logger.info("Matching master data to asset list")
matched = [] matched = []
unmatched = [] unmatched = []
for _, row in tqdm(master_data.iterrows(), total=len(master_data)): for _, row in tqdm(master_data.iterrows(), total=len(master_data)):
if pd.isnull(row["POSTCODE"]): if pd.isnull(row[postcode_col]):
continue continue
postcode_no_space = row["POSTCODE"].strip().replace(" ", "").lower() postcode_no_space = row[postcode_col].strip().replace(" ", "").lower()
df = self.standardised_asset_list[ df = self.standardised_asset_list[
( (
@ -2310,7 +2314,7 @@ class AssetList:
) )
] ]
house_no = row["NO"] house_no = row[house_no_col]
if house_no in df["house_no"].values: if house_no in df["house_no"].values:
df = df[df["house_no"] == house_no] df = df[df["house_no"] == house_no]
@ -2326,7 +2330,7 @@ class AssetList:
df = df[ df = df[
df[self.STANDARD_FULL_ADDRESS].str.lower().apply( df[self.STANDARD_FULL_ADDRESS].str.lower().apply(
lambda x: process.extractOne( lambda x: process.extractOne(
" ".join([row["NO"], row["Street / Block Name"], row["TOWN"]]).lower(), " ".join([row[house_no_col], row["Street / Block Name"], row["TOWN"]]).lower(),
x x
)[1] )[1]
) > 90 ) > 90
@ -2337,11 +2341,11 @@ class AssetList:
continue continue
if any(df[self.STANDARD_FULL_ADDRESS].str.lower().str.contains( if any(df[self.STANDARD_FULL_ADDRESS].str.lower().str.contains(
" ".join([row["NO"], row["Street / Block Name"]]).lower() " ".join([row[house_no_col], row["Street / Block Name"]]).lower()
)): )):
df = df[ df = df[
df[self.STANDARD_FULL_ADDRESS].str.lower().str.contains( df[self.STANDARD_FULL_ADDRESS].str.lower().str.contains(
" ".join([row["NO"], row["Street / Block Name"]]).lower() " ".join([row[house_no_col], row["Street / Block Name"]]).lower()
) )
] ]
@ -2384,7 +2388,7 @@ class AssetList:
unmatched_df = master_data[ unmatched_df = master_data[
master_data["row_id"].isin(unmatched) master_data["row_id"].isin(unmatched)
] ]
submissions_unmatched.append(unmatched_df) unmatched_submissions.append(unmatched_df)
master_surveyed = pd.concat(master_surveyed) master_surveyed = pd.concat(master_surveyed)
master_surveyed = master_surveyed[~pd.isnull(master_surveyed[self.STANDARD_LANDLORD_PROPERTY_ID])] master_surveyed = master_surveyed[~pd.isnull(master_surveyed[self.STANDARD_LANDLORD_PROPERTY_ID])]
@ -2404,3 +2408,7 @@ class AssetList:
self.standardised_asset_list = self.standardised_asset_list.merge( self.standardised_asset_list = self.standardised_asset_list.merge(
self.master_surveyed, how="left", on=self.STANDARD_LANDLORD_PROPERTY_ID self.master_surveyed, how="left", on=self.STANDARD_LANDLORD_PROPERTY_ID
) )
# Finally, we keep a record of the unmatched
if unmatched_submissions:
self.unmatched_submissions = pd.concat(unmatched_submissions)

6
asset_list/app.py
View file

@ -943,5 +943,11 @@ def app():
if not asset_list.outcomes_for_output.empty: if not asset_list.outcomes_for_output.empty:
asset_list.outcomes_for_output.to_excel(writer, sheet_name="Outcomes", index=False) asset_list.outcomes_for_output.to_excel(writer, sheet_name="Outcomes", index=False)
if not asset_list.unmatched_submissions.empty:
asset_list.unmatched_submissions.to_excel(writer, sheet_name="Unmatched Submissions", index=False)
if not asset_list.outcomes_no_match.empty:
asset_list.outcomes_no_match.to_excel(writer, sheet_name="Unmatched Outcomes", index=False)
# Store the Hubspot export as a csv # Store the Hubspot export as a csv
hubspot_data.to_csv(os.path.join(data_folder, "Hubspot Export.csv"), index=False) hubspot_data.to_csv(os.path.join(data_folder, "Hubspot Export.csv"), index=False)