survey-extraction/etl/db/hubSpotLoad.py

from etl.hubSpotClient.hubspot import HubSpotClient, DealStage
from etl.surveyPrice.surveyPrice import SurveyPrice
from etl.surveyedData.surveryedData import surveyedDataProcessor
from etl.scraper.scraper import SharePointScraper, SharePointInstaller
from etl.db.db import get_db_session, init_db
import pandas as pd
from etl.db.db import get_db_session, init_db
from etl.utils.utils import get_sharepoint_path

class HubspotTodb():
    def __init__(self):
        init_db()
        self.hubspot = HubSpotClient()
        self.deals_in_hubspot = None
        self.data_in_sharepoint = []
        self.sp = SurveyPrice()

    def get_all_deals(self):
        self.deals_in_hubspot = self.sp.get_all_surveys_from_hubspot()
        return self.deals_in_hubspot
    
            
    def get_sharepoint_scraper(self, installer):
        sp = None
        if installer.upper() == "J & J CRUMP":
            sp = SharePointScraper(SharePointInstaller.JJC)
        elif installer.upper() == "SCIS":
            sp = SharePointScraper(SharePointInstaller.SOUTH_COAST_INSULATION)
        elif installer.upper() == "SGEC":
            sp = SharePointScraper(SharePointInstaller.SGEC)
        else:
            sp = None

        return sp
    
    def create_files_locally(self, sp, path, address):
        address_paths = {}
        file_names_to_download = {}
        avoid = [".jpg",".mov", ".JPG", ".heic", ".HEIC", ".png", ".PNG", ".jpeg", ".JPEG", ".mov", ".MOV", ".mp4", ".MP4"]


        microsoft_graph_data = sp.get_folders_in_path(path)
        for file in microsoft_graph_data['value']:
            if 'file' in file:
                if any(file["name"].endswith(ext) for ext in avoid):
                    continue
                file_names_to_download.update({file["name"]: file['@microsoft.graph.downloadUrl']})
        
        each_file = []
        for file_name, url in file_names_to_download.items():
            content = sp.get_file_content(url)
            file_path = sp.create_temp_file(content, f"{address}/{file_name}")
            each_file.append(file_path)
        address_paths.update({address: each_file})
        return address_paths
    
    def string_to_installer(self, installer):
        if installer.upper() == "J & J CRUMP":
            return SharePointInstaller.JJC
        elif installer.upper() == "SCIS":
            return SharePointInstaller.SOUTH_COAST_INSULATION
        elif installer.upper() == "SGEC":
            return SharePointInstaller.SGEC
        else:
            return None

    def work_out_invoice(self, row):
        survey = self.gather_data_from_sharepoint_url(row)
        installer = self.string_to_installer(row["HUBSPOT_INSTALLER"])
        survey_pd = pd.DataFrame([self.sp.survey_to_pandas_format(surveyInfo=survey, installer=installer)])
        hubspot_data = pd.DataFrame([row])
        merged_df = self.sp.merge_hub_spot_and_survey_information_from_sharepoint_url(hubspot_data, survey_pd)
        return self.sp.calculate_all_price(merged_df)



        # self.sp.calculate_one_price_with_sharepoint_url(row, )

    def gather_data_from_sharepoint_url(self, row):
        sp = self.get_sharepoint_scraper(row["HUBSPOT_INSTALLER"])
        path = get_sharepoint_path(row["HUBSPOT_SHAREPOINT_PATH"])
        data_loc = self.create_files_locally(sp, path, row["HUBSPOT_DEAL_ADDRESS"])

        for add, file_loc in data_loc.items():
            sdp = surveyedDataProcessor(add, file_loc)
            sdp.hubspot_deal_id = row["HUBSPOT_DEAL_ID"]
            with  get_db_session() as session:
                self.load_one_pre_site_note(session, sdp, row)
            return sdp


    def gather_data_from_each_sharepoint(self):
        self.get_all_deals()
        for _, row in self.deals_in_hubspot.iterrows():
            sp = self.get_sharepoint_scraper(row["HUBSPOT_INSTALLER"])
            path = self.get_sharepoint_path(row["HUBSPOT_SHAREPOINT_PATH"])
            data_loc = self.create_files_locally(sp, path, row["HUBSPOT_DEAL_ADDRESS"])

            for add, file_loc in data_loc.items():
                sdp = surveyedDataProcessor(add, file_loc)
                sdp.hubspot_deal_id = row["HUBSPOT_DEAL_ID"]
                self.data_in_sharepoint.append(sdp)

    
    def load_all(self, fast=False):
        if fast is False:
            self.gather_data_from_each_sharepoint()
        with get_db_session() as session:
            self.load_all_pre_site_note(session)
            session.commit()

    def load_one_pre_site_note(self, db_session, surveyedData, hubspot_data):
        df = hubspot_data
        assessor = surveyedData.load_assessor_table(db_session)

        # Loads the pre site summary information
        summary_info = surveyedData.load_pre_site_notes_summary_table(db_session)

        property_description = surveyedData.load_property_description(db_session)

        # Creates the a final pre site note table that links all information
        presitenote = surveyedData.create_pre_site_note_table(db_session, assessor, summary_info, property_description)

        building_table = surveyedData.create_buildings_table(
            db_session,
            df["HUBSPOT_LANDLORD_ID"],
            df["HUBSPOT_DOMNA_ID"],
        )
        documents = surveyedData.create_document_table_via_pre_site_note(db_session, presitenote, assessor, building_table)

    def load_all_pre_site_note(self, db_session):
        # Loads all pre
        for surveyedData in self.data_in_sharepoint:
            self.load_one_pre_site_note(surveyedData=surveyedData, db_session=db_session)