diff --git a/.github/workflows/hubspot_abri_sync.yml b/.github/workflows/hubspot_abri_sync.yml new file mode 100644 index 0000000..c9f4b5e --- /dev/null +++ b/.github/workflows/hubspot_abri_sync.yml @@ -0,0 +1,30 @@ +name: Hubspot Sync Abri + +on: + schedule: + - cron: '0 6 * * 1' # Every Monday at 06:00 UTC + workflow_dispatch: + +jobs: + surveyed-needs-sign-off: + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Install dependencies + run: | + pip install poetry + poetry install --no-root + + - name: Run scripts + env: + PYTHONPATH: ${{ github.workspace }} + run: | + pwd + ls -la + poetry run python etl/hubSpotClient/scripts/hubspot_update_abri_script.py \ No newline at end of file diff --git a/alembic/versions/57c0dc06cd25_add_company_info.py b/alembic/versions/57c0dc06cd25_add_company_info.py new file mode 100644 index 0000000..4c3684f --- /dev/null +++ b/alembic/versions/57c0dc06cd25_add_company_info.py @@ -0,0 +1,46 @@ +"""add company info + +Revision ID: 57c0dc06cd25 +Revises: 23a4e2cc5467 +Create Date: 2025-10-27 20:25:27.686455 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +import sqlmodel + +# revision identifiers, used by Alembic. +revision: str = '57c0dc06cd25' +down_revision: Union[str, None] = '23a4e2cc5467' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + +def upgrade() -> None: + """Upgrade schema.""" + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('hubspot_company_data', + sa.Column('id', sa.Uuid(), nullable=False), + sa.Column('company_id', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('company_name', sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column('group_id', sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column( + 'created_at', + sa.DateTime(timezone=True), + server_default=sa.text('(CURRENT_TIMESTAMP AT TIME ZONE \'UTC\')'), + nullable=False, + ), + sa.Column('updated_at', sa.DateTime(timezone=True), nullable=True), + sa.PrimaryKeyConstraint('id') + ) + op.create_index(op.f('ix_hubspot_company_data_company_id'), 'hubspot_company_data', ['company_id'], unique=False) + # ### end Alembic commands ### + + +def downgrade() -> None: + """Downgrade schema.""" + # ### commands auto generated by Alembic - please adjust! ### + op.drop_index(op.f('ix_hubspot_company_data_company_id'), table_name='hubspot_company_data') + op.drop_table('hubspot_company_data') + # ### end Alembic commands ### diff --git a/alembic/versions/66d2c9c325d6_auto_update.py b/alembic/versions/66d2c9c325d6_auto_update.py new file mode 100644 index 0000000..e155bfb --- /dev/null +++ b/alembic/versions/66d2c9c325d6_auto_update.py @@ -0,0 +1,57 @@ +"""auto update + +Revision ID: 66d2c9c325d6 +Revises: 57c0dc06cd25 +Create Date: 2025-10-28 11:58:28.356864 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision: str = '66d2c9c325d6' +down_revision: Union[str, None] = '57c0dc06cd25' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + """Upgrade schema safely.""" + # 1️⃣ Fill existing NULLs with current UTC time + op.execute("UPDATE hubspot_company_data SET updated_at = NOW() AT TIME ZONE 'utc' WHERE updated_at IS NULL;") + op.execute("UPDATE hubspot_deal_data SET updated_at = NOW() AT TIME ZONE 'utc' WHERE updated_at IS NULL;") + + # 2️⃣ Now alter the column defaults and nullability + op.alter_column( + 'hubspot_company_data', + 'updated_at', + existing_type=sa.TIMESTAMP(timezone=True), + server_default=sa.text("NOW() AT TIME ZONE 'utc'"), + nullable=False, + ) + + op.alter_column( + 'hubspot_deal_data', + 'updated_at', + existing_type=sa.TIMESTAMP(timezone=True), + server_default=sa.text("NOW() AT TIME ZONE 'utc'"), + nullable=False, + ) + + + +def downgrade() -> None: + """Downgrade schema.""" + # ### commands auto generated by Alembic - please adjust! ### + op.alter_column('hubspot_deal_data', 'updated_at', + existing_type=postgresql.TIMESTAMP(timezone=True), + server_default=None, + nullable=True) + op.alter_column('hubspot_company_data', 'updated_at', + existing_type=postgresql.TIMESTAMP(timezone=True), + server_default=None, + nullable=True) + # ### end Alembic commands ### diff --git a/etl/hubSpotClient/scripts/hubspot_abri_etl_first_time.py b/etl/hubSpotClient/scripts/hubspot_abri_etl_first_time.py new file mode 100644 index 0000000..9dc4e4c --- /dev/null +++ b/etl/hubSpotClient/scripts/hubspot_abri_etl_first_time.py @@ -0,0 +1,45 @@ +from etl.hubSpotClient.hubspotClient import HubSpotClient, Companies, Pipeline +from tqdm import tqdm +from etl.db.hubSpotLoad import HubspotTodb + +''' +# TODO: + get one deal from db, from db + for avri only so far + add it to the db + show in frontend +''' + +# get ALL deals +hubspot = HubSpotClient() + +# All deals from a pipeline_id via filter +deals = hubspot.get_deal_ids_by_pipeline( + pipeline_id=Pipeline.OPERATIONS_SOCIAL_HOUSING.value, + ) + +# deals from companies we care about +valueable_deals = [ + Companies.ABRI.value +] +deals_to_add = [] + + +deal_to_companies = {} +loader = HubspotTodb() +# Get all deals we care about +for i,deal in enumerate(tqdm(deals)): + company = hubspot.from_deal_get_associated_company_id(deal) + if company in valueable_deals: + deals_to_add.append(deal) + deal_to_companies.update({deal: company}) + deal_data = hubspot.from_deal_get_info(deal_id=deal) + listing_data = hubspot.from_deal_get_associated_listing(deal_id=deal) + loader.new_record_to_hubspot_data(deal_data, deal_to_companies[deal], listing_data) + + + +#TODO check if database has abri data +# make companies table +# make a scrip that updates table + diff --git a/etl/hubSpotClient/scripts/hubspot_company.py b/etl/hubSpotClient/scripts/hubspot_company.py new file mode 100644 index 0000000..b8f8342 --- /dev/null +++ b/etl/hubSpotClient/scripts/hubspot_company.py @@ -0,0 +1,15 @@ +from etl.hubSpotClient.hubspotClient import HubSpotClient, Companies, Pipeline +from tqdm import tqdm +from etl.db.hubSpotLoad import HubspotTodb + +hubspot = HubSpotClient() + +# All deals from a pipeline_id via filter +company = hubspot.get_company_information(Companies.ABRI.value) + +loader = HubspotTodb() +loader.new_record_company(company) + + +# make a scrip that updates table, with khalim scoping + diff --git a/etl/hubSpotClient/scripts/hubspot_update_abri_script.py b/etl/hubSpotClient/scripts/hubspot_update_abri_script.py new file mode 100644 index 0000000..428fd99 --- /dev/null +++ b/etl/hubSpotClient/scripts/hubspot_update_abri_script.py @@ -0,0 +1,23 @@ +from etl.hubSpotClient.hubspotClient import HubSpotClient, Companies, Pipeline +from tqdm import tqdm +from etl.db.hubSpotLoad import HubspotTodb + +hubspot = HubSpotClient() +db = HubspotTodb() + +records = db.find_all_deals_with_company_id(Companies.ABRI.value) + +updated_count = 0 # Counter for deals that needed updating +checked_count = 0 # Optional: total processed counter + +for deal in tqdm(records, desc="Checking HubSpot deals"): + checked_count += 1 + was_up_to_date = db.update_deal(deal, hubspot) + + # update_deal() returns False when discrepancies are found + if not was_up_to_date: + updated_count += 1 + +print(f"\n✅ Finished checking {checked_count} deals.") +print(f"🧩 {updated_count} deal(s) were updated.") +print(f"📈 {checked_count - updated_count} deal(s) were already up to date.") \ No newline at end of file