added hubspot sync

.github/workflows/hubspot_abri_sync.yml

@@ -0,0 +1,30 @@
+name: Hubspot Sync Abri
+
+on:
+  schedule:
+    - cron: '0 6 * * 1'   # Every Monday at 06:00 UTC
+  workflow_dispatch:
+
+jobs:
+  surveyed-needs-sign-off:
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+
+      - name: Install dependencies
+        run: |
+          pip install poetry
+          poetry install --no-root
+
+      - name: Run scripts
+        env:
+          PYTHONPATH: ${{ github.workspace }}
+        run: |
+          pwd
+          ls -la
+          poetry run python etl/hubSpotClient/scripts/hubspot_update_abri_script.py

alembic/versions/57c0dc06cd25_add_company_info.py

@@ -0,0 +1,46 @@
+"""add company info
+
+Revision ID: 57c0dc06cd25
+Revises: 23a4e2cc5467
+Create Date: 2025-10-27 20:25:27.686455
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+import sqlmodel
+
+# revision identifiers, used by Alembic.
+revision: str = '57c0dc06cd25'
+down_revision: Union[str, None] = '23a4e2cc5467'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+def upgrade() -> None:
+    """Upgrade schema."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('hubspot_company_data',
+    sa.Column('id', sa.Uuid(), nullable=False),
+    sa.Column('company_id', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
+    sa.Column('company_name', sqlmodel.sql.sqltypes.AutoString(), nullable=True),
+    sa.Column('group_id', sqlmodel.sql.sqltypes.AutoString(), nullable=True),
+    sa.Column(
+        'created_at',
+        sa.DateTime(timezone=True),
+        server_default=sa.text('(CURRENT_TIMESTAMP AT TIME ZONE \'UTC\')'),
+        nullable=False,
+    ),
+    sa.Column('updated_at', sa.DateTime(timezone=True), nullable=True),
+    sa.PrimaryKeyConstraint('id')
+    )
+    op.create_index(op.f('ix_hubspot_company_data_company_id'), 'hubspot_company_data', ['company_id'], unique=False)
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    """Downgrade schema."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_index(op.f('ix_hubspot_company_data_company_id'), table_name='hubspot_company_data')
+    op.drop_table('hubspot_company_data')
+    # ### end Alembic commands ###

alembic/versions/66d2c9c325d6_auto_update.py

@@ -0,0 +1,57 @@
+"""auto update
+
+Revision ID: 66d2c9c325d6
+Revises: 57c0dc06cd25
+Create Date: 2025-10-28 11:58:28.356864
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision: str = '66d2c9c325d6'
+down_revision: Union[str, None] = '57c0dc06cd25'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    """Upgrade schema safely."""
+    # 1️⃣ Fill existing NULLs with current UTC time
+    op.execute("UPDATE hubspot_company_data SET updated_at = NOW() AT TIME ZONE 'utc' WHERE updated_at IS NULL;")
+    op.execute("UPDATE hubspot_deal_data SET updated_at = NOW() AT TIME ZONE 'utc' WHERE updated_at IS NULL;")
+
+    # 2️⃣ Now alter the column defaults and nullability
+    op.alter_column(
+        'hubspot_company_data',
+        'updated_at',
+        existing_type=sa.TIMESTAMP(timezone=True),
+        server_default=sa.text("NOW() AT TIME ZONE 'utc'"),
+        nullable=False,
+    )
+
+    op.alter_column(
+        'hubspot_deal_data',
+        'updated_at',
+        existing_type=sa.TIMESTAMP(timezone=True),
+        server_default=sa.text("NOW() AT TIME ZONE 'utc'"),
+        nullable=False,
+    )
+
+
+
+def downgrade() -> None:
+    """Downgrade schema."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.alter_column('hubspot_deal_data', 'updated_at',
+               existing_type=postgresql.TIMESTAMP(timezone=True),
+               server_default=None,
+               nullable=True)
+    op.alter_column('hubspot_company_data', 'updated_at',
+               existing_type=postgresql.TIMESTAMP(timezone=True),
+               server_default=None,
+               nullable=True)
+    # ### end Alembic commands ###

etl/hubSpotClient/scripts/hubspot_abri_etl_first_time.py

@@ -0,0 +1,45 @@
+from etl.hubSpotClient.hubspotClient import HubSpotClient, Companies, Pipeline
+from tqdm import tqdm
+from etl.db.hubSpotLoad import HubspotTodb
+
+'''
+# TODO:
+     get one deal from db, from db
+     for avri only so far
+     add it to the db
+     show in frontend
+'''
+
+# get ALL deals
+hubspot = HubSpotClient()
+
+# All deals from a pipeline_id via filter
+deals = hubspot.get_deal_ids_by_pipeline(
+    pipeline_id=Pipeline.OPERATIONS_SOCIAL_HOUSING.value,
+    )
+
+# deals from companies we care about
+valueable_deals = [
+    Companies.ABRI.value
+]
+deals_to_add = []
+
+
+deal_to_companies = {}
+loader = HubspotTodb()
+# Get all deals we care about
+for i,deal in enumerate(tqdm(deals)):
+    company = hubspot.from_deal_get_associated_company_id(deal)
+    if company in valueable_deals:
+        deals_to_add.append(deal)
+        deal_to_companies.update({deal: company})
+        deal_data = hubspot.from_deal_get_info(deal_id=deal)
+        listing_data = hubspot.from_deal_get_associated_listing(deal_id=deal)
+        loader.new_record_to_hubspot_data(deal_data, deal_to_companies[deal], listing_data)
+    
+
+
+#TODO check if database has abri data
+# make companies table
+# make a scrip that updates table
+

etl/hubSpotClient/scripts/hubspot_company.py

@@ -0,0 +1,15 @@
+from etl.hubSpotClient.hubspotClient import HubSpotClient, Companies, Pipeline
+from tqdm import tqdm
+from etl.db.hubSpotLoad import HubspotTodb
+
+hubspot = HubSpotClient()
+
+# All deals from a pipeline_id via filter
+company = hubspot.get_company_information(Companies.ABRI.value)
+
+loader = HubspotTodb()
+loader.new_record_company(company)
+    
+
+# make a scrip that updates table, with khalim scoping
+

etl/hubSpotClient/scripts/hubspot_update_abri_script.py

@@ -0,0 +1,23 @@
+from etl.hubSpotClient.hubspotClient import HubSpotClient, Companies, Pipeline
+from tqdm import tqdm
+from etl.db.hubSpotLoad import HubspotTodb
+
+hubspot = HubSpotClient()
+db = HubspotTodb()
+
+records = db.find_all_deals_with_company_id(Companies.ABRI.value)
+
+updated_count = 0  # Counter for deals that needed updating
+checked_count = 0  # Optional: total processed counter
+
+for deal in tqdm(records, desc="Checking HubSpot deals"):
+    checked_count += 1
+    was_up_to_date = db.update_deal(deal, hubspot)
+
+    # update_deal() returns False when discrepancies are found
+    if not was_up_to_date:
+        updated_count += 1
+
+print(f"\n✅ Finished checking {checked_count} deals.")
+print(f"🧩 {updated_count} deal(s) were updated.")
+print(f"📈 {checked_count - updated_count} deal(s) were already up to date.")