diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml index 301ea1a..5bceb59 100644 --- a/.devcontainer/docker-compose.yml +++ b/.devcontainer/docker-compose.yml @@ -11,35 +11,35 @@ services: - ../.db-env volumes: - ..:/workspaces/survey-extractor - depends_on: - - db + # depends_on: + # - db networks: - survey-net - db: - image: postgres:17.4 - restart: unless-stopped - ports: - - 5432:5432 - env_file: - - ../.db-env - volumes: - - postgres-data-two:/var/lib/postgresql/data - networks: - - survey-net + # db: + # image: postgres:17.4 + # restart: unless-stopped + # ports: + # - 5432:5432 + # env_file: + # - ../.db-env + # volumes: + # - postgres-data-two:/var/lib/postgresql/data + # networks: + # - survey-net - pgadmin: - image: dpage/pgadmin4 - hostname: pgadmin - ports: - - 5555:80 - env_file: - - ../.db-env - restart: unless-stopped - depends_on: - - db - networks: - - survey-net + # pgadmin: + # image: dpage/pgadmin4 + # hostname: pgadmin + # ports: + # - 5555:80 + # env_file: + # - ../.db-env + # restart: unless-stopped + # depends_on: + # - db + # networks: + # - survey-net networks: survey-net: diff --git a/.github/workflows/hubspot_sync.yml b/.github/workflows/hubspot_sync.yml index 7c86de3..c6bb1b3 100644 --- a/.github/workflows/hubspot_sync.yml +++ b/.github/workflows/hubspot_sync.yml @@ -2,7 +2,14 @@ name: Hubspot Sync on: schedule: - - cron: '0 2 * * *' # Every day at 06:00 UTC + # Every 15 minutes, 07:00–18:59, Monday–Friday (UTC) + - cron: '*/15 7-18 * * 1-5' + + # Once on Saturday at 09:00 UTC + - cron: '0 9 * * 6' + + # Once on Sunday at 09:00 UTC + - cron: '0 9 * * 0' workflow_dispatch: jobs: @@ -21,16 +28,6 @@ jobs: pip install poetry poetry install --no-root - - name: Run scripts - env: - PYTHONPATH: ${{ github.workspace }} - DATABASE_URL: ${{ secrets.PROD_DATABASE_URL }} - run: | - pwd - ls -la - poetry run python etl/hubSpotClient/scripts/hubspot_gather_all_deals.py - - # - name: Run scripts # env: # PYTHONPATH: ${{ github.workspace }} @@ -38,7 +35,17 @@ jobs: # run: | # pwd # ls -la - # poetry run python etl/hubSpotClient/scripts/hubspot_update_script.py + # poetry run python etl/hubSpotClient/scripts/hubspot_gather_all_deals.py + + + - name: Run scripts + env: + PYTHONPATH: ${{ github.workspace }} + DATABASE_URL: ${{ secrets.PROD_DATABASE_URL }} + run: | + pwd + ls -la + poetry run python etl/hubSpotClient/scripts/hubspot_update_script.py diff --git a/etl/db/hubSpotLoad.py b/etl/db/hubSpotLoad.py index 0c47a40..fad4741 100644 --- a/etl/db/hubSpotLoad.py +++ b/etl/db/hubSpotLoad.py @@ -138,6 +138,10 @@ class HubspotTodb: "company_id": company, "major_condition_issue_description": deal_data.get("major_condition_issue_description"), "major_condition_issue_photos": deal_data.get("major_condition_issue_photos"), + "major_condition_issue_description": deal_data.get("major_condition_issue_description"), + "major_condition_issue_photos": deal_data.get("major_condition_issue_photos"), + "coordination_status": deal_data.get("coordination_status__stage_1_"), + "design_status": deal_data.get("retrofit_design_status"), }.items(): setattr(existing, attr, value or getattr(existing, attr)) @@ -166,6 +170,8 @@ class HubspotTodb: company_id=company, major_condition_issue_description=deal_data.get("major_condition_issue_description"), major_condition_issue_photos=deal_data.get("major_condition_issue_photos"), + coordination_status=deal_data.get("coordination_status__stage_1_"), + design_status=deal_data.get("retrofit_design_status"), ) # Handle upload at insert time diff --git a/etl/hubSpotClient/hubspotClient.py b/etl/hubSpotClient/hubspotClient.py index 6fc4777..6bc3eab 100644 --- a/etl/hubSpotClient/hubspotClient.py +++ b/etl/hubSpotClient/hubspotClient.py @@ -161,6 +161,8 @@ class HubSpotClient(): 'project_code', 'major_condition_issue_description', 'major_condition_issue_photos', + 'coordination_status__stage_1_', # Coordiantion Status (Stage 1), + 'retrofit_design_status', # Retrofit Design Status ] ) diff --git a/etl/hubSpotClient/scripts/hubspot_update_script.py b/etl/hubSpotClient/scripts/hubspot_update_script.py index 2e1f4e3..ed893cb 100644 --- a/etl/hubSpotClient/scripts/hubspot_update_script.py +++ b/etl/hubSpotClient/scripts/hubspot_update_script.py @@ -7,10 +7,7 @@ hubspot.get_deal_stages() db = HubspotTodb() companies = [ - Companies.ABRI, - Companies.LIVEWEST, - Companies.SOUTHERN_HOUSING_GROUP, - Companies.SURESERVE, + Companies.THE_GUINESS_PARTNERSHIP, ] # Global trackers diff --git a/etl/models/topLevel.py b/etl/models/topLevel.py index cdf1f9f..a92c0c6 100644 --- a/etl/models/topLevel.py +++ b/etl/models/topLevel.py @@ -85,6 +85,9 @@ class HubspotDealData(SQLModel, table=True): major_condition_issue_photos: Optional[str] = Field(default=None) major_condition_issue_evidence_s3_url: Optional[str] = Field(default=None) + coordination_status: Optional[str] = Field(default=None) + design_status: Optional[str] = Field(default=None) + created_at: datetime = Field( sa_column=Column( DateTime(timezone=True), diff --git a/migration_db.sh b/migration_db.sh index c0046bc..392b58f 100644 --- a/migration_db.sh +++ b/migration_db.sh @@ -1,5 +1,4 @@ -#poetry run alembic revision --autogenerate -m "s3 add " - +#poetry run alembic revision --autogenerate -m "add coorodiantion and design status" poetry run alembic upgrade head # See which hash I'm at