diff --git a/etl/db/__init__.py b/etl/db/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/etl/db/db.py b/etl/db/db.py new file mode 100644 index 0000000..1fd7621 --- /dev/null +++ b/etl/db/db.py @@ -0,0 +1,22 @@ +from sqlmodel import SQLModel, create_engine, Session +from pydantic_settings import BaseSettings +from typing import Optional, List + +class Settings(BaseSettings): + DATABASE_URL: Optional[str] = None # Default to None if not set + + class Config: + env_file = ".env" # Load from an optional .env file + +settings = Settings() +engine = create_engine(settings.DATABASE_URL) if settings.DATABASE_URL else None + + +def get_db_session(): + if engine is None: + raise RuntimeError("Database is not configured. Set DATABASE_URL in environment variables.") + return Session(engine) + +def init_db(): + if engine: + SQLModel.metadata.create_all(engine) \ No newline at end of file diff --git a/etl/load.py b/etl/load.py index d4b456a..b8f8c64 100644 --- a/etl/load.py +++ b/etl/load.py @@ -1,38 +1,23 @@ # A very simple script to read a presite note and load to a postgres database import os from etl.surveyedData.surveryedData import surveyedDataProcessor -from etl.transform.types import get_db_session, init_db, AssessorInfo +from etl.db.db import get_db_session, init_db +from etl.transform.types import AssessorInfo + pre_site_note_path = os.path.join(os.getcwd(), "..", "example_data", "pre_site_note.pdf") -survey_one = surveyedDataProcessor("123 Fake Street", [pre_site_note_path]) +survey_one = surveyedDataProcessor("122 Fake Street", [pre_site_note_path]) assert survey_one.pre_site_note.assessor_information.accreditation_number == "QUID211435" init_db() -assessor1 = AssessorInfo( - accreditation_number=survey_one.pre_site_note.assessor_information.accreditation_number, - name=survey_one.pre_site_note.assessor_information.name, - phone_number=survey_one.pre_site_note.assessor_information.phone_number, - email_address=survey_one.pre_site_note.assessor_information.email_address, + +assessor0 = AssessorInfo( + **survey_one.pre_site_note.assessor_information.__dict__ ) with get_db_session() as session: - session.add(assessor1) + session.add(assessor0) session.commit() - - - - - - - - - - - -# Download a pdf file and save it in git for easier processes and tests -# scrape the correct data -# Pydantic structure -# Load to database successfully locally \ No newline at end of file diff --git a/etl/transform/types.py b/etl/transform/types.py index dada10e..3342bee 100644 --- a/etl/transform/types.py +++ b/etl/transform/types.py @@ -1,31 +1,11 @@ -from sqlmodel import Field, SQLModel, create_engine, Relationship, Session +from sqlmodel import Field, SQLModel import uuid from datetime import datetime from pydantic import field_validator, EmailStr -from pydantic_settings import BaseSettings from typing import Optional, List from sqlalchemy import Column from sqlalchemy.dialects.postgresql import UUID -class Settings(BaseSettings): - DATABASE_URL: Optional[str] = None # Default to None if not set - - class Config: - env_file = ".env" # Load from an optional .env file - -settings = Settings() -engine = create_engine(settings.DATABASE_URL) if settings.DATABASE_URL else None - - -def get_db_session(): - if engine is None: - raise RuntimeError("Database is not configured. Set DATABASE_URL in environment variables.") - return Session(engine) - -def init_db(): - if engine: - SQLModel.metadata.create_all(engine) - class BaseModel(SQLModel): id: uuid.UUID = Field( default_factory=uuid.uuid4,