Model/backend/app/db/functions/non_intrusive_surveys.py

from sqlalchemy.orm import Session
from backend.app.db.models.non_intrusive_surveys import NonIntrusiveSurvey, NonIntrusiveSurveyNotes


def upload_non_intrusive_survey_notes(session: Session, non_invasive_notes, batch_size=500):
    """
    Uploads a list of non-intrusive survey notes into the database in batches. Each dictionary in the list represents
    one survey and its associated notes.

    :param session: SQLAlchemy Session object through which all database transactions are handled.
    :param non_invasive_notes: List of dictionaries where each dictionary contains survey details including 'uprn',
                               'survey_date', 'surveyor', and other notes as key-value pairs.
    :param batch_size: The size of each batch to be processed (default is 500).
    :return: None
    """

    # Helper function to process each batch
    def process_batch(batch):
        surveys = []
        notes = []

        for note in batch:
            survey = NonIntrusiveSurvey(
                uprn=note['uprn'],
                survey_date=note['survey_date'],
                surveyor=note['surveyor']
            )
            surveys.append(survey)

        session.add_all(surveys)
        session.flush()  # Get IDs for surveys

        for note, survey in zip(batch, surveys):
            for key, value in note.items():
                if key not in ['uprn', 'survey_date', 'surveyor']:
                    notes.append(NonIntrusiveSurveyNotes(
                        survey_id=survey.id,
                        title=key,
                        note=value
                    ))

        session.bulk_save_objects(notes)
        session.commit()

    # Split the data into batches and process each batch
    total = len(non_invasive_notes)
    for start in range(0, total, batch_size):
        end = min(start + batch_size, total)
        batch = non_invasive_notes[start:end]
        process_batch(batch)