from sqlalchemy.orm import Session from backend.app.db.models.non_intrusive_surveys import NonIntrusiveSurvey, NonIntrusiveSurveyNotes def upload_non_intrusive_survey_notes(session: Session, non_invasive_notes, batch_size=500): """ Uploads a list of non-intrusive survey notes into the database in batches. Each dictionary in the list represents one survey and its associated notes. :param session: SQLAlchemy Session object through which all database transactions are handled. :param non_invasive_notes: List of dictionaries where each dictionary contains survey details including 'uprn', 'survey_date', 'surveyor', and other notes as key-value pairs. :param batch_size: The size of each batch to be processed (default is 500). :return: None """ # Helper function to process each batch def process_batch(batch): surveys = [] notes = [] for note in batch: survey = NonIntrusiveSurvey( uprn=note['uprn'], survey_date=note['survey_date'], surveyor=note['surveyor'] ) surveys.append(survey) session.add_all(surveys) session.flush() # Get IDs for surveys for note, survey in zip(batch, surveys): for key, value in note.items(): if key not in ['uprn', 'survey_date', 'surveyor']: notes.append(NonIntrusiveSurveyNotes( survey_id=survey.id, title=key, note=value )) session.bulk_save_objects(notes) session.commit() # Split the data into batches and process each batch total = len(non_invasive_notes) for start in range(0, total, batch_size): end = min(start + batch_size, total) batch = non_invasive_notes[start:end] process_batch(batch)