Model/etl/non_intrusive_surveys/photos/app.py

import boto3
import os
from PIL import Image
from pathlib import Path
from dotenv import load_dotenv

# Inputs
ENV_FILEPATH = "etl/non_intrusive_surveys/photos/.env"
PHOTO_DIRECTORY = "/Users/khalimconn-kowlessar/Downloads/IMMO - Dudley Pilot - non-invasive raw data"
FOLDER_UPRN_LOOKUP = {
    "91 Osprey Drive DY1 2JS": 90048026,
    "195 Ashenhurst Rd DY1 2JB": 90051858,
    "6 Beech Rd DY1 4BP": 90055152,
    "53 Bromley DY5 4PJ": 90060989,
    "5 Oaklands B62 0JA": 90028499,
    "47 Fairfield Rd DY8 5UJ": 90077535,
    "150 Huntingtree Rd B63 4HP": 90093693,
    "27 Milton Rd DY1 2JB": 90106884,
    "21 Wells Rd DY5 3TB": 90022227,
    "8 Corporation Rd DY2 7PX": 90070461
}

load_dotenv(ENV_FILEPATH)
CLOUDFRONT_DISTRIBUTION_DOMAIN_NAME = os.getenv("CLOUDFRONT_DISTRIBUTION_DOMAIN_NAME", None)
CDN_BUCKET_NAME = os.getenv("CDN_BUCKET_NAME", None)


def list_subdirectories(directory_path):
    """
    List all subdirectories within a given directory.

    :param directory_path: Path to the directory.
    :return: A list of paths to the subdirectories.
    """
    directory = Path(directory_path)
    subdirectories = [subdir for subdir in directory.iterdir() if subdir.is_dir()]
    return subdirectories


def list_files_in_directory(directory_path, file_extension=".jpg"):
    """
    List all files with a specific extension within a given directory and its subdirectories.

    :param directory_path: Path to the directory to scan.
    :param file_extension: File extension to filter by.
    :return: A list of paths to the files.
    """
    # Convert the directory path to a Path object if it's not already one
    directory = Path(directory_path) if not isinstance(directory_path, Path) else directory_path

    # List all files of the specified type in the directory and subdirectories
    file_list = [file for file in directory.rglob(f'*{file_extension}')]

    return file_list


def create_images(input_path, uprn):
    # Define the base directory path
    base_directory = f"non_intrusive_photos/{uprn}"
    print(f"Creating directory: {base_directory}")  # Debug: print the directory to be created

    # Need to create local directory if it doesn't exist
    os.makedirs(base_directory, exist_ok=True)

    # Define output paths
    thumbnail_path = os.path.join(base_directory, "thumbnail.jpg")
    full_hd_path = os.path.join(base_directory, "1080p.jpg")
    webp_path = os.path.join(base_directory, "webp.webp")  # Save as WebP format

    # Load the image
    with Image.open(input_path) as img:
        # Create a thumbnail
        thumbnail = img.copy()
        thumbnail.thumbnail((128, 128), Image.Resampling.LANCZOS)
        thumbnail.save(thumbnail_path, 'JPEG', quality=85)

        # Create a 1080p version
        full_hd = img.copy()
        full_hd.thumbnail((1920, 1080), Image.Resampling.LANCZOS)
        full_hd.save(full_hd_path, 'JPEG', quality=90)

        # Convert to WebP for better compression
        webp = img.copy()
        webp.save(webp_path, 'WEBP', quality=90)

    # Return paths to the processed images
    return thumbnail_path, full_hd_path, webp_path


def upload_to_s3(bucket_name, file_path, object_name):
    s3_client = boto3.client('s3')
    s3_client.upload_file(file_path, bucket_name, object_name)
    print(f"Uploaded {object_name} to S3 bucket {bucket_name}")


def upload_photos_to_s3(bucket_name, photo_paths):
    # Upload each photo
    for path in photo_paths:
        object_name = path.split('/')[-1]  # Assuming the path format is folder/filename
        upload_to_s3(bucket_name, path, object_name)


def generate_cdn_url(distribution_domain, object_name):
    return f"https://{distribution_domain}/{object_name}"


def process_and_upload_images(uprn, input_image_path, bucket_name, distribution_domain):
    # Create images
    thumbnail, full_hd, original = create_images(input_image_path, uprn=str(uprn))

    # Upload images
    upload_photos_to_s3(bucket_name, photo_paths=[thumbnail, full_hd, original])

    # Generate CDN links
    cdn_links = [generate_cdn_url(distribution_domain, path.split('/')[-1]) for path in [thumbnail, full_hd, original]]

    # Delete local files
    for path in [thumbnail, full_hd, original]:
        os.remove(path)

    return cdn_links


def app():
    """
    This application is tasked with uploading the photos, recorded during the non-invasive surveys, to s3 and the
    database.
    To begin with, this app will simply read the files from the local machine, however we will come up with a more
    efficient way to do this in the future.

    :return:
    """

    # List all files in the directory using pathlib
    property_directories = list_subdirectories(PHOTO_DIRECTORY)

    # For each property, we want to list all of the photos in the directory
    for property_dir in property_directories:
        photo_files = list_files_in_directory(property_dir)
        uprn = FOLDER_UPRN_LOOKUP[property_dir.name]

        # We now want to convert each file, and upload it to s3
        for photo_filepath in photo_files:
            process_and_upload_images(
                uprn=uprn,
                input_image_path=photo_filepath,
                bucket_name=CDN_BUCKET_NAME,
                distribution_domain=CLOUDFRONT_DISTRIBUTION_DOMAIN_NAME
            )