From baf93e220df7228896d88b462500006ae38215f2 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Sat, 8 Mar 2025 06:29:07 +0000 Subject: [PATCH] moved files --- .devcontainer/post-install.sh | 26 +++++++++++++++ etl/{src/etl => }/__init__.py | 0 etl/{src/etl => }/main.py | 32 ++++++++++++++----- etl/{src/etl => }/pdfReader/__init__.py | 0 .../etl => }/pdfReader/pdfReaderToText.py | 0 etl/{src/etl => }/scraper/__init__.py | 0 etl/{src/etl => }/scraper/scraper.py | 9 ++++-- etl/{src/etl => }/utils/logger.py | 0 .../etl => }/utils/sharepoint/__init__.py | 0 .../etl => }/utils/sharepoint/sharepoint.py | 0 etl/{src/etl => }/validator/__init__.py | 0 etl/{src/etl => }/validator/validator.py | 0 12 files changed, 57 insertions(+), 10 deletions(-) rename etl/{src/etl => }/__init__.py (100%) rename etl/{src/etl => }/main.py (69%) rename etl/{src/etl => }/pdfReader/__init__.py (100%) rename etl/{src/etl => }/pdfReader/pdfReaderToText.py (100%) rename etl/{src/etl => }/scraper/__init__.py (100%) rename etl/{src/etl => }/scraper/scraper.py (97%) rename etl/{src/etl => }/utils/logger.py (100%) rename etl/{src/etl => }/utils/sharepoint/__init__.py (100%) rename etl/{src/etl => }/utils/sharepoint/sharepoint.py (100%) rename etl/{src/etl => }/validator/__init__.py (100%) rename etl/{src/etl => }/validator/validator.py (100%) diff --git a/.devcontainer/post-install.sh b/.devcontainer/post-install.sh index c441450..cf16197 100644 --- a/.devcontainer/post-install.sh +++ b/.devcontainer/post-install.sh @@ -1 +1,27 @@ +#!/bin/bash cd etl && poetry install; + +# Get the Poetry virtual environment path +VENV_PATH=$(poetry env info --path 2>/dev/null) + +if [ -z "$VENV_PATH" ]; then + echo "No Poetry environment found. Did you run 'poetry install'?" + exit 1 +fi + +# Ensure VS Code settings directory exists +SETTINGS_DIR="/home/vscode/.vscode-server/data/Machine" +SETTINGS_FILE="$SETTINGS_DIR/settings.json" + +mkdir -p "$SETTINGS_DIR" + +# If settings.json doesn't exist, create a default one +if [ ! -f "$SETTINGS_FILE" ]; then + echo "{}" > "$SETTINGS_FILE" +fi + +# Update VS Code settings to use the Poetry virtual environment +jq --arg venv "$VENV_PATH/bin/python" '.["python.defaultInterpreterPath"] = $venv' \ + "$SETTINGS_FILE" > "$SETTINGS_FILE.tmp" && mv "$SETTINGS_FILE.tmp" "$SETTINGS_FILE" + +echo "✅ Updated VS Code to use Poetry environment: $VENV_PATH" diff --git a/etl/src/etl/__init__.py b/etl/__init__.py similarity index 100% rename from etl/src/etl/__init__.py rename to etl/__init__.py diff --git a/etl/src/etl/main.py b/etl/main.py similarity index 69% rename from etl/src/etl/main.py rename to etl/main.py index b5ceddd..6bba6a7 100644 --- a/etl/src/etl/main.py +++ b/etl/main.py @@ -1,10 +1,10 @@ import os from pdfReader.pdfReaderToText import pdfReaderToText -from scraper.scraper import SharePointScraper, SharePointInstaller +from etl.scraper.scraper import SharePointScraper, SharePointInstaller from pprint import pprint, pformat import logging import tempfile -from utils.logger import Logger +from etl.utils.logger import Logger from etl.validator.validator import DomnaSharePointValidator @@ -29,9 +29,16 @@ def main(): # logger.info(pformat(list_of_house_ass_names)) # POC of work completed - south_coast_scraper = SharePointScraper(SharePointInstaller.SOUTH_COAST_INSULATION_SERVICE) - surverys_completed = south_coast_scraper.get_number_of_surverys_completed() - logger.info(pformat(surverys_completed)) + # south_coast_scraper = SharePointScraper(SharePointInstaller.SOUTH_COAST_INSULATION_SERVICE) + # surverys_completed = south_coast_scraper.get_number_of_surverys_completed() + # logger.info(pformat(surverys_completed)) + print("hello world") + print("1") + a = 1 + import sys + print(sys.executable) + print("Current working directory:", os.getcwd()) + # def create_temp_file(dict_content): # with tempfile.NamedTemporaryFile(mode='w+', delete=False, suffix='.txt') as temp_file: @@ -46,12 +53,21 @@ if __name__ == "__main__": main() -# Make a validaation checker for housing assosciation - # Get and read files +# CSV file of the survey number + + # Work out productivity metirc (number of address in submission folder, with at least one file included) # Khalim would like these metrics from the pdf -# address, uprn, assessor's name validation, current sap rating, current annual emissions. DImension \ No newline at end of file +# address, uprn, assessor's name validation, current sap rating, current annual emissions. DImension + + + +# next week mvp +# Gather file data form sharepoint +# Start making a scheme or csv with the data we 'care about' +# {id: uuid, 'surveryor_name': 'khalim', 'address': '123 fake street', 'prorperty_type': 'house', sharepoint_loc} +# data base of historical data. SQL data \ No newline at end of file diff --git a/etl/src/etl/pdfReader/__init__.py b/etl/pdfReader/__init__.py similarity index 100% rename from etl/src/etl/pdfReader/__init__.py rename to etl/pdfReader/__init__.py diff --git a/etl/src/etl/pdfReader/pdfReaderToText.py b/etl/pdfReader/pdfReaderToText.py similarity index 100% rename from etl/src/etl/pdfReader/pdfReaderToText.py rename to etl/pdfReader/pdfReaderToText.py diff --git a/etl/src/etl/scraper/__init__.py b/etl/scraper/__init__.py similarity index 100% rename from etl/src/etl/scraper/__init__.py rename to etl/scraper/__init__.py diff --git a/etl/src/etl/scraper/scraper.py b/etl/scraper/scraper.py similarity index 97% rename from etl/src/etl/scraper/scraper.py rename to etl/scraper/scraper.py index 70c17ea..0f343c1 100644 --- a/etl/src/etl/scraper/scraper.py +++ b/etl/scraper/scraper.py @@ -1,10 +1,10 @@ from pprint import pformat from enum import Enum import os -from utils.logger import Logger +from etl.utils.logger import Logger import logging # Awesome Khalim's Sharepoint code that I'll probably will need to learn how it work in the future -from utils.sharepoint.sharepoint import SharePointClient +from etl.utils.sharepoint.sharepoint import SharePointClient from functools import wraps import re from etl.validator.validator import DomnaSharePointValidator @@ -42,6 +42,7 @@ class SharePointScraper(): self.surveyor_work_completed = {} + def ensure_surveyor_names_loaded(func): """Decorator to ensure surveyor names are loaded before calling the function.""" @wraps(func) @@ -174,3 +175,7 @@ class SharePointScraper(): self.surveyor_work_completed.update({name: len(address_folders['value'])}) return self.surveyor_work_completed + + @ensure_housing_assosiation_is_loaded + def download_file_for_each_address(self): + pass \ No newline at end of file diff --git a/etl/src/etl/utils/logger.py b/etl/utils/logger.py similarity index 100% rename from etl/src/etl/utils/logger.py rename to etl/utils/logger.py diff --git a/etl/src/etl/utils/sharepoint/__init__.py b/etl/utils/sharepoint/__init__.py similarity index 100% rename from etl/src/etl/utils/sharepoint/__init__.py rename to etl/utils/sharepoint/__init__.py diff --git a/etl/src/etl/utils/sharepoint/sharepoint.py b/etl/utils/sharepoint/sharepoint.py similarity index 100% rename from etl/src/etl/utils/sharepoint/sharepoint.py rename to etl/utils/sharepoint/sharepoint.py diff --git a/etl/src/etl/validator/__init__.py b/etl/validator/__init__.py similarity index 100% rename from etl/src/etl/validator/__init__.py rename to etl/validator/__init__.py diff --git a/etl/src/etl/validator/validator.py b/etl/validator/validator.py similarity index 100% rename from etl/src/etl/validator/validator.py rename to etl/validator/validator.py