diff --git a/.devcontainer/post-install.sh b/.devcontainer/post-install.sh index cf16197..473e420 100644 --- a/.devcontainer/post-install.sh +++ b/.devcontainer/post-install.sh @@ -1,5 +1,5 @@ #!/bin/bash -cd etl && poetry install; +poetry install; # Get the Poetry virtual environment path VENV_PATH=$(poetry env info --path 2>/dev/null) diff --git a/etl/main.py b/etl/main.py index 6bba6a7..755b0e2 100644 --- a/etl/main.py +++ b/etl/main.py @@ -19,9 +19,9 @@ def main(): # pprint(list_) # POC Scraper -> This part of the code get ths names of wrong format - #south_coast_scraper = SharePointScraper(SharePointInstaller.SOUTH_COAST_INSULATION_SERVICE) - #list_of_names = south_coast_scraper.list_of_names_that_has_the_wrong_date_format() - #logger.info(pformat(list_of_names)) + south_coast_scraper = SharePointScraper(SharePointInstaller.SOUTH_COAST_INSULATION_SERVICE) + list_of_names = south_coast_scraper.list_of_names_that_has_the_wrong_date_format() + logger.info(pformat(list_of_names)) # POC Scraper -> This part of the code gets every variation of housing_assocation names # south_coast_scraper = SharePointScraper(SharePointInstaller.SOUTH_COAST_INSULATION_SERVICE) diff --git a/etl/pdfReader/pdfReaderToText.py b/etl/pdfReader/pdfReaderToText.py index 9b87c74..2b0790d 100644 --- a/etl/pdfReader/pdfReaderToText.py +++ b/etl/pdfReader/pdfReaderToText.py @@ -1,4 +1,4 @@ -from utils.logger import Logger +from etl.utils.logger import Logger import logging import pymupdf diff --git a/etl/run_etl.sh b/etl/run_etl.sh deleted file mode 100644 index 28f6730..0000000 --- a/etl/run_etl.sh +++ /dev/null @@ -1 +0,0 @@ -poetry run python src/etl/main.py \ No newline at end of file diff --git a/etl/utils/sharepoint/sharepoint.py b/etl/utils/sharepoint/sharepoint.py index 80dc9b5..a2bb7d0 100644 --- a/etl/utils/sharepoint/sharepoint.py +++ b/etl/utils/sharepoint/sharepoint.py @@ -11,7 +11,7 @@ from functools import wraps import time import logging from io import BytesIO -from utils.logger import Logger +from etl.utils.logger import Logger logger = Logger(name="SharePoint.py", level=logging.DEBUG).get_logger() diff --git a/etl/validator/validator.py b/etl/validator/validator.py index b0b9409..45f9822 100644 --- a/etl/validator/validator.py +++ b/etl/validator/validator.py @@ -1,6 +1,6 @@ import os import logging -from utils.logger import Logger +from etl.utils.logger import Logger import re class DomnaSharePointValidator(): diff --git a/etl/poetry.lock b/poetry.lock similarity index 100% rename from etl/poetry.lock rename to poetry.lock diff --git a/etl/pyproject.toml b/pyproject.toml similarity index 89% rename from etl/pyproject.toml rename to pyproject.toml index 02cced5..0a5afb0 100644 --- a/etl/pyproject.toml +++ b/pyproject.toml @@ -13,7 +13,7 @@ dependencies = [ ] [tool.poetry] -packages = [{include = "etl", from = "src"}] +packages = [{include = "etl"}] [build-system] diff --git a/run_etl.sh b/run_etl.sh new file mode 100644 index 0000000..f130f2f --- /dev/null +++ b/run_etl.sh @@ -0,0 +1 @@ +poetry run python etl/main.py \ No newline at end of file