diff --git a/etl/src/etl/validator/__init__.py b/etl/src/etl/filePathValidator/__init__.py similarity index 100% rename from etl/src/etl/validator/__init__.py rename to etl/src/etl/filePathValidator/__init__.py diff --git a/etl/src/etl/validator/retrohome.py b/etl/src/etl/filePathValidator/retrohome.py similarity index 87% rename from etl/src/etl/validator/retrohome.py rename to etl/src/etl/filePathValidator/retrohome.py index 6aa0b7a..a4be660 100644 --- a/etl/src/etl/validator/retrohome.py +++ b/etl/src/etl/filePathValidator/retrohome.py @@ -1,16 +1,17 @@ import os +import logging from utils.logger import Logger class RetroHomeFileStructureValidator(): def __init__(self, source_loc_path): self.source_path = source_loc_path - self.logger = Logger(name='RetroHomeFileStructureValidator').get_logger() + self.logger = Logger(name='RetroHomeFileStructureValidator', level=logging.DEBUG).get_logger() self.innocent = [] self.guilty = [] self.validate() def validate(self): - self.logger.info(f"Starting File Structure Validation on '{self.source_path}'") + self.logger.debug(f"Starting File Structure Validation on '{self.source_path}'") for filepath in os.listdir(self.source_path): if os.path.isdir(os.path.join(self.source_path, filepath)): @@ -18,7 +19,7 @@ class RetroHomeFileStructureValidator(): else: self.logger.warning(f"Found a file when expecting directory. Ignoring file {filepath}") - self.logger.info(self.innocent) + self.logger.debug(self.innocent) self.valid_name() self.valid_file_structure() diff --git a/etl/src/etl/main.py b/etl/src/etl/main.py index 588f4ef..49b20b7 100644 --- a/etl/src/etl/main.py +++ b/etl/src/etl/main.py @@ -1,5 +1,5 @@ import os -from validator.retrohome import RetroHomeFileStructureValidator +from filePathValidator.retrohome import RetroHomeFileStructureValidator DATA_LOC = "/workspaces/survey-extraction/data/" INTERESTING_FILE_LOC = "/workspaces/survey-extraction/data/first last/Submission 03.03.25/customer/10 Sandbeck Lane DN21 3LZ/PRE SITE NOTES.pdf"