From b3983fd7b28491c49938cb615677de8e5c1ea4ac Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Mon, 17 Mar 2025 08:30:11 +0000 Subject: [PATCH] check this submissions --- etl/daily_script.py | 16 ++++++++++++++-- etl/scraper/scraper.py | 2 +- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/etl/daily_script.py b/etl/daily_script.py index 4365e4a..27e3df1 100644 --- a/etl/daily_script.py +++ b/etl/daily_script.py @@ -1,10 +1,12 @@ import os from pdfReader.pdfReaderToText import pdfReaderToText -from etl.scraper.scraper import SharePointScraper, SharePointInstaller +from etl.scraper.scraper import SharePointScraper, SharePointInstaller, WEEK_COMMENCING from pprint import pprint, pformat import logging from etl.utils.logger import Logger from etl.validator.validator import DomnaSharePointValidator +from collections import Counter + logger = Logger(name="main.py", level=logging.INFO).get_logger() @@ -16,17 +18,23 @@ def main(): # Correct dates format checker south_coast_scraper = SharePointScraper(SharePointInstaller.SOUTH_COAST_INSULATION) south_coast_names = south_coast_scraper.list_of_names_that_has_the_wrong_date_format() + south_coast_submissions = south_coast_scraper.get_number_of_surverys_completed() jjc_scraper = SharePointScraper(SharePointInstaller.JJC) jjc_names = jjc_scraper.list_of_names_that_has_the_wrong_date_format() + jjc_coast_submission = jjc_scraper.get_number_of_surverys_completed() SGEC = SharePointScraper(SharePointInstaller.SGEC) sgec_names = SGEC.list_of_names_that_has_the_wrong_date_format() + sgec_submission = SGEC.get_number_of_surverys_completed() BAXTER_KELLY = SharePointScraper(SharePointInstaller.BAXTER_KELLY) b_names = BAXTER_KELLY.list_of_names_that_has_the_wrong_date_format() + BAXTER_KELLY_submissions = BAXTER_KELLY.get_number_of_surverys_completed() - + total_dict = dict(Counter(south_coast_submissions) + Counter(jjc_coast_submission) + Counter(sgec_submission) + Counter(BAXTER_KELLY_submissions)) + + logger.info("-------------------------------------------") logger.info("Good morning Cyrus") if south_coast_names: logger.info("South Coast with wrong date format:") @@ -44,6 +52,10 @@ def main(): if b_names: logger.info("Baxter Kelly with wrong date format") logger.info(pformat(b_names)) + logger.info("-------------------------------------------") + logger.info(f"For week commencing: {WEEK_COMMENCING}") + logger.info(f"Submissions: {pformat(total_dict)}") + logger.info("-------------------------------------------") # Make a quick script that checks if the Pictures folder exists in a certain fail directory diff --git a/etl/scraper/scraper.py b/etl/scraper/scraper.py index de7a9ac..0fb83a8 100644 --- a/etl/scraper/scraper.py +++ b/etl/scraper/scraper.py @@ -9,7 +9,7 @@ from functools import wraps import re from etl.validator.validator import DomnaSharePointValidator -WEEK_COMMENCING = os.getenv("WEEK_COMMENCING", "W.C. 03.03.2025") +WEEK_COMMENCING = os.getenv("WEEK_COMMENCING", "W.C. 10.03.2025") class SharePointInstaller(Enum): SOUTH_COAST_INSULATION = os.getenv("SOUTH_COAST_INSULATION_SERVICE_SHAREPOINT_ID", None)