From 96295f1c3be32e1b42617e9acb7612f688e2f4d4 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Wed, 19 Jul 2023 13:13:50 +0100 Subject: [PATCH 1/2] testing creation of a combined requirements file for serverless deployment --- make_serverless_requirements.sh | 23 ++++++++++++++++ serverless.yml | 2 +- serverless_requirements.txt | 49 +++++++++++++++++++++++++++++++-- 3 files changed, 71 insertions(+), 3 deletions(-) create mode 100644 make_serverless_requirements.sh diff --git a/make_serverless_requirements.sh b/make_serverless_requirements.sh new file mode 100644 index 00000000..690cc82c --- /dev/null +++ b/make_serverless_requirements.sh @@ -0,0 +1,23 @@ +#!/bin/bash +# This script combines the requirements from base.txt and requirements.txt into serverless_requirements.txt + +# Navigate to the directory containing this script +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +cd "$DIR" + +# Concatenate the requirements files with a newline character in between +cat backend/requirements/base.txt > serverless_requirements.txt +echo "" >> serverless_requirements.txt +cat model_data/requirements/requirements.txt >> serverless_requirements.txt + +# Identify duplicates +duplicates=$(sort serverless_requirements.txt | uniq -d) + +if [ -n "$duplicates" ]; then + echo "Warning: Duplicate entries found:" + echo "$duplicates" + echo "Please resolve these conflicts manually in the combined requirements file." +fi + +# Print a success message +echo "Combined requirements files into serverless_requirements.txt" diff --git a/serverless.yml b/serverless.yml index 8cf60038..b91a7371 100644 --- a/serverless.yml +++ b/serverless.yml @@ -14,7 +14,7 @@ provider: package: - # individually: true + individually: true # include: # - backend/** # # Might need to refine the paths that are included diff --git a/serverless_requirements.txt b/serverless_requirements.txt index 11ec7436..762019ef 100644 --- a/serverless_requirements.txt +++ b/serverless_requirements.txt @@ -1,2 +1,47 @@ --r backend/requirements/base.txt --r model_data/requirements/requirements.txt \ No newline at end of file +anyio==3.7.1 +cffi==1.15.1 +click==8.1.3 +cryptography==37.0.4 +ecdsa==0.18.0 +exceptiongroup==1.1.2 +fastapi==0.99.1 +h11==0.14.0 +httptools==0.5.0 +idna==3.4 +mangum==0.17.0 +pyasn1==0.5.0 +pycparser==2.21 +pydantic==1.10.11 +PyJWT==2.7.0 +python-dotenv==1.0.0 +python-jose==3.3.0 +PyYAML==6.0 +rsa==4.9 +six==1.16.0 +sniffio==1.3.0 +starlette==0.27.0 +typing_extensions==4.7.1 +uvicorn==0.22.0 +uvloop==0.17.0 +watchfiles==0.19.0 +websockets==11.0.3 +boto3 +pandas==2.0.3 +numpy==1.25.1 +pytz==2023.3 +tzdata==2023.3 +epc-api-python==1.0.2 +tqdm +mypy +fuzzywuzzy +python-Levenshtein +dbfread +pyproj +pint +mip +seaborn +statsmodels +scikit-learn +pyspellchecker +textblob +xgboost \ No newline at end of file From 5975bede3c9f39a6bd8f4f2074f979becbc841c3 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Wed, 19 Jul 2023 13:49:46 +0100 Subject: [PATCH 2/2] getting deployment working --- backend/requirements/base.txt | 3 +-- model_data/Property.py | 5 +++-- model_data/requirements/dev.txt | 1 + model_data/requirements/requirements.txt | 8 +------- model_data/requirements/static.txt | 7 ++++++- serverless.yml | 1 + serverless_requirements.txt | 9 +-------- 7 files changed, 14 insertions(+), 20 deletions(-) diff --git a/backend/requirements/base.txt b/backend/requirements/base.txt index 5a1693c4..cab350c7 100644 --- a/backend/requirements/base.txt +++ b/backend/requirements/base.txt @@ -24,5 +24,4 @@ typing_extensions==4.7.1 uvicorn==0.22.0 uvloop==0.17.0 watchfiles==0.19.0 -websockets==11.0.3 -boto3 \ No newline at end of file +websockets==11.0.3 \ No newline at end of file diff --git a/model_data/Property.py b/model_data/Property.py index 98b7a0b0..9fb46639 100644 --- a/model_data/Property.py +++ b/model_data/Property.py @@ -1,4 +1,4 @@ -import pandas as pd +from datetime import datetime import re from epc_api.client import EpcClient from model_data.config import EPC_AUTH_TOKEN @@ -130,7 +130,8 @@ class Property(BaseUtility): """ if self.full_sap_epc: - self.year_built = pd.to_datetime(self.full_sap_epc["lodgement-date"]).year + self.year_built = datetime.strptime(self.full_sap_epc["lodgement-date"], '%Y-%m-%d').year + return if self.data["construction-age-band"] not in self.DATA_ANOMALY_MATCHES: diff --git a/model_data/requirements/dev.txt b/model_data/requirements/dev.txt index c48188c6..9b941c18 100644 --- a/model_data/requirements/dev.txt +++ b/model_data/requirements/dev.txt @@ -4,3 +4,4 @@ mock pytest-cov pytest-mock pip-check-reqs +seaborn diff --git a/model_data/requirements/requirements.txt b/model_data/requirements/requirements.txt index 42e3e369..09572e62 100644 --- a/model_data/requirements/requirements.txt +++ b/model_data/requirements/requirements.txt @@ -1,5 +1,3 @@ -pandas==2.0.3 -numpy==1.25.1 pytz==2023.3 tzdata==2023.3 epc-api-python==1.0.2 @@ -11,9 +9,5 @@ dbfread pyproj pint mip -seaborn -statsmodels -scikit-learn pyspellchecker -textblob -xgboost \ No newline at end of file +textblob \ No newline at end of file diff --git a/model_data/requirements/static.txt b/model_data/requirements/static.txt index b9b8e7c5..e2ffa2ce 100644 --- a/model_data/requirements/static.txt +++ b/model_data/requirements/static.txt @@ -1 +1,6 @@ -geopandas \ No newline at end of file +geopandas +xgboost +statsmodels +scikit-learn +pandas==2.0.3 +numpy==1.25.1 \ No newline at end of file diff --git a/serverless.yml b/serverless.yml index b91a7371..81aab643 100644 --- a/serverless.yml +++ b/serverless.yml @@ -21,6 +21,7 @@ package: # - model_data/** exclude: - model_data/local_data/** + - model_data/tests/** - infrastructure/** - data_collection/** - node_modules/** diff --git a/serverless_requirements.txt b/serverless_requirements.txt index 762019ef..579cd5af 100644 --- a/serverless_requirements.txt +++ b/serverless_requirements.txt @@ -25,9 +25,6 @@ uvicorn==0.22.0 uvloop==0.17.0 watchfiles==0.19.0 websockets==11.0.3 -boto3 -pandas==2.0.3 -numpy==1.25.1 pytz==2023.3 tzdata==2023.3 epc-api-python==1.0.2 @@ -39,9 +36,5 @@ dbfread pyproj pint mip -seaborn -statsmodels -scikit-learn pyspellchecker -textblob -xgboost \ No newline at end of file +textblob \ No newline at end of file