diff --git a/README.md b/README.md index 56faa33..af40814 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,11 @@ Dash + Github Actions (etl of hubspot) + S3 ( data stroage) + devcontainer for TODO: - [x] Make devcontainer with poetry python and frontend - [] Make a script that get all the hubspot data into a json + - [] Ask typhaine what deal properties she needs to gather this data + - Expected commnecment week etc + - [] Upload the data to json + - make a basic file upload to json and reader - [] Make the script run everyday at a certain time - [] Make a python dashboard that reads the latest json file and print it out - [] Show the results in a dashing board diff --git a/backend/src/dashboard/models/__init__.py b/backend/src/dashboard/models/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/src/dashboard/services/__init__.py b/backend/src/dashboard/services/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/src/dashboard/services/file_manager.py b/backend/src/dashboard/services/file_manager.py new file mode 100644 index 0000000..3c949aa --- /dev/null +++ b/backend/src/dashboard/services/file_manager.py @@ -0,0 +1,46 @@ +import os +import requests +import boto3 + +class FileManager: + def __init__(self, download_dir="downloads", aws_region="us-east-1"): + self.download_dir = download_dir + os.makedirs(download_dir, exist_ok=True) + self.s3 = boto3.client("s3", region_name=aws_region) + + def download_file(self, url: str, filename: str = None) -> str: + """ + Download a file from a URL and return the local file path. + """ + if not filename: + filename = url.split("/")[-1] or "downloaded_file" + + filepath = os.path.join(self.download_dir, filename) + + response = requests.get(url, stream=True) + response.raise_for_status() + + with open(filepath, "wb") as f: + for chunk in response.iter_content(chunk_size=8192): + if chunk: + f.write(chunk) + + return filepath + + def upload_to_s3(self, file_path: str, bucket: str, object_name: str = None): + """ + Upload a file to S3. + """ + if object_name is None: + object_name = os.path.basename(file_path) + + self.s3.upload_file(file_path, bucket, object_name) + return f"s3://{bucket}/{object_name}" + + +# Example usage: +# fm = FileManager() +# local_path = fm.download_file("https://example.com/file.txt") +# print("Saved to:", local_path) +# s3_path = fm.upload_to_s3(local_path, "my-bucket") +# print("Uploaded to:", s3_path) diff --git a/backend/src/dashboard/utils/__init__.py b/backend/src/dashboard/utils/__init__.py new file mode 100644 index 0000000..e69de29