diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index fbf7e21..927b1ca 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -3,6 +3,12 @@ FROM library/python:3.12-bullseye ARG USER=vscode ARG DEBIAN_FRONTEND=noninteractive +# DO NOT PUSH IMAGE TO ECR!!! as anyone with access to image can log on to our aws +# Will log on as aws Jun-te account, change in the future to development account +ENV AWS_ACCESS_KEY_ID=AKIAU5A36PPNK7RXX52V +ENV AWS_SECRET_ACCESS_KEY=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 +ENV AWS_DEFAULT_REGION=eu-west-2 + # Install system dependencies in a single layer RUN apt update && apt install -y --no-install-recommends \ sudo jq vim curl\ diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index d050974..dac0087 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -6,9 +6,8 @@ "workspaceFolder": "/workspaces/survey-extractor", "postStartCommand": "bash .devcontainer/post-install.sh", "mounts": [ - "source=${localEnv:HOME},target=/workspaces/home,type=bind", - // Make sure you aws credentials are saved at ~/.aws - "source=${localEnv:HOME}/.aws/,target=/home/vscode/.aws/,type=bind" + // Optional, just makes getting from Downloads (local env) easier + "source=${localEnv:HOME},target=/workspaces/home,type=bind" ], "customizations": { "vscode": { diff --git a/.github/workflows/actions/lambda-deploy/action.yml b/.github/workflows/actions/lambda-deploy/action.yml new file mode 100644 index 0000000..ba19c67 --- /dev/null +++ b/.github/workflows/actions/lambda-deploy/action.yml @@ -0,0 +1,86 @@ +name: "Build and Push Lambda Image to ECR" +description: "Reusable action for building and pushing lambda Docker image to ECR" + +inputs: + lambda_name: + description: "Lambda name / ECR repo name" + required: true + dockerfile_path: + description: "Path to Dockerfile" + required: true + ecr_tf_dir: + description: "Path to ECR terraform directory" + required: true + lambda_tf_dir: + description: "Path to Lambda terraform directory" + required: true + aws-access-key-id: + description: "AWS access key" + required: true + aws-secret-access-key: + description: "AWS secret key" + required: true + aws-region: + description: "AWS region" + required: true + git-sha: + description: "Git commit SHA" + required: true + git-ref: + description: "Git ref name" + required: true + +runs: + using: "composite" + steps: + - uses: actions/checkout@v4 + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-access-key-id: ${{ inputs.aws-access-key-id }} + aws-secret-access-key: ${{ inputs.aws-secret-access-key }} + aws-region: ${{ inputs.aws-region }} + + - name: Log in to Amazon ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@v2 + + - name: Deploy ECR + uses: ./.github/workflows/actions/terraform-deploy + with: + working_directory: ${{ inputs.ecr_tf_dir }} + aws-access-key-id: ${{ inputs.aws-access-key-id }} + aws-secret-access-key: ${{ inputs.aws-secret-access-key }} + aws-region: ${{ inputs.aws-region }} + - name: Set Docker image tag + id: set_tag + shell: bash + run: | + SHORT_SHA=$(echo "${{ inputs.git-sha }}" | cut -c1-7) + BRANCH=$(echo "${{ inputs.git-ref }}" | tr '/' '-') + TAG="${BRANCH}-${SHORT_SHA}" + echo "IMAGE_TAG=${TAG}" >> $GITHUB_ENV + echo "tag=$TAG" >> $GITHUB_OUTPUT + + - name: Build and push Docker image + shell: bash + run: | + IMAGE_URI=${{ steps.login-ecr.outputs.registry }}/${{ inputs.lambda_name }}:${{ steps.set_tag.outputs.tag }} + echo "Building Docker image for ${{ inputs.lambda_name }}..." + docker build -t $IMAGE_URI -f ${{ inputs.dockerfile_path }} . + + echo "Pushing to ECR..." + docker push $IMAGE_URI + + - name: Deploy Lambda + uses: ./.github/workflows/actions/terraform-deploy + with: + working_directory: ${{ inputs.lambda_tf_dir }} + aws-access-key-id: ${{ inputs.aws-access-key-id }} + aws-secret-access-key: ${{ inputs.aws-secret-access-key }} + aws-region: ${{ inputs.aws-region }} + lambda-image-tag: ${{ steps.set_tag.outputs.tag }} + + + diff --git a/.github/workflows/actions/terraform-deploy/action.yml b/.github/workflows/actions/terraform-deploy/action.yml new file mode 100644 index 0000000..685a0ac --- /dev/null +++ b/.github/workflows/actions/terraform-deploy/action.yml @@ -0,0 +1,54 @@ +name: "Terraform Plan Shared Config" +description: "Plans shared Terraform config for Lambdas" + +inputs: + working_directory: + description: "Directory containing Terraform config" + required: true + aws-access-key-id: + description: "AWS access key" + required: true + aws-secret-access-key: + description: "AWS secret key" + required: true + aws-region: + description: "AWS region" + required: true + lambda-image-tag: + description: "Tag of the Lambda image (e.g., GitHub SHA)" + required: false + +runs: + using: "composite" + steps: + - uses: actions/checkout@v4 + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-access-key-id: ${{ inputs.aws-access-key-id }} + aws-secret-access-key: ${{ inputs.aws-secret-access-key }} + aws-region: ${{ inputs.aws-region }} + + - name: Setup Terraform + uses: hashicorp/setup-terraform@v3 + + - name: Terraform Init + working-directory: ${{ inputs.working_directory }} + shell: bash + run: terraform init -reconfigure + + - name: Terraform Plan + working-directory: ${{ inputs.working_directory }} + shell: bash + run: | + if [ -n "${{ inputs.lambda-image-tag }}" ]; then + terraform plan -out=tfplan -var="lambda_image_tag=${{ inputs.lambda-image-tag }}" + else + terraform plan -out=tfplan + fi + + - name: Terraform Apply + working-directory: ${{ inputs.working_directory }} + shell: bash + run: terraform apply -auto-approve tfplan diff --git a/.github/workflows/lambda_main.yml b/.github/workflows/lambda_main.yml new file mode 100644 index 0000000..39b9e25 --- /dev/null +++ b/.github/workflows/lambda_main.yml @@ -0,0 +1,69 @@ +name: Lambda Main Workflow + +on: + push: + branches: [main, feature/seperate_terraform_with_different_states] + +env: + AWS_REGION: eu-west-2 + +jobs: + shared-lambda-terraform: + runs-on: ubuntu-latest + steps: + - name: Checkout repo + uses: actions/checkout@v4 + - name: Deploy shared Lambda Config Terraform + uses: ./.github/workflows/actions/terraform-deploy + with: + working_directory: ./deployment/lambda/lambda_shared + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ secrets.AWS_REGION }} + + lambda-ecr-example: + runs-on: ubuntu-latest + needs: shared-lambda-terraform + permissions: + id-token: write + contents: read + + steps: + - name: Checkout repo + uses: actions/checkout@v4 + - name: Build and deploy Lambda example + uses: ./.github/workflows/actions/lambda-deploy + with: + lambda_name: lambda_example + dockerfile_path: ./deployment/lambda/lambda_example/docker/Dockerfile + ecr_tf_dir: ./deployment/lambda/lambda_example/docker/ + lambda_tf_dir: ./deployment/lambda/lambda_example/ + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ secrets.AWS_REGION }} + git-sha: ${{ github.sha }} + git-ref: ${{ github.ref_name }} + + extractor-and-loader: + runs-on: ubuntu-latest + needs: shared-lambda-terraform + permissions: + id-token: write + contents: read + + steps: + - name: Checkout repo + uses: actions/checkout@v4 + - name: Build and deploy Extractor & Loader Lambda + uses: ./.github/workflows/actions/lambda-deploy + with: + lambda_name: extractor_and_loader + dockerfile_path: ./deployment/lambda/extractor_and_loader/docker/Dockerfile + ecr_tf_dir: ./deployment/lambda/extractor_and_loader/docker/ + lambda_tf_dir: ./deployment/lambda/extractor_and_loader/ + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ secrets.AWS_REGION }} + git-sha: ${{ github.sha }} + git-ref: ${{ github.ref_name }} + diff --git a/.github/workflows/push_docker_image_to_ecr.yml b/.github/workflows/push_docker_image_to_ecr.yml deleted file mode 100644 index 7bc8040..0000000 --- a/.github/workflows/push_docker_image_to_ecr.yml +++ /dev/null @@ -1,49 +0,0 @@ -name: Build and Push Docker Image to ECR lambda example - -on: - push: - branches: [feature/energy_report_etl, main] - -env: - AWS_REGION: eu-west-2 - ECR_REPOSITORY: lambda_example - -jobs: - build-and-push-to-elastic-container-registry: - runs-on: ubuntu-latest - - permissions: - id-token: write - contents: read - - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: AWS credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - # as of 14/07/2025 it'll be using user:Junte's keys - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ secrets.AWS_REGION }} - - - name: Log in to Amazon ECR - id: login-ecr - uses: aws-actions/amazon-ecr-login@v2 - - - name: Build, tag, and push Docker image to ECR - env: - ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} - IMAGE_TAG: latest - run: | - IMAGE_URI=${{ env.ECR_REGISTRY }}/${{ env.ECR_REPOSITORY }}:${{ env.IMAGE_TAG }} - echo "pwd" - pwd - ls -la - echo "Building Docker image..." - docker build -t $IMAGE_URI -f deployment/lambda_example/Dockerfile . - - echo "Pushing Docker image to ECR..." - docker push $IMAGE_URI - diff --git a/deployment/main.tf b/deployment/database/main.tf similarity index 100% rename from deployment/main.tf rename to deployment/database/main.tf diff --git a/deployment/output.tf b/deployment/database/output.tf similarity index 100% rename from deployment/output.tf rename to deployment/database/output.tf diff --git a/deployment/provider.tf b/deployment/database/provider.tf similarity index 64% rename from deployment/provider.tf rename to deployment/database/provider.tf index 46e887f..8f8274a 100644 --- a/deployment/provider.tf +++ b/deployment/database/provider.tf @@ -2,14 +2,13 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = "~> 4.16" + version = "~> 6.3.0" } } backend "s3" { bucket = "survey-extractor-tf-state" region = "eu-west-2" - profile = "domna.dev" # /home/vscode/aws/credentials - key = "terraform.tfstate" + key = "env:/dev/terraform.tfstate" } required_version = ">= 1.2.0" diff --git a/deployment/variables.tf b/deployment/database/variables.tf similarity index 99% rename from deployment/variables.tf rename to deployment/database/variables.tf index a237b6c..fd38c81 100644 --- a/deployment/variables.tf +++ b/deployment/database/variables.tf @@ -14,4 +14,4 @@ variable allocated_storage { description = "The allocated storage in gigabytes" type = number default = 20 -} \ No newline at end of file +} diff --git a/deployment/vpc.tf b/deployment/database/vpc.tf similarity index 100% rename from deployment/vpc.tf rename to deployment/database/vpc.tf diff --git a/deployment/lambda/extractor_and_loader/docker/.dockerignore b/deployment/lambda/extractor_and_loader/docker/.dockerignore new file mode 100644 index 0000000..d587d34 --- /dev/null +++ b/deployment/lambda/extractor_and_loader/docker/.dockerignore @@ -0,0 +1,21 @@ +# Ignore junk and large files +*.pdf +*.csv +*.xml +*.parquet +*.ipynb +*.mp4 +*.mov +*.jpg +*.png +*.zip +*.tar.gz +__pycache__/ +*.pyc +*.pyo +*.pyd +build/ +dist/ +.etl_cache/ +tests/ +docs/ diff --git a/deployment/lambda/extractor_and_loader/docker/Dockerfile b/deployment/lambda/extractor_and_loader/docker/Dockerfile new file mode 100644 index 0000000..cdd1f8a --- /dev/null +++ b/deployment/lambda/extractor_and_loader/docker/Dockerfile @@ -0,0 +1,25 @@ +FROM public.ecr.aws/lambda/python:3.12 + +# Install Poetry (you could pin a version if you like) +RUN curl -sSL https://install.python-poetry.org | python3 - + +# Add Poetry to PATH +ENV PATH="/root/.local/bin:$PATH" + +# Set working directory +WORKDIR /var/task + +# Copy Poetry files first to leverage Docker layer caching +COPY pyproject.toml poetry.lock README.md ./ +COPY etl/ etl/ + + +# Install dependencies into /var/task +RUN poetry config virtualenvs.create false \ + && poetry install --only main --no-interaction --no-ansi + +# Copy app code +COPY deployment/lambda/extractor_and_loader/docker/app.py ./ + +# Set Lambda handler +CMD ["app.handler"] \ No newline at end of file diff --git a/deployment/lambda/extractor_and_loader/docker/app.py b/deployment/lambda/extractor_and_loader/docker/app.py new file mode 100644 index 0000000..4a281ed --- /dev/null +++ b/deployment/lambda/extractor_and_loader/docker/app.py @@ -0,0 +1,30 @@ +""" +A quick example of lambda working a function in python +""" +from etl.read_stuff_from_s3_example import print_hello_from_etl_module + +def handler(event, context): + print("Outside try statment") + print_hello_from_etl_module() + try: + print("show me something.. anything...") + s3_uri = event.get("file_location") + if not s3_uri: + print("failed to get s3_uri") + return { + "statusCode": 400, + "body": "Missing 'file_location' in event" + } + print(f"s3 uri is {s3_uri}") + + return { + "statusCode": 200, + "body": f"s3 uri {s3_uri}" + } + + except Exception as e: + print(f"❌ Error: {e}") + return { + "statusCode": 500, + "body": str(e) + } \ No newline at end of file diff --git a/deployment/lambda/extractor_and_loader/docker/ecr.tf b/deployment/lambda/extractor_and_loader/docker/ecr.tf new file mode 100644 index 0000000..87bedd9 --- /dev/null +++ b/deployment/lambda/extractor_and_loader/docker/ecr.tf @@ -0,0 +1,62 @@ +# ECR repo +resource "aws_ecr_repository" "extractor_and_loader" { + name = "extractor_and_loader" +} + +# ECR policy to allow Lambda access +resource "aws_ecr_repository_policy" "extractor_loader_ecr_access" { + repository = aws_ecr_repository.extractor_and_loader.name + + policy = jsonencode({ + Version = "2008-10-17", + Statement = [{ + Sid = "AllowLambdaPull", + Effect = "Allow", + Principal = { + Service = "lambda.amazonaws.com" + }, + Action = [ + "ecr:GetDownloadUrlForLayer", + "ecr:BatchGetImage", + "ecr:BatchCheckLayerAvailability" + ] + }] + }) +} + + +# ECR lifecycle policy to delete tagged images older than 14 days +resource "aws_ecr_lifecycle_policy" "extractor_loader_lifecycle" { + repository = aws_ecr_repository.extractor_and_loader.name + + policy = jsonencode({ + "rules": [ + { + "rulePriority": 2, + "description": "Expire images older than 14 days", + "selection": { + "tagStatus": "untagged", + "countType": "sinceImagePushed", + "countUnit": "days", + "countNumber": 1 + }, + "action": { + "type": "expire" + } + }, + { + "rulePriority": 1, + "description": "Keep last 5 images", + "selection": { + "tagStatus": "tagged", + "tagPrefixList": ["feature"], + "countType": "imageCountMoreThan", + "countNumber": 5 + }, + "action": { + "type": "expire" + } + } + ] + }) +} \ No newline at end of file diff --git a/deployment/lambda/extractor_and_loader/docker/main.tf b/deployment/lambda/extractor_and_loader/docker/main.tf new file mode 100644 index 0000000..e69de29 diff --git a/deployment/lambda/extractor_and_loader/docker/provider.tf b/deployment/lambda/extractor_and_loader/docker/provider.tf new file mode 100644 index 0000000..2d361d4 --- /dev/null +++ b/deployment/lambda/extractor_and_loader/docker/provider.tf @@ -0,0 +1,15 @@ +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 6.3.0" + } + } + backend "s3" { + bucket = "survey-extractor-tf-state" + region = "eu-west-2" + key = "env:/dev/lambda/ecr/extractor_and_loader.tfstate" + } + + required_version = ">= 1.2.0" +} diff --git a/deployment/lambda/extractor_and_loader/extractor_and_loader_lambda.tf b/deployment/lambda/extractor_and_loader/extractor_and_loader_lambda.tf new file mode 100644 index 0000000..455a192 --- /dev/null +++ b/deployment/lambda/extractor_and_loader/extractor_and_loader_lambda.tf @@ -0,0 +1,71 @@ +# Reference existing IAM role +data "aws_iam_role" "lambda_exec_role" { + name = "lambda-exec-role" +} + +# Reference existing ECR repository +data "aws_ecr_repository" "extractor_and_loader" { + name = "extractor_and_loader" +} + +# SQS queue for extractor_and_loader +resource "aws_sqs_queue" "extractor_and_loader_queue" { + name = "extractor-loader-queue" +} + + +# IAM policy specific to this Lambda +resource "aws_iam_policy" "extractor_loader_policy" { + name = "extractor-loader-policy" + + policy = jsonencode({ + Version = "2012-10-17", + Statement = [ + { + Effect = "Allow", + Action = [ + "sqs:ReceiveMessage", + "sqs:DeleteMessage", + "sqs:GetQueueAttributes" + ], + Resource = aws_sqs_queue.extractor_and_loader_queue.arn + }, + { + Effect = "Allow", + Action = [ + "ecr:GetDownloadUrlForLayer", + "ecr:BatchGetImage", + "ecr:BatchCheckLayerAvailability" + ], + Resource = data.aws_ecr_repository.extractor_and_loader.arn + }, + { + Effect = "Allow", + Action = ["ecr:GetAuthorizationToken"], + Resource = "*" + } + ] + }) +} + +resource "aws_iam_role_policy_attachment" "extractor_loader_policy_attach" { + role = data.aws_iam_role.lambda_exec_role.name + policy_arn = aws_iam_policy.extractor_loader_policy.arn +} + +# Lambda function +resource "aws_lambda_function" "extractor_and_loader" { + function_name = "extractor-and-loader" + role = data.aws_iam_role.lambda_exec_role.arn + package_type = "Image" + image_uri = "${data.aws_ecr_repository.extractor_and_loader.repository_url}:${var.lambda_image_tag}" + timeout = 30 +} + +# SQS trigger +resource "aws_lambda_event_source_mapping" "extractor_and_loader_trigger" { + event_source_arn = aws_sqs_queue.extractor_and_loader_queue.arn + function_name = aws_lambda_function.extractor_and_loader.arn + batch_size = 1 +} + diff --git a/deployment/lambda/extractor_and_loader/main.tf b/deployment/lambda/extractor_and_loader/main.tf new file mode 100644 index 0000000..e69de29 diff --git a/deployment/lambda/extractor_and_loader/provider.tf b/deployment/lambda/extractor_and_loader/provider.tf new file mode 100644 index 0000000..51eca0c --- /dev/null +++ b/deployment/lambda/extractor_and_loader/provider.tf @@ -0,0 +1,15 @@ +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 6.3.0" + } + } + backend "s3" { + bucket = "survey-extractor-tf-state" + region = "eu-west-2" + key = "env:/dev/lambda/eachlambda/extractor_and_loader_lambda.tfstate" + } + + required_version = ">= 1.2.0" +} diff --git a/deployment/lambda/extractor_and_loader/vars.tf b/deployment/lambda/extractor_and_loader/vars.tf new file mode 100644 index 0000000..ecdf359 --- /dev/null +++ b/deployment/lambda/extractor_and_loader/vars.tf @@ -0,0 +1,5 @@ +variable "lambda_image_tag" { + description = "Docker image tag (e.g. GitHub SHA)" + type = string + default = "local-dev-latest" +} \ No newline at end of file diff --git a/deployment/lambda_example/Dockerfile b/deployment/lambda/lambda_example/docker/Dockerfile similarity index 71% rename from deployment/lambda_example/Dockerfile rename to deployment/lambda/lambda_example/docker/Dockerfile index c1caf03..3aa661b 100644 --- a/deployment/lambda_example/Dockerfile +++ b/deployment/lambda/lambda_example/docker/Dockerfile @@ -2,7 +2,7 @@ FROM public.ecr.aws/lambda/python:3.11 # Copy function code -COPY deployment/lambda_example/app.py ./ +COPY deployment/lambda/lambda_example/docker/app.py ./ # Set the CMD to your handler CMD ["app.handler"] \ No newline at end of file diff --git a/deployment/lambda/lambda_example/docker/app.py b/deployment/lambda/lambda_example/docker/app.py new file mode 100644 index 0000000..4bd6ec9 --- /dev/null +++ b/deployment/lambda/lambda_example/docker/app.py @@ -0,0 +1,19 @@ +""" +A quick example of lambda working a function in python +""" + +def handler(event, context): + try: + print("Printing from lambda example") + + return { + "statusCode": 200, + "body": f"s3 uri {s3_uri}" + } + + except Exception as e: + print(f"❌ Error: {e}") + return { + "statusCode": 500, + "body": str(e) + } \ No newline at end of file diff --git a/deployment/lambda/lambda_example/docker/ecr.tf b/deployment/lambda/lambda_example/docker/ecr.tf new file mode 100644 index 0000000..276697f --- /dev/null +++ b/deployment/lambda/lambda_example/docker/ecr.tf @@ -0,0 +1,61 @@ +# ECR repo for lambda_example +resource "aws_ecr_repository" "lambda_example" { + name = "lambda_example" +} + +# ECR policy to allow Lambda access +resource "aws_ecr_repository_policy" "lambda_example_ecr_access" { + repository = aws_ecr_repository.lambda_example.name + + policy = jsonencode({ + Version = "2008-10-17", + Statement = [{ + Sid = "AllowLambdaPull", + Effect = "Allow", + Principal = { + Service = "lambda.amazonaws.com" + }, + Action = [ + "ecr:GetDownloadUrlForLayer", + "ecr:BatchGetImage", + "ecr:BatchCheckLayerAvailability" + ] + }] + }) +} + +# ECR lifecycle policy to delete tagged images older than 14 days +resource "aws_ecr_lifecycle_policy" "lambda_example_ecr_lifecycle" { + repository = aws_ecr_repository.lambda_example.name + + policy = jsonencode({ + "rules": [ + { + "rulePriority": 2, + "description": "Expire images older than 14 days", + "selection": { + "tagStatus": "untagged", + "countType": "sinceImagePushed", + "countUnit": "days", + "countNumber": 1 + }, + "action": { + "type": "expire" + } + }, + { + "rulePriority": 1, + "description": "Keep last 5 images", + "selection": { + "tagStatus": "tagged", + "tagPrefixList": ["feature"], + "countType": "imageCountMoreThan", + "countNumber": 5 + }, + "action": { + "type": "expire" + } + } + ] + }) +} diff --git a/deployment/lambda/lambda_example/docker/main.tf b/deployment/lambda/lambda_example/docker/main.tf new file mode 100644 index 0000000..e69de29 diff --git a/deployment/lambda/lambda_example/docker/output.tf b/deployment/lambda/lambda_example/docker/output.tf new file mode 100644 index 0000000..1b797cf --- /dev/null +++ b/deployment/lambda/lambda_example/docker/output.tf @@ -0,0 +1,3 @@ +output "ecr_repo_url" { + value = aws_ecr_repository.lambda_example.repository_url +} \ No newline at end of file diff --git a/deployment/lambda/lambda_example/docker/provider.tf b/deployment/lambda/lambda_example/docker/provider.tf new file mode 100644 index 0000000..f210110 --- /dev/null +++ b/deployment/lambda/lambda_example/docker/provider.tf @@ -0,0 +1,15 @@ +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 6.3.0" + } + } + backend "s3" { + bucket = "survey-extractor-tf-state" + region = "eu-west-2" + key = "env:/dev/lambda/ecr/lambda_example_ecr.tfstate" + } + + required_version = ">= 1.2.0" +} diff --git a/deployment/lambda/lambda_example/lambda_example_and_config.tf b/deployment/lambda/lambda_example/lambda_example_and_config.tf new file mode 100644 index 0000000..4f87771 --- /dev/null +++ b/deployment/lambda/lambda_example/lambda_example_and_config.tf @@ -0,0 +1,69 @@ +# Reference existing IAM role +data "aws_iam_role" "lambda_exec_role" { + name = "lambda-exec-role" +} + +# Reference existing ECR repository +data "aws_ecr_repository" "lambda_example" { + name = "lambda_example" +} + +# SQS queue for lambda_example +resource "aws_sqs_queue" "lambda_example_queue" { + name = "lambda-example-queue" +} + +# Custom IAM policy specific to lambda_example +resource "aws_iam_policy" "lambda_example_policy" { + name = "lambda-example-policy" + + policy = jsonencode({ + Version = "2012-10-17", + Statement = [ + { + Effect = "Allow", + Action = [ + "sqs:ReceiveMessage", + "sqs:DeleteMessage", + "sqs:GetQueueAttributes" + ], + Resource = aws_sqs_queue.lambda_example_queue.arn + }, + { + Effect = "Allow", + Action = [ + "ecr:GetDownloadUrlForLayer", + "ecr:BatchGetImage", + "ecr:BatchCheckLayerAvailability" + ], + Resource = data.aws_ecr_repository.lambda_example.arn + }, + { + Effect = "Allow", + Action = ["ecr:GetAuthorizationToken"], + Resource = "*" + } + ] + }) +} + +resource "aws_iam_role_policy_attachment" "lambda_example_policy_attach" { + role = data.aws_iam_role.lambda_exec_role.name + policy_arn = aws_iam_policy.lambda_example_policy.arn +} + +# Lambda function +resource "aws_lambda_function" "lambda_example" { + function_name = "lambda-example" + role = data.aws_iam_role.lambda_exec_role.arn + package_type = "Image" + image_uri = "${data.aws_ecr_repository.lambda_example.repository_url}:${var.lambda_image_tag}" + timeout = 10 +} + +# SQS trigger +resource "aws_lambda_event_source_mapping" "lambda_example_trigger" { + event_source_arn = aws_sqs_queue.lambda_example_queue.arn + function_name = aws_lambda_function.lambda_example.arn + batch_size = 1 +} diff --git a/deployment/lambda/lambda_example/main.tf b/deployment/lambda/lambda_example/main.tf new file mode 100644 index 0000000..e69de29 diff --git a/deployment/lambda/lambda_example/provider.tf b/deployment/lambda/lambda_example/provider.tf new file mode 100644 index 0000000..7aa8df6 --- /dev/null +++ b/deployment/lambda/lambda_example/provider.tf @@ -0,0 +1,15 @@ +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 6.3.0" + } + } + backend "s3" { + bucket = "survey-extractor-tf-state" + region = "eu-west-2" + key = "env:/dev/lambda/eachlambda/lambda_example.tfstate" + } + + required_version = ">= 1.2.0" +} diff --git a/deployment/lambda/lambda_example/vars.tf b/deployment/lambda/lambda_example/vars.tf new file mode 100644 index 0000000..ecdf359 --- /dev/null +++ b/deployment/lambda/lambda_example/vars.tf @@ -0,0 +1,5 @@ +variable "lambda_image_tag" { + description = "Docker image tag (e.g. GitHub SHA)" + type = string + default = "local-dev-latest" +} \ No newline at end of file diff --git a/deployment/lambda/lambda_shared/lambda_shared_config.tf b/deployment/lambda/lambda_shared/lambda_shared_config.tf new file mode 100644 index 0000000..1533dfb --- /dev/null +++ b/deployment/lambda/lambda_shared/lambda_shared_config.tf @@ -0,0 +1,21 @@ +# IAM role for both Lambdas (can be shared) +resource "aws_iam_role" "lambda_exec_role" { + name = "lambda-exec-role" + + assume_role_policy = jsonencode({ + Version = "2012-10-17", + Statement = [{ + Effect = "Allow", + Principal = { + Service = "lambda.amazonaws.com" + }, + Action = "sts:AssumeRole" + }] + }) +} + + +resource "aws_iam_role_policy_attachment" "lambda_basic_execution" { + role = aws_iam_role.lambda_exec_role.name + policy_arn = "arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole" +} \ No newline at end of file diff --git a/deployment/lambda/lambda_shared/main.tf b/deployment/lambda/lambda_shared/main.tf new file mode 100644 index 0000000..e69de29 diff --git a/deployment/lambda/lambda_shared/output.tf b/deployment/lambda/lambda_shared/output.tf new file mode 100644 index 0000000..9d4094d --- /dev/null +++ b/deployment/lambda/lambda_shared/output.tf @@ -0,0 +1,9 @@ +output "lambda_exec_role_arn" { + description = "The ARN of the IAM role used by the Lambda functions" + value = aws_iam_role.lambda_exec_role.arn +} + +output "lambda_exec_role_name" { + description = "The ARN of the IAM role used by the Lambda functions" + value = aws_iam_role.lambda_exec_role.name +} diff --git a/deployment/lambda/lambda_shared/provider.tf b/deployment/lambda/lambda_shared/provider.tf new file mode 100644 index 0000000..521355b --- /dev/null +++ b/deployment/lambda/lambda_shared/provider.tf @@ -0,0 +1,15 @@ +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 6.3.0" + } + } + backend "s3" { + bucket = "survey-extractor-tf-state" + region = "eu-west-2" + key = "env:/dev/lambda/lambda_share_configuration.tfstate" + } + + required_version = ">= 1.2.0" +} diff --git a/deployment/lambda_example.tf b/deployment/lambda_example.tf deleted file mode 100644 index 0748af4..0000000 --- a/deployment/lambda_example.tf +++ /dev/null @@ -1,121 +0,0 @@ -# This is an example file to setup a lamda function with a sqs and cloudwatch. -# Please us this as a template for future lambda. -# Be sure to push the image you are using to ECR or it won't deploy properly - -# Create an SQS queue that will trigger the Lambda -resource "aws_sqs_queue" "my_queue" { - name = "my-lambda-queue" -} - -# Create an ECR repository to store the Docker image for the Lambda function -resource "aws_ecr_repository" "lambda_repo" { - name = "lambda_example" -} - -# IAM role that the Lambda function will assume -resource "aws_iam_role" "lambda_exec_role" { - name = "lambda-exec-role" - - assume_role_policy = jsonencode({ - Version = "2012-10-17", - Statement = [ - { - Action = "sts:AssumeRole", - Effect = "Allow", - Principal = { - Service = "lambda.amazonaws.com" - } - } - ] - }) -} - -# Attach AWS-managed policy for basic Lambda execution (CloudWatch logging) -resource "aws_iam_role_policy_attachment" "lambda_basic_execution" { - role = aws_iam_role.lambda_exec_role.name - policy_arn = "arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole" -} - -# Custom policy: SQS access + ECR image pull permissions -resource "aws_iam_policy" "lambda_custom_policy" { - name = "lambda-sqs-ecr-policy" - - policy = jsonencode({ - Version = "2012-10-17", - Statement = [ - # Allow Lambda to read from SQS - { - Effect = "Allow", - Action = [ - "sqs:ReceiveMessage", - "sqs:DeleteMessage", - "sqs:GetQueueAttributes" - ], - Resource = aws_sqs_queue.my_queue.arn - }, - # Allow Lambda to pull images from ECR - { - Effect = "Allow", - Action = [ - "ecr:GetDownloadUrlForLayer", - "ecr:BatchGetImage", - "ecr:BatchCheckLayerAvailability" - ], - Resource = aws_ecr_repository.lambda_repo.arn - }, - # Needed to authenticate to ECR (pulling the image) - { - Effect = "Allow", - Action = [ - "ecr:GetAuthorizationToken" - ], - Resource = "*" - } - ] - }) -} - -# Attach the custom policy to the Lambda role -resource "aws_iam_role_policy_attachment" "lambda_custom_policy_attach" { - role = aws_iam_role.lambda_exec_role.name - policy_arn = aws_iam_policy.lambda_custom_policy.arn -} - -# Define the Lambda function using a Docker image from ECR -resource "aws_lambda_function" "lambda_docker" { - function_name = "docker-hello-world-python-example" - role = aws_iam_role.lambda_exec_role.arn - package_type = "Image" - image_uri = "${aws_ecr_repository.lambda_repo.repository_url}:latest" - timeout = 10 -} - -# Connect the SQS queue to the Lambda so it gets triggered by incoming messages -resource "aws_lambda_event_source_mapping" "sqs_trigger" { - event_source_arn = aws_sqs_queue.my_queue.arn - function_name = aws_lambda_function.lambda_docker.arn - batch_size = 1 -} - - -resource "aws_ecr_repository_policy" "lambda_ecr_access" { - repository = aws_ecr_repository.lambda_repo.name - - policy = jsonencode({ - Version = "2008-10-17", - Statement = [ - { - Sid = "AllowLambdaPull", - Effect = "Allow", - Principal = { - Service = "lambda.amazonaws.com" - }, - Action = [ - "ecr:GetDownloadUrlForLayer", - "ecr:BatchGetImage", - "ecr:BatchCheckLayerAvailability" - ] - } - ] - }) -} \ No newline at end of file diff --git a/deployment/lambda_example/app.py b/deployment/lambda_example/app.py deleted file mode 100644 index 05e2693..0000000 --- a/deployment/lambda_example/app.py +++ /dev/null @@ -1,11 +0,0 @@ -""" -A quick example of lambda working a function in python -""" - -def handler(event, context): - print("Hello from Python function. This shold be running from a dockerfile env and executed on a aws lambda!") - return { - 'statusCode': 200, - 'body': 'Hello World' - } - diff --git a/etl/epr_etl_example.py b/etl/epr_etl_example.py index 866c75a..e0a7032 100644 --- a/etl/epr_etl_example.py +++ b/etl/epr_etl_example.py @@ -1,7 +1,7 @@ from etl.surveyedData.surveryedData import surveyedDataProcessor files = [ - "/tmp/sharepoint/Sandwell/SANDWELL-001/26 Willow close B64 6EG/Content (13).pdf", + # "/tmp/sharepoint/Sandwell/SANDWELL-001/26 Willow close B64 6EG/Content (13).pdf", "/tmp/sharepoint/Livewest/Livewest-001/12 Birch End/Summary Information 12 Birch End.pdf" ] diff --git a/etl/fileReader/pdfReaderToText.py b/etl/fileReader/pdfReaderToText.py index 9668eed..c1d5834 100644 --- a/etl/fileReader/pdfReaderToText.py +++ b/etl/fileReader/pdfReaderToText.py @@ -33,7 +33,6 @@ class pdfReaderToText(): self.all_text += text self.text_list = self.all_text.split('\n') - pprint(self.text_list) def get_list_of_text(self): return self.text_list diff --git a/etl/fileReader/sitenotes.py b/etl/fileReader/sitenotes.py index 1df8747..7799252 100644 --- a/etl/fileReader/sitenotes.py +++ b/etl/fileReader/sitenotes.py @@ -298,7 +298,6 @@ class WarmHomesConditionReport(SiteNotesExtractor): # Gable 2 data = self.get_data_between("2.2.4. External Elevation - Gable 2", "2.3. Conservatory or Outbuilding") - pprint(self.raw_data) state = True if self.get_next_value(data, "Is there a 4th external elevation?").lower() == "yes" else False if state is False: gable_two = ExternalElevationGableTwo(is_there_a_fourth_external_elevation=state) @@ -1601,9 +1600,9 @@ class EnergyPerformanceReportWithData(SiteNotesExtractor): class EnergyPerformanceReportSummaryInformation(SiteNotesExtractor): def __init__(self, data_list): - super().__init__(data_list) + self.raw_data = data_list self.type = ReportType.ENERGY_PERFORMANCE_REPORT_SUMMARY_INFORMATION - self.master_obj = self.setup() + self.setup() def setup(self): pass \ No newline at end of file diff --git a/etl/hubSpotClient/hubspot.py b/etl/hubSpotClient/hubspot.py index 3b4913e..27e6903 100644 --- a/etl/hubSpotClient/hubspot.py +++ b/etl/hubSpotClient/hubspot.py @@ -18,7 +18,7 @@ class DealStage(Enum): SURVEYED_NO_ACCESS_NEED_SIGN_OFF = "1617223915" CUSTOMER_CONTACTED = "888730834" SURVEYED_COMPLETED_SIGNED_OFF = "1617223916" - NEEDS_ADDITIONAL_INFORMATION_FROM_ASSESSOR = "1887736000" + FILES_MISSING_FROM_ASSESSOR = "1887736000" class HubSpotClient(): def __init__(self): @@ -206,7 +206,7 @@ class HubSpotClient(): after = response.paging.next.after all_deals = [] - for deal in found_deals: + for i,deal in enumerate(found_deals): domna_id, landlord_id, uprn = self.get_domna_and_landlord_id(deal.id) try: deal_name = deal.properties['dealname'] @@ -263,10 +263,8 @@ class HubSpotClient(): self.add_note_to_deal(deal_id, format_error_note(e)) else: self.logger.error(f"Non-validation error occurred: {str(e)}", exc_info=True) - - self.logger.info(f"Deal name <{deal_name}> moving to 'needs additional information'") - self.move_deals_to_different_stage([deal_id], DealStage.NEEDS_ADDITIONAL_INFORMATION_FROM_ASSESSOR.value) + self.move_deals_to_different_stage([deal_id], DealStage.FILES_MISSING_FROM_ASSESSOR.value) return all_deals def print_all_pipeline_ids(self): diff --git a/etl/hubSpotClient/types.py b/etl/hubSpotClient/types.py index e004243..35afaa0 100644 --- a/etl/hubSpotClient/types.py +++ b/etl/hubSpotClient/types.py @@ -60,14 +60,20 @@ class SubmissionInfoFromDeal(BaseModel): raise ValueError(f"Error accessing SharePoint path: {self.submission_folder_path}. Error: {str(e)}") try: - # Check if sharepoint link is reachable and has any contents - files = sp.get_folders_in_path(path) - if "value" in files and len(files["value"]) > 0: - pass - else: - raise ValueError(f"SharePoint folder is empty: {self.submission_folder_path}") + try: + files = sp.get_folders_in_path(path) + if files.get("value"): + pass + except Exception as e: + print("Trying SGEC") + sp = SharePointScraper(SharePointInstaller.SGEC) + files = sp.get_folders_in_path(path) + if files.get("value"): + pass + else: + raise ValueError(f"[SharePoint Folder Empty] Folder has no contents after multiple attempts: {self.submission_folder_path}") except Exception as e: - raise ValueError(str(e)) + raise ValueError(f"[Folder Access Error] {str(e)}") # download files in url and check files are there: try: @@ -80,8 +86,8 @@ class SubmissionInfoFromDeal(BaseModel): if sdp.condition_report is None: missing_items.append("Condition Report") - if sdp.epr_summary_information is None: - missing_items.append("EPR Energy report with data is missing") + if sdp.epr_with_data is None: + missing_items.append("EPR Energy report with data") if sdp.rd_sap_xml is None: missing_items.append("RDSAP XML") @@ -90,7 +96,7 @@ class SubmissionInfoFromDeal(BaseModel): missing_items.append("LIG SAP XML") if sdp.epr_summary_information is None: - missing_items.append("EPR Summary information is missing") + missing_items.append("EPR Summary information") if missing_items: raise ValueError(f"Missing required items: {', '.join(missing_items)}") diff --git a/etl/hubspot_surveyed_needs_sign_off.py b/etl/hubspot_surveyed_needs_sign_off.py index 7794d07..6b5f48c 100644 --- a/etl/hubspot_surveyed_needs_sign_off.py +++ b/etl/hubspot_surveyed_needs_sign_off.py @@ -10,6 +10,8 @@ os.environ["SHAREPOINT_CLIENT_SECRET"] = "SOf8Q~-is4wdQiqvEEm9FlJQRAY9ELGaj5Qz-a os.environ["SHAREPOINT_TENANT_ID"] = "c3f7519c-2719-4547-af04-6da6cbfd8f8f" os.environ["SOUTH_COAST_INSULATION_SERVICE_SHAREPOINT_ID"] = "b5a51507-9427-4ee0-b03e-90ec7681e2d3" os.environ["JJC_SERVICE_SHAREPOINT_ID"] = "7fdd0485-bbf3-4b29-b30f-98c81c2a6284" +os.environ["SGEC_SERVICE_SHAREPOINT_ID"] = "52018e5c-3215-4fe4-a4e3-bbf0d0aa7cd9" + from etl.hubSpotClient.hubspot import DealStage, HubSpotClient # Local development @@ -18,7 +20,7 @@ os.environ["DATABASE_URL"] = "postgresql://postgres:makingwarmhomes@db:5432/post hubspotClient = HubSpotClient() # files missing from assessor column -deals = hubspotClient.get_deals_from_deal_stage(DealStage.NEEDS_ADDITIONAL_INFORMATION_FROM_ASSESSOR) +deals = hubspotClient.get_deals_from_deal_stage(DealStage.FILES_MISSING_FROM_ASSESSOR) for deal in deals: diff --git a/etl/month_end_automation_wave_2_layout.py b/etl/month_end_automation_wave_2_layout.py new file mode 100644 index 0000000..7dca011 --- /dev/null +++ b/etl/month_end_automation_wave_2_layout.py @@ -0,0 +1,236 @@ +# Wave 2's month end automation + +from tqdm import tqdm +from monday import MondayClient +from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset_ids +from pprint import pprint +import pandas as pd +import json + +monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY" +monday = MondayClient(monday_key) +# NCHA SHDF Westville Wave 1 & 2 +board_ids = ["3900434153"] + +rate_card_data = { + "job_type": [ + "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", + "Design Archetype", "Design Repetitive", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", + "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", + "RA no show", "ATT no show", "post EPC no show", "Design Revision" + ], + "rate": [ + 207.65, 101, 186.4, 98, 98, + 450, 150, 163, 135, 120, + "60 - Needs to be verified (Post EPR)", 45, 90.5, 40, + 25, 25, 25, "check with Kevin" + ] +} + +rate_card_df = pd.DataFrame(rate_card_data) + + +for board in tqdm(board_ids): + board_data = monday.boards.fetch_boards_by_id(board) + columns = board_data["data"]["boards"][0]["columns"] + col_id_map = {col["title"].lower(): col["id"] for col in columns} + reversed_col_id_map = {v: k for k, v in col_id_map.items()} + + + items = get_all_items(board, monday) + + all_records = [] + for row in tqdm(items): + data = {} + data.update({"address": row['name']}) + data.update({"client": row['group']['title']}) + for col in row.get("column_values", []): + if col.get("id") in reversed_col_id_map: + if col.get("type") == "file": + value = col.get("value") + no_of_files = 0 + + if value: + value = json.loads(col["value"]) + no_of_files = len(value.get('files', [])) + data.update({reversed_col_id_map[col.get("id")]: no_of_files}) + elif "no show" in reversed_col_id_map[col.get("id")]: + def extract_number_from_text(text): + number_str = '' + + for char in text: + if char.isnumeric(): + number_str += char + elif number_str: + break # stop once a number sequence ends + + return int(number_str) if number_str else None + text = col.get("text") + if text is None: + data.update({ + reversed_col_id_map[col.get("id")]: col.get("text") + }) + else: + data.update({ + reversed_col_id_map[col.get("id")]: extract_number_from_text(text) + }) + else: + data.update({ + reversed_col_id_map[col.get("id")]: col.get("text") + }) + all_records.append(data) + +# Convert to DataFrame +df = pd.DataFrame(all_records) + +filtered_dfs = [] + +# RA +ra = df[ + df["ra"].str.lower().isin(["completed rdsap 10", "completed rdsap 9.9"]) +].copy() +ra["job_type"] = "RA" +filtered_dfs.append(ra) + + +# ATT +att = df[ + df["att"].str.lower().isin(["completed"]) +].copy() +att["job_type"] = "ATT" +filtered_dfs.append(att) + +# V1 Coordination +v1 = df[ + df["v1 coordination status"].str.lower().isin(["rc complete"]) +].copy() +v1["job_type"] = "Coordination Stage 1 v1" +filtered_dfs.append(v1) + +# V2 Coordination +_ = df[df["v2 invoiced"].fillna('').str.lower().isin(['to be invoiced'])] +v2 = _[_["v2 dc/ima/pas"] > 0].copy() +v2["job_type"] = "Coordination Stage 1 v2 remodel" +filtered_dfs.append(v2) + +# V3 Coordination +v3 = df[ + df["v3 invoiced"].str.lower().isin(["to be invoiced"]) +].copy() +v3["job_type"] = "Coordination Stage 1 v3 remodel" +filtered_dfs.append(v3) + +# Coordination stage 2 Please complete +cors2 = df[ + df["rc stg. 2"].str.lower().isin(["to invoice"]) +].copy() +cors2["job_type"] = "Coordination Stage 2" +filtered_dfs.append(cors2) + +# Design type archietype +design1 = df[ + (df["design type for invoicing"].str.lower().isin(["archetype"])) & (df["design invoice status"].str.lower().isin(["to invoice"])) +].copy() +design1["job_type"] = "Design Archetype" +filtered_dfs.append(design1) + +# design type reptitive +design1 = df[ + (df["design type for invoicing"].str.lower().isin(["repetitive"])) & df["design invoice status"].str.lower().isin(["to invoice"]) +].copy() +design1["job_type"] = "Design Repetitive" +filtered_dfs.append(design1) + +# Design stage revisions +design2 = df[ + df["design revision invoice status"].str.lower().isin(["to invoice"]) +].copy() +design2["job_type"] = "Design Revision" +filtered_dfs.append(design2) + +# Lodgement Phase 1 +lodg1 = df[ + df["lodg. phase 1 invoice status"].str.lower().isin(["to invoice"]) +].copy() +lodg1["job_type"] = "Lodgement phase 1" +filtered_dfs.append(lodg1) + +# Full Lodgement Phase +lodg2 = df[ + df["full lodgement invoice status"].str.lower().isin(["to invoice"]) +].copy() +lodg2["job_type"] = "Full Lodgement phase 2" +filtered_dfs.append(lodg2) + +# POST EPC +post_epc = df[ + df["post-epc status"].str.lower().isin(["epc files uploaded"]) +].copy() +post_epc["job_type"] = "Post EPC" +filtered_dfs.append(post_epc) + + +# POST EPR +post_epr = df[ + df["post-epc status"].str.lower().isin(["post epr completed"]) +].copy() +post_epr["job_type"] = "Post EPR" +filtered_dfs.append(post_epr) + + + +# Post ATT +post_att = df[ + df["post-att"].str.lower().isin(["post-att uploaded"]) +].copy() +post_att["job_type"] = "POST ATT" +filtered_dfs.append(post_att) + + +# Retrofit Evaluation +retro = df[ + df["retrofit evaluation"].str.lower().isin(["complete"]) +].copy() +retro["job_type"] = "Retrofit Evaluation" +filtered_dfs.append(retro) + +# RA NO Show +ra_ns = df[ + (df["ra no show evidence"].fillna(-9999) != df["ra no show invoice"].fillna(-9999)) & + (df["ra no show evidence"] != 0) +].copy() +ra_ns["job_type"] = "RA NO SHOW" +filtered_dfs.append(ra_ns) + + + +# ATT NO Show +att_ns = df[ + (df["att no show evidence"].fillna(-9999) != df["att no show invoice"].fillna(-9999)) & + (df["att no show evidence"] != 0) +].copy() +att_ns["job_type"] = "ATT NO SHOW" +filtered_dfs.append(att_ns) + + +# Post visit no show +epc_ns = df[ + (df["epc no show evidence"].fillna(-9999) != df["epc no show invoice"].fillna(-9999)) & + (df["epc no show evidence"] != 0) +].copy() +epc_ns["job_type"] = "post EPC NO SHOW" +filtered_dfs.append(epc_ns) + +final_df = pd.concat(filtered_dfs).reset_index(drop=True) + + +final_df["job_type"] = final_df["job_type"].str.lower() +rate_card_df["job_type"] = rate_card_df["job_type"].str.lower() + +# Now perform the merge +combined_with_rates = final_df.merge(rate_card_df, on="job_type", how="left") +import datetime +timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M') + +attribute = ['address', 'client', 'job_type', 'rate'] +combined_with_rates[attribute].to_excel(f'NCHA SHDF Westville Wave 1 & 2_{timestamp}.xlsx', index=False) \ No newline at end of file diff --git a/etl/month_end_automation_wave_2_no_10.py b/etl/month_end_automation_wave_2_no_10.py new file mode 100644 index 0000000..d854fdc --- /dev/null +++ b/etl/month_end_automation_wave_2_no_10.py @@ -0,0 +1,208 @@ +# Wave 2's month end automation +from tqdm import tqdm +from monday import MondayClient +from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset_ids +from pprint import pprint +import pandas as pd +import json + +monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY" +monday = MondayClient(monday_key) +# NCHA Derbyshire Dales (DDDCC) SHDF +board_ids = ["6947307148"] + +rate_card_data = { + "job_type": [ + "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", + "Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", + "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", + "RA no show", "ATT no show", "post EPC no show" + ], + "rate": [ + 259, 125, 280, 125, 125, + 650, 415, 195, 175, 135, + 120, "Post EPR Please Marianne", 85, 125, 60, + 25, 25, 25 + ] +} + +rate_card_df = pd.DataFrame(rate_card_data) + + +for board in tqdm(board_ids): + board_data = monday.boards.fetch_boards_by_id(board) + columns = board_data["data"]["boards"][0]["columns"] + col_id_map = {col["title"].lower(): col["id"] for col in columns} + reversed_col_id_map = {v: k for k, v in col_id_map.items()} + + + items = get_all_items(board, monday) + + all_records = [] + for row in tqdm(items): + data = {} + data.update({"address": row['name']}) + data.update({"client": row['group']['title']}) + for col in row.get("column_values", []): + if col.get("id") in reversed_col_id_map: + if col.get("type") == "file": + value = col.get("value") + no_of_files = 0 + + if value: + value = json.loads(col["value"]) + no_of_files = len(value.get('files', [])) + data.update({reversed_col_id_map[col.get("id")]: no_of_files}) + elif "no show" in reversed_col_id_map[col.get("id")]: + def extract_number_from_text(text): + number_str = '' + + for char in text: + if char.isnumeric(): + number_str += char + elif number_str: + break # stop once a number sequence ends + + return int(number_str) if number_str else None + text = col.get("text") + if text is None: + data.update({ + reversed_col_id_map[col.get("id")]: col.get("text") + }) + else: + data.update({ + reversed_col_id_map[col.get("id")]: extract_number_from_text(text) + }) + else: + data.update({ + reversed_col_id_map[col.get("id")]: col.get("text") + }) + all_records.append(data) + +# Convert to DataFrame +df = pd.DataFrame(all_records) + +filtered_dfs = [] + +def get_df(df, column_name, success_critera, job_name): + _ = df[ + df[column_name].str.lower().isin(success_critera) + ].copy() + _["job_type"] = job_name + return _ + + +# RA +ra = get_df(df, "ra", ["completed rdsap 9.9", "completed rd sap 10"], "RA") +filtered_dfs.append(ra) + + +# ATT +att = get_df(df, "att", ["completed"], "ATT") +filtered_dfs.append(att) + +# V1 Coordination +v1 = get_df(df, "coordination status".lower(), [ + "rc complete", +], "Coordination Stage 1 v1") +filtered_dfs.append(v1) + +# V2 Coordination +# v2 = get_df(df, "v2 coordination status", ["rc v2 complete", "uploaded"], "V2 Coordination") +# filtered_dfs.append(v2) + +# # V3 Coordination +# v3 = get_df(df, "") +# # filtered_dfs.append(v3) + +# Coordination stage 2 Please complete +cors2 = df[ + df["rc stg. 2"].str.lower().isin(["to invoice"]) +] +cors2["joby_type"] = "Coordination Stage 2" +filtered_dfs.append(cors2) + +# # Design stage 1 +# design1 = get_df(df, "design upload to sharepoint", ["done"], "Design") +# filtered_dfs.append(design1) + +# Design revision +# design2 = get_df(df, "design revision invoice", [ +# "Rev. A to invoice".lower(), +# "Rev. B to invoice".lower(), +# "Rev. C to invoice".lower(), +# "Rev. D to invoice".lower(), +# ], "Design Revision") +# filtered_dfs.append(design2) + +# Lodgement Phase 1 +lodg1 = get_df(df, "Lodg. Phase 1 Invoice Status".lower(), ["to invoice"], "Lodgement Phase 1") +filtered_dfs.append(lodg1) + + +# Full Lodgement Phase +full_lodgement = get_df(df, "full lodgement invoice status".lower(), ["to invoice"], "Full lodgement phase 2") +filtered_dfs.append(full_lodgement) + +# POST EPC +post_epc = get_df(df, "lodged epc", ["complete"], "POST EPC") +filtered_dfs.append(post_epc) + + +# POST EPR +post_epr = df[ + df["lodged epc"].str.lower().isin(["post epr completed"]) +].copy() +post_epr["job_type"] = "Post EPR" +filtered_dfs.append(post_epr) + +# Post ATT +post_att = get_df(df, "post att", ["done"], "POST ATT") +filtered_dfs.append(post_att) + +post_att = get_df(df, "post-test status", ["complete"], "POST ATT") +filtered_dfs.append(post_att) + +# Retrofit Evaluation +retro = get_df(df, "retrofit evaluation", ["done"], "Retrofit Evaluation") +filtered_dfs.append(retro) + +# RA NO Show +ra_ns = df[ + (df["ra no show evidence"].fillna(-9999) != df["ra no show invoice"].fillna(-9999)) & + (df["ra no show evidence"] !=0 ) +].copy() +ra_ns["job_type"] = "RA NO SHOW" +filtered_dfs.append(ra_ns) + + +# ATT NO Show +att_ns = df[ + (df["att no show evidence"].fillna(-9999) != df["att no show invoice"].fillna(-9999)) & + (df["att no show evidence"] != 0) +].copy() +att_ns["job_type"] = "ATT NO SHOW" +filtered_dfs.append(att_ns) + + +# Post visit no show +# epc_ns = df[ +# df["post epc no show evidence"].fillna(-9999) != df["post epc no show invoice"].fillna(-9999) +# ].copy() +# epc_ns["job_type"] = "post EPC NO SHOW" +# filtered_dfs.append(epc_ns) + +final_df = pd.concat(filtered_dfs).reset_index(drop=True) + +final_df[['address', 'client', 'job_type']] + +final_df["job_type"] = final_df["job_type"].str.lower() +rate_card_df["job_type"] = rate_card_df["job_type"].str.lower() + +# Now perform the merge +combined_with_rates = final_df.merge(rate_card_df, on="job_type", how="left") +import datetime +timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M') + +attribute = ['address', 'client', 'job_type', 'rate'] +combined_with_rates[attribute].to_excel(f'NCHA Derbyshire Dales (DDDCC) SHDF_{timestamp}.xlsx', index=False) diff --git a/etl/month_end_automation_wave_2_no_11.py b/etl/month_end_automation_wave_2_no_11.py new file mode 100644 index 0000000..f05ef13 --- /dev/null +++ b/etl/month_end_automation_wave_2_no_11.py @@ -0,0 +1,201 @@ +# Wave 2's month end automation +from tqdm import tqdm +from monday import MondayClient +from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset_ids +from pprint import pprint +import pandas as pd +import json + +monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY" +monday = MondayClient(monday_key) +# Northumberland LAD2 & HUG2 +board_ids = ["5121300882"] + +rate_card_data = { + "job_type": [ + "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", + "Design Archetype", "Design Repetitive", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", + "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", + "RA no show", "ATT no show", "post EPC no show" + ], + "rate": [ + 259, 125, 280, 125, 125, + 650, 195, 175, 135, + 120, "Post EPR Please Marianne", 85, 125, 60, + 25, 25, 25 + ] +} + +rate_card_df = pd.DataFrame(rate_card_data) + +for board in tqdm(board_ids): + board_data = monday.boards.fetch_boards_by_id(board) + columns = board_data["data"]["boards"][0]["columns"] + col_id_map = {col["title"].lower(): col["id"] for col in columns} + reversed_col_id_map = {v: k for k, v in col_id_map.items()} + + + items = get_all_items(board, monday) + + all_records = [] + for row in tqdm(items): + data = {} + data.update({"address": row['name']}) + data.update({"client": row['group']['title']}) + for col in row.get("column_values", []): + if col.get("id") in reversed_col_id_map: + if col.get("type") == "file": + value = col.get("value") + no_of_files = 0 + + if value: + value = json.loads(col["value"]) + no_of_files = len(value.get('files', [])) + data.update({reversed_col_id_map[col.get("id")]: no_of_files}) + elif "no show" in reversed_col_id_map[col.get("id")]: + def extract_number_from_text(text): + number_str = '' + + for char in text: + if char.isnumeric(): + number_str += char + elif number_str: + break # stop once a number sequence ends + + return int(number_str) if number_str else None + text = col.get("text") + if text is None: + data.update({ + reversed_col_id_map[col.get("id")]: col.get("text") + }) + else: + data.update({ + reversed_col_id_map[col.get("id")]: extract_number_from_text(text) + }) + else: + data.update({ + reversed_col_id_map[col.get("id")]: col.get("text") + }) + all_records.append(data) + +# Convert to DataFrame +df = pd.DataFrame(all_records) + +filtered_dfs = [] + +def get_df(df, column_name, success_critera, job_name=None): + _ = df[ + df[column_name].str.lower().isin(success_critera) + ].copy() + _["job_type"] = job_name + return _ + + +# RA +ra = get_df(df, "ra", ["completed rdsap 9.9", "completed rd sap 10"], "RA") +filtered_dfs.append(ra) + + +# ATT +att = get_df(df, "att", ["completed"], "ATT") +filtered_dfs.append(att) + +# V1 Coordination +v1 = get_df(df, "lite ima status".lower(), [ + "rc complete", +], "Coordination Stage 1 v1") +filtered_dfs.append(v1) + +# V2 Coordination +v2 = get_df(df, "ima-mtp status", ["ima-mtp completed"], "Coordination Stage 1 v2 remodel") +filtered_dfs.append(v2) + +# # V3 Coordination +# v3 = get_df(df, "") +# filtered_dfs.append(v3) + +# Coordination stage 2 Please complete +cors2 = df[ + df["rc stg. 2"].str.lower().isin(["to invoice"]) +] +cors2["joby_type"] = "Coordination Stage 2" +filtered_dfs.append(cors2) + +# Design stage 1 +# design1 = get_df(df, "", ["done"], "Design") +# filtered_dfs.append(design1) + +# Design revision +# design2 = get_df(df, "design revision invoice", [ +# "Rev. A to invoice".lower(), +# "Rev. B to invoice".lower(), +# "Rev. C to invoice".lower(), +# "Rev. D to invoice".lower(), +# ], "Design Revision") +# filtered_dfs.append(design2) + +# Lodgement Phase 1 +lodg1 = get_df(df, "tm phase 1 invoiced".lower(), ["to invoice"], "Lodgement Phase 1") +filtered_dfs.append(lodg1) + + +# Full Lodgement Phase +full_lodgement = get_df(df, "trustmark lodgement".lower(), ["done"], "Full lodgement phase 2") +filtered_dfs.append(full_lodgement) + +# POST EPC +post_epc = get_df(df, "post-epc status", ["uploaded & completed", "to invoice"], "POST EPC") +filtered_dfs.append(post_epc) + + +# POST EPR +post_epr = df[ + df["post-epc status"].str.lower().isin(["post epr completed"]) +].copy() +post_epr["job_type"] = "POST epr" +filtered_dfs.append(post_epr) + +# Post ATT +post_att = get_df(df, "post att status", ["uploaded & completed", "to invoice"], "POST ATT") +filtered_dfs.append(post_att) + +# Retrofit Evaluation +retro = get_df(df, "retrofit evaluation", ["done", "to invoice"], "Retrofit Evaluation") +filtered_dfs.append(retro) + +# RA NO Show +# ra_ns = df[ +# df["ra no show evidence"].fillna(-9999) != df["ra no show invoice"].fillna(-9999) +# ].copy() +# ra_ns["job_type"] = "RA NO SHOW" +# filtered_dfs.append(ra_ns) + + +# ATT NO Show +att_ns = df[ + (df["att no show evidence"].fillna(-9999) != df["att no show invoice"].fillna(-9999)) & + (df["att no show evidence"] != 0) +].copy() +att_ns["job_type"] = "ATT NO SHOW" +filtered_dfs.append(att_ns) + + +# Post visit no show +# epc_ns = df[ +# df["post epc no show evidence"].fillna(-9999) != df["post epc no show invoice"].fillna(-9999) +# ].copy() +# epc_ns["job_type"] = "post EPC NO SHOW" +# filtered_dfs.append(epc_ns) + +final_df = pd.concat(filtered_dfs).reset_index(drop=True) + +final_df["job_type"] = final_df["job_type"].str.lower() +rate_card_df["job_type"] = rate_card_df["job_type"].str.lower() + +# Now perform the merge +combined_with_rates = final_df.merge(rate_card_df, on="job_type", how="left") +import datetime +timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M') + +attribute = ['address', 'client', 'job_type', 'rate'] +combined_with_rates[attribute].to_excel(f'Northumberland LAD2 & HUG2_{timestamp}.xlsx', index=False) diff --git a/etl/month_end_automation_wave_2_no_12.py b/etl/month_end_automation_wave_2_no_12.py new file mode 100644 index 0000000..7fa4c1a --- /dev/null +++ b/etl/month_end_automation_wave_2_no_12.py @@ -0,0 +1,207 @@ +# Wave 2's month end automation +from tqdm import tqdm +from monday import MondayClient +from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset_ids +from pprint import pprint +import pandas as pd +import json + +monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY" +monday = MondayClient(monday_key) +# Shropshire Council HUG2 +board_ids = ["4718185486"] + +empty = "nothing on rate card" + +rate_card_data = { + "job_type": [ + "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", + "Design Archetype", "Design Repetitive", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", + "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", + "RA no show", "ATT no show", "post EPC no show", "Design Sign off" + ], + "rate": [ + "(185) - Kevin Check as this depends on property size", 115, 200, 125, 125, + empty, "(135)- Mariane said 'Remaining RC & design sign off", 185, 135, + 120, "(60) Post EPR, please verify", 65, 115, 60, + 50, 50, 50, "(60) - Please add price for design sign off, check with kevin and marianne" + ] +} + +rate_card_df = pd.DataFrame(rate_card_data) + +for board in tqdm(board_ids): + board_data = monday.boards.fetch_boards_by_id(board) + columns = board_data["data"]["boards"][0]["columns"] + col_id_map = {col["title"].lower(): col["id"] for col in columns} + reversed_col_id_map = {v: k for k, v in col_id_map.items()} + + + items = get_all_items(board, monday) + + all_records = [] + for row in tqdm(items): + data = {} + data.update({"address": row['name']}) + data.update({"client": row['group']['title']}) + for col in row.get("column_values", []): + if col.get("id") in reversed_col_id_map: + if col.get("type") == "file": + value = col.get("value") + no_of_files = 0 + + if value: + value = json.loads(col["value"]) + no_of_files = len(value.get('files', [])) + data.update({reversed_col_id_map[col.get("id")]: no_of_files}) + elif "no show" in reversed_col_id_map[col.get("id")]: + def extract_number_from_text(text): + number_str = '' + + for char in text: + if char.isnumeric(): + number_str += char + elif number_str: + break # stop once a number sequence ends + + return int(number_str) if number_str else None + text = col.get("text") + if text is None: + data.update({ + reversed_col_id_map[col.get("id")]: col.get("text") + }) + else: + data.update({ + reversed_col_id_map[col.get("id")]: extract_number_from_text(text) + }) + else: + data.update({ + reversed_col_id_map[col.get("id")]: col.get("text") + }) + all_records.append(data) + +# Convert to DataFrame +df = pd.DataFrame(all_records) + +filtered_dfs = [] + +def get_df(df, column_name, success_critera, job_name): + _ = df[ + df[column_name].str.lower().isin(success_critera) + ].copy() + _["job_type"] = job_name + return _ + + +# RA +ra = get_df(df, "ra", ["completed rdsap 9.9", "completed rd sap 10"], "RA") +filtered_dfs.append(ra) + + +# ATT +att = get_df(df, "pre- att", ["completed"], "ATT") +filtered_dfs.append(att) + +# V1 Coordination +v1 = get_df(df, "ima lite invoiced".lower(), [ + "to invoice", +], "Coordination Stage 1 v1") +filtered_dfs.append(v1) + +# V2 Coordination +v2 = get_df(df, "coordination v2 invoiced", ["to invoice"], "Coordination Stage 1 v2 remodel") +filtered_dfs.append(v2) + +# # V3 Coordination +# v3 = get_df(df, "") +# filtered_dfs.append(v3) + +# Coordination stage 2 Please complete +cors2 = df[ + df["rc stage 2 invoice"].str.lower().isin(["to invoice"]) +].copy() +cors2["joby_type"] = "Coordination Stage 2" +filtered_dfs.append(cors2) + +# Design stage 1 +# design1 = get_df(df, "", ["done"], "Design") +# filtered_dfs.append(design1) + +# Design revision +# design2 = get_df(df, "design revision invoice", [ +# "Rev. A to invoice".lower(), +# "Rev. B to invoice".lower(), +# "Rev. C to invoice".lower(), +# "Rev. D to invoice".lower(), +# ], "Design Revision") +# filtered_dfs.append(design2) + +# Design sign off +design_sign_pff = get_df(df, "design payment step 3", ["ready to invoice"], "design sign off") + +# Lodgement Phase 1 +lodg1 = get_df(df, "tm ph1 invoice status".lower(), ["to invoice"], "Lodgement Phase 1") +filtered_dfs.append(lodg1) + + +# Full Lodgement Phase +full_lodgement = get_df(df, "tm ph2 invoice status".lower(), ["to invoice"], "Full lodgement phase 2") +filtered_dfs.append(full_lodgement) + +# POST EPC +post_epc = get_df(df, "post-epc status", ["uploaded", "completed"], "POST EPC") +filtered_dfs.append(post_epc) + + +# POST EPR +post_epr = df[ + df["post-epc status"].str.lower().isin(["post epr completed"]) +].copy() +post_epr["job_type"] = "POST epr" +filtered_dfs.append(post_epr) + +# Post ATT +post_att = get_df(df, "post-att", ["uploaded", "completed", "to invoice"], "POST ATT") +filtered_dfs.append(post_att) + +# Retrofit Evaluation +retro = get_df(df, "retrofit evaluation", ["completed", "to invoice"], "Retrofit Evaluation") +filtered_dfs.append(retro) + +# RA NO Show +ra_ns = df[ + (df["ra no show evidence"].fillna(-9999) != df["ra no show invoice"].fillna(-9999)) & + (df["ra no show evidence"] != 0) +].copy() +ra_ns["job_type"] = "RA NO SHOW" +filtered_dfs.append(ra_ns) + + +# ATT NO Show +# att_ns = df[ +# df["att no show evidence"].fillna(-9999) != df["att no show invoice"].fillna(-9999) +# ].copy() +# att_ns["job_type"] = "ATT NO SHOW" +# filtered_dfs.append(att_ns) + + +# Post visit no show +epc_ns = df[ + (df["epc no show evidence"].fillna(-9999) != df["epc no show invoice"].fillna(-9999)) & + (df["epc no show evidence"] != 0) +].copy() +epc_ns["job_type"] = "post EPC NO SHOW" +filtered_dfs.append(epc_ns) + +final_df = pd.concat(filtered_dfs).reset_index(drop=True) + +final_df["job_type"] = final_df["job_type"].str.lower() +rate_card_df["job_type"] = rate_card_df["job_type"].str.lower() + +# Now perform the merge +combined_with_rates = final_df.merge(rate_card_df, on="job_type", how="left") +import datetime +timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M') + +attribute = ['address', 'client', 'job_type', 'rate'] +combined_with_rates[attribute].to_excel(f'Shropshire Council HUG2_{timestamp}.xlsx', index=False) diff --git a/etl/month_end_automation_wave_2_no_13.py b/etl/month_end_automation_wave_2_no_13.py new file mode 100644 index 0000000..d4a6a1d --- /dev/null +++ b/etl/month_end_automation_wave_2_no_13.py @@ -0,0 +1,200 @@ +# Wave 2's month end automation +from tqdm import tqdm +from monday import MondayClient +from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset_ids +from pprint import pprint +import pandas as pd +import json + +monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY" +monday = MondayClient(monday_key) +# Stonewater SHDF 3.0 - Operations +board_ids = ["6222522864"] + +rate_card_data = { + "job_type": [ + "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", + "Design Archetype", "Design Repetitive", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", + "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", + "RA no show", "ATT no show", "post EPC no show" + ], + "rate": [ + 259, 125, 280, 125, 125, + 650, 195, 175, 135, + 120, "Post EPR Please Marianne", 85, 125, 60, + 25, 25, 25 + ] +} + +rate_card_df = pd.DataFrame(rate_card_data) + +for board in tqdm(board_ids): + board_data = monday.boards.fetch_boards_by_id(board) + columns = board_data["data"]["boards"][0]["columns"] + col_id_map = {col["title"].lower(): col["id"] for col in columns} + reversed_col_id_map = {v: k for k, v in col_id_map.items()} + + + items = get_all_items(board, monday) + + all_records = [] + for row in tqdm(items): + data = {} + data.update({"address": row['name']}) + data.update({"client": row['group']['title']}) + for col in row.get("column_values", []): + if col.get("id") in reversed_col_id_map: + if col.get("type") == "file": + value = col.get("value") + no_of_files = 0 + + if value: + value = json.loads(col["value"]) + no_of_files = len(value.get('files', [])) + data.update({reversed_col_id_map[col.get("id")]: no_of_files}) + elif "no show" in reversed_col_id_map[col.get("id")]: + def extract_number_from_text(text): + number_str = '' + + for char in text: + if char.isnumeric(): + number_str += char + elif number_str: + break # stop once a number sequence ends + + return int(number_str) if number_str else None + text = col.get("text") + if text is None: + data.update({ + reversed_col_id_map[col.get("id")]: col.get("text") + }) + else: + data.update({ + reversed_col_id_map[col.get("id")]: extract_number_from_text(text) + }) + else: + data.update({ + reversed_col_id_map[col.get("id")]: col.get("text") + }) + all_records.append(data) + +# Convert to DataFrame +df = pd.DataFrame(all_records) + +filtered_dfs = [] + +def get_df(df, column_name, success_critera, job_name): + _ = df[ + df[column_name].str.lower().isin(success_critera) + ].copy() + _["job_type"] = job_name + return _ + +# RA +ra = get_df(df, "ra", ["completed rdsap 9.9", "completed rdsap 10"], "RA") +filtered_dfs.append(ra) + + +# ATT +att = get_df(df, "att", ["completed"], "ATT") +filtered_dfs.append(att) + +# V1 Coordination +v1 = get_df(df, "coordination status (mtp)".lower(), [ + "ima/mtp complete", +], "Coordination Stage 1 v1") +filtered_dfs.append(v1) + +# V2 Coordination +v2 = get_df(df, "v2 mtp status", ["ima/mtp complete"], "Coordination Stage 1 v2 remodel") +filtered_dfs.append(v2) + +# # V3 Coordination +# v3 = get_df(df, "") +# filtered_dfs.append(v3) + +# Coordination stage 2 Please complete +cors2 = df[ + df["rc stage 2"].str.lower().isin(["to invoice"]) +] +cors2["joby_type"] = "Coordination Stage 2" +filtered_dfs.append(cors2) + +# Design stage 1 +# design1 = get_df(df, "", ["done"], "Design") +# filtered_dfs.append(design1) + +# Design revision +# design2 = get_df(df, "design revision invoice", [ +# "Rev. A to invoice".lower(), +# "Rev. B to invoice".lower(), +# "Rev. C to invoice".lower(), +# "Rev. D to invoice".lower(), +# ], "Design Revision") +# filtered_dfs.append(design2) + +# Lodgement Phase 1 +# lodg1 = get_df(df, "tm ph1 invoice status".lower(), ["to invoice"], "Lodgement Phase 1") +# filtered_dfs.append(lodg1) + + +# Full Lodgement Phase +# full_lodgement = get_df(df, "tm ph2 invoice status".lower(), ["to invoice"], "Full lodgement phase 2") +# filtered_dfs.append(full_lodgement) + +# POST EPC +post_epc = get_df(df, "post epc", ["done"], "POST EPC") +filtered_dfs.append(post_epc) + + +# POST EPR +post_epr = df[ + df["post epc"].str.lower().isin(["post epr completed"]) +].copy() +post_epr["job_type"] = "POST EPR" +filtered_dfs.append(post_epr) + +# Post ATT +# post_att = get_df(df, "post-att", ["uploaded", "completed", "to invoice"], "POST ATT") +# filtered_dfs.append(post_att) + +# Retrofit Evaluation +retro = get_df(df, "retrofit evaluation", ["done"], "Retrofit Evaluation") +filtered_dfs.append(retro) + +# RA NO Show +ra_ns = df[ + (df["ra no show evidence"].fillna(-9999) != df["ra no show invoice"].fillna(-9999)) & + (df["ra no show evidence"] != 0 ) +].copy() +ra_ns["job_type"] = "RA NO SHOW" +filtered_dfs.append(ra_ns) + + +# ATT NO Show +# att_ns = df[ +# df["att no show evidence"].fillna(-9999) != df["att no show invoice"].fillna(-9999) +# ].copy() +# att_ns["job_type"] = "ATT NO SHOW" +# filtered_dfs.append(att_ns) + + +# Post visit no show +# epc_ns = df[ +# df["epc no show evidence"].fillna(-9999) != df["epc no show invoice"].fillna(-9999) +# ].copy() +# epc_ns["job_type"] = "post EPC NO SHOW" +# filtered_dfs.append(epc_ns) + +final_df = pd.concat(filtered_dfs).reset_index(drop=True) + +final_df["job_type"] = final_df["job_type"].str.lower() +rate_card_df["job_type"] = rate_card_df["job_type"].str.lower() + +# Now perform the merge +combined_with_rates = final_df.merge(rate_card_df, on="job_type", how="left") +import datetime +timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M') + +attribute = ['address', 'client', 'job_type', 'rate'] +combined_with_rates[attribute].to_excel(f'Stonewater SHDF 3.0 - Operations_{timestamp}.xlsx', index=False) diff --git a/etl/month_end_automation_wave_2_no_14.py b/etl/month_end_automation_wave_2_no_14.py new file mode 100644 index 0000000..4a644ee --- /dev/null +++ b/etl/month_end_automation_wave_2_no_14.py @@ -0,0 +1,196 @@ +# Wave 2's month end automation +from tqdm import tqdm +from monday import MondayClient +from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset_ids +from pprint import pprint +import pandas as pd +import json + +monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY" +monday = MondayClient(monday_key) +# Decent Homes Stonewater - Operations +board_ids = ["9319118237"] + +empty = "Rate card info missing" +rate_card_data_example = { + "job_type": [ + "RA", "ATT", "Coordination Stage 1 v1", + "Lodgement phase 1", "Full lodgement phase 2", + "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", + "RA no show", "ATT no show", "post EPC no show" + ], + "rate": [ + 259, 125, 280, 125, + 260, 126, 281, 126, + 262, 127, 282, 127, + ] +} + + +rate_card_df = pd.DataFrame(rate_card_data_example) + +for board in tqdm(board_ids): + board_data = monday.boards.fetch_boards_by_id(board) + columns = board_data["data"]["boards"][0]["columns"] + col_id_map = {col["title"].lower(): col["id"] for col in columns} + reversed_col_id_map = {v: k for k, v in col_id_map.items()} + + + items = get_all_items(board, monday) + + all_records = [] + for row in tqdm(items): + data = {} + data.update({"address": row['name']}) + data.update({"client": row['group']['title']}) + for col in row.get("column_values", []): + if col.get("id") in reversed_col_id_map: + if col.get("type") == "file": + value = col.get("value") + no_of_files = 0 + + if value: + value = json.loads(col["value"]) + no_of_files = len(value.get('files', [])) + data.update({reversed_col_id_map[col.get("id")]: no_of_files}) + elif "no show" in reversed_col_id_map[col.get("id")]: + def extract_number_from_text(text): + number_str = '' + + for char in text: + if char.isnumeric(): + number_str += char + elif number_str: + break # stop once a number sequence ends + + return int(number_str) if number_str else None + text = col.get("text") + if text is None: + data.update({ + reversed_col_id_map[col.get("id")]: col.get("text") + }) + else: + data.update({ + reversed_col_id_map[col.get("id")]: extract_number_from_text(text) + }) + else: + data.update({ + reversed_col_id_map[col.get("id")]: col.get("text") + }) + all_records.append(data) + +# Convert to DataFrame +df = pd.DataFrame(all_records) + +filtered_dfs = [] + +def get_df(df, column_name, success_critera, job_name): + _ = df[ + df[column_name].str.lower().isin(success_critera) + ].copy() + _["job_type"] = job_name + return _ + +# RA +ra = get_df(df, "ra", ["completed rdsap 9.9", "completed rdsap 10"], "RA") +filtered_dfs.append(ra) + + +# ATT +att = get_df(df, "att", ["completed"], "ATT") +filtered_dfs.append(att) + +# V1 Coordination +v1 = get_df(df, "v1 coordination status (ioe,mtp)".lower(), [ + "rc complete", +], "Coordination Stage 1 v1") +filtered_dfs.append(v1) + +# # V2 Coordination +# v2 = get_df(df, "mtp v2 status", ["rc v2 complete"], "V2 Coordination") +# filtered_dfs.append(v2) + +# # # V3 Coordination +# v3 = get_df(df, "v3 rc status", ["uploaded"], "V3 Coordination") +# filtered_dfs.append(v3) + +# v3 = get_df(df, "v3 invoice status", ["to be invoice"], "V3 Coordination") +# filtered_dfs.append(v3) + +# Coordination stage 2 Please complete +# cors2 = df[ +# df["rc stg. 2"].str.lower().isin(["to invoice", "completed"]) +# ] +# cors2["joby_type"] = "Coordination Stage 2" +# filtered_dfs.append(cors2) + +# Design stage 1 +# design1 = get_df(df, "design invoice status", ["to invoice"], "Design") +# filtered_dfs.append(design1) + +# Design revision +# design2 = get_df(df, "design revision invoice", [ +# "Rev. A to invoice".lower(), +# "Rev. B to invoice".lower(), +# "Rev. C to invoice".lower(), +# "Rev. D to invoice".lower(), +# ], "Design Revision") +# filtered_dfs.append(design2) + +# Lodgement Phase 1 +lodg1 = get_df(df, "phase 1 invoice status (lodgement)".lower(), ["to be invoice"], "Lodgement Phase 1") +filtered_dfs.append(lodg1) + + +# Full Lodgement Phase +full_lodgement = get_df(df, "lodgement invoice status".lower(), ["to invoice"], "Full lodgement phase 2") +filtered_dfs.append(full_lodgement) + +# POST EPC +post_epc = get_df(df, "post epc", ["completed & uploaded"], "POST EPC") +filtered_dfs.append(post_epc) + + +# POST EPR +post_epr = df[ + df["post epc"].str.lower().isin(["post epr completed"]) +].copy() +post_epr["job_type"] = "POST epr" +filtered_dfs.append(post_epr) + +# Post ATT +post_att = get_df(df, "post att", ["completed & uploaded"], "POST ATT") +filtered_dfs.append(post_att) + +# Retrofit Evaluation +retro = get_df(df, "retrofit evaluation", ["completed & uploaded"], "Retrofit Evaluation") +filtered_dfs.append(retro) + +# RA NO Show +ra_ns = df[ + (df["ra no show evidence"].fillna(-9999) != df["ra no show invoice"].fillna(-9999)) & + (df["ra no show evidence"] != 0) +].copy() +ra_ns["job_type"] = "RA NO SHOW" +filtered_dfs.append(ra_ns) + + +# ATT NO Show +# att_ns = df[ +# df["att no show evidence"].fillna(-9999) != df["att no show invoice"].fillna(-9999) +# ].copy() +# att_ns["job_type"] = "ATT NO SHOW" +# filtered_dfs.append(att_ns) + + +# Post visit no show +epc_ns = df[ + (df["post epc no show evidence"].fillna(-9999) != df["post epc no show invoice"].fillna(-9999)) & + (df["post epc no show evidence"] != 0) +].copy() +epc_ns["job_type"] = "post EPC NO SHOW" +filtered_dfs.append(epc_ns) + +final_df = pd.concat(filtered_dfs).reset_index(drop=True) + +final_df[['address', 'client', 'job_type']] \ No newline at end of file diff --git a/etl/month_end_automation_wave_2_no_15.py b/etl/month_end_automation_wave_2_no_15.py new file mode 100644 index 0000000..8644db9 --- /dev/null +++ b/etl/month_end_automation_wave_2_no_15.py @@ -0,0 +1,214 @@ +# Wave 2's month end automation +from tqdm import tqdm +from monday import MondayClient +from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset_ids +from pprint import pprint +import pandas as pd +import json + +monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY" +monday = MondayClient(monday_key) +# WCHG SHDF 2.1 Mansard +board_ids = ["5636990610"] + + +empty = "Rate card was empty" + +rate_card_data = { + "job_type": [ + "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", + "Design Archetype", "Design Repetitive", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", + "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", + "RA no show", "ATT no show", "post EPC no show", "Design Revision" + ], + "rate": [ + "259 (new RA rate for PAS2035:2023 - old rates for other works - to discuss with KN)", 40, 178.5, empty, empty, + empty, 180, 275, 135, + 120, "60 - please verify with Marianne", 45, 45, 40, + 25, 25, 25, "Please check price for design revision with Andreas" + ] +} + +rate_card_df = pd.DataFrame(rate_card_data) + + +for board in tqdm(board_ids): + board_data = monday.boards.fetch_boards_by_id(board) + columns = board_data["data"]["boards"][0]["columns"] + col_id_map = {col["title"].lower(): col["id"] for col in columns} + reversed_col_id_map = {v: k for k, v in col_id_map.items()} + + + items = get_all_items(board, monday) + + all_records = [] + for row in tqdm(items): + data = {} + data.update({"address": row['name']}) + data.update({"client": row['group']['title']}) + for col in row.get("column_values", []): + if col.get("id") in reversed_col_id_map: + if col.get("type") == "file": + value = col.get("value") + no_of_files = 0 + + if value: + value = json.loads(col["value"]) + no_of_files = len(value.get('files', [])) + data.update({reversed_col_id_map[col.get("id")]: no_of_files}) + elif "no show" in reversed_col_id_map[col.get("id")]: + def extract_number_from_text(text): + number_str = '' + + for char in text: + if char.isnumeric(): + number_str += char + elif number_str: + break # stop once a number sequence ends + + return int(number_str) if number_str else None + text = col.get("text") + if text is None: + data.update({ + reversed_col_id_map[col.get("id")]: col.get("text") + }) + else: + data.update({ + reversed_col_id_map[col.get("id")]: extract_number_from_text(text) + }) + else: + data.update({ + reversed_col_id_map[col.get("id")]: col.get("text") + }) + all_records.append(data) + +# Convert to DataFrame +df = pd.DataFrame(all_records) + +filtered_dfs = [] + +def get_df(df, column_name, success_critera, job_name=None): + _ = df[ + df[column_name].str.lower().isin(success_critera) + ].copy() + if job_name: + _["job_type"] = job_name + return _ + +# RA +ra = get_df(df, "ra status", ["completed rdsap 9.9", "completed rdsap 10"], "RA") +filtered_dfs.append(ra) + + +# PRE- ATT +att = get_df(df, "pre att", ["completed"], "ATT") +filtered_dfs.append(att) + +# V1 Coordination +v1 = get_df(df, "coordination status (ioe mtp)".lower(), [ + "rc complete", +], "Coordination Stage 1 v1") +filtered_dfs.append(v1) + +# V2 Coordination +v2 = get_df(df, "v2 coordination status (ioe mtp)", ["rc complete"], "Coordination Stage 1 v2 remodel") +filtered_dfs.append(v2) + +# # V3 Coordination +# v3 = get_df(df, "v3 rc status", ["uploaded"], "V3 Coordination") +# filtered_dfs.append(v3) + + +# v3 = get_df(df, "v3 invoice status", ["to be invoice"], "V3 Coordination") +# filtered_dfs.append(v3) + +# Coordination stage 2 Please complete +cors2 = df[ + df["rc stg 2"].str.lower().isin(["to invoice"]) +] +cors2["joby_type"] = "Coordination Stage 2" +filtered_dfs.append(cors2) + +# Design stage +design = get_df(df, "design invoice status", ["to invoice"]) + +# Design archeytpe +de = get_df(design, "prop type for invoicing", ["archetype"], "Design Archetype") +filtered_dfs.append(de) +# Design repetitive +de = get_df(design, "prop type for invoicing", ["repetitive"], "Design Repetitive") +filtered_dfs.append(de) + +# Design revision +design2 = get_df(df, "design revision invoice status", [ + "to invoice".lower(), +], "Design Revision") +filtered_dfs.append(design2) + +# Lodgement Phase 1 +lodg1 = get_df(df, "tm phase 1 invoice satus (lodgment)".lower(), ["to invoice"], "Lodgement Phase 1") +filtered_dfs.append(lodg1) + + +# Full Lodgement Phase +full_lodgement = get_df(df, "lodgement invoice status".lower(), ["to invoice"], "Full lodgement phase 2") +filtered_dfs.append(full_lodgement) + +# POST EPC +post_epc = get_df(df, "post-epc status", ["uploaded", "completed"], "POST EPC") +filtered_dfs.append(post_epc) + + +# # POST EPR +post_epr = df[ + df["post-epc status"].str.lower().isin(["post epr completed"]) +].copy() +post_epr["job_type"] = "POST epr" +filtered_dfs.append(post_epr) + +# Post ATT +post_att = get_df(df, "post-att status", ["uploaded", "completed"], "POST ATT") +filtered_dfs.append(post_att) + +# Retrofit Evaluation +retro = get_df(df, "retrofit evaluation", ["completed", "uploaded"], "Retrofit Evaluation") +filtered_dfs.append(retro) + +# RA NO Show +ra_ns = df[ + (df["ra no show evidence"].fillna(-9999) != df["ra no show invoice"].fillna(-9999)) & + (df["ra no show evidence"] != 0) +].copy() +ra_ns["job_type"] = "RA NO SHOW" +filtered_dfs.append(ra_ns) + + +# ATT NO Show +att_ns = df[ + (df["att no show evidence"].fillna(-9999) != df["att no show invoice"].fillna(-9999)) & + (df["att no show evidence"] != 0) +].copy() +att_ns["job_type"] = "ATT NO SHOW" +filtered_dfs.append(att_ns) + + +# Post visit no show +epc_ns = df[ + (df["epc no show evidence"].fillna(-9999) != df["epc no show invoice"].fillna(-9999)) & + (df["epc no show evidence"] != 0) +].copy() +epc_ns["job_type"] = "post EPC NO SHOW" +filtered_dfs.append(epc_ns) + +final_df = pd.concat(filtered_dfs).reset_index(drop=True) + +final_df["job_type"] = final_df["job_type"].str.lower() +rate_card_df["job_type"] = rate_card_df["job_type"].str.lower() + +# Now perform the merge +combined_with_rates = final_df.merge(rate_card_df, on="job_type", how="left") +import datetime +timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M') + +attribute = ['address', 'client', 'job_type', 'rate'] +combined_with_rates[attribute].to_excel(f'WCHG SHDF 2.1 Mansard {timestamp}.xlsx', index=False) diff --git a/etl/month_end_automation_wave_2_no_16.py b/etl/month_end_automation_wave_2_no_16.py new file mode 100644 index 0000000..0af8bfe --- /dev/null +++ b/etl/month_end_automation_wave_2_no_16.py @@ -0,0 +1,208 @@ +# Wave 2's month end automation +from tqdm import tqdm +from monday import MondayClient +from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset_ids +from pprint import pprint +import pandas as pd +import json + +monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY" +monday = MondayClient(monday_key) +# NCHA SHDF Wave 3 On Hold +board_ids = ["6946967610"] + +rate_card_data = { + "job_type": [ + "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", + "Design Archetype", "Design Repetitive", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", + "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", + "RA no show", "ATT no show", "post EPC no show" + ], + "rate": [ + 207.65, 101, 186.4, 98, 98, + 450, 150, 163, 135, 120, + "Marianne EPR Please", 45, 90.5, 40, + 25, 25, 25 + ] +} + +rate_card_df = pd.DataFrame(rate_card_data) + +for board in tqdm(board_ids): + board_data = monday.boards.fetch_boards_by_id(board) + columns = board_data["data"]["boards"][0]["columns"] + col_id_map = {col["title"].lower(): col["id"] for col in columns} + reversed_col_id_map = {v: k for k, v in col_id_map.items()} + + + items = get_all_items(board, monday) + + all_records = [] + for row in tqdm(items): + data = {} + data.update({"address": row['name']}) + data.update({"client": row['group']['title']}) + for col in row.get("column_values", []): + if col.get("id") in reversed_col_id_map: + if col.get("type") == "file": + value = col.get("value") + no_of_files = 0 + + if value: + value = json.loads(col["value"]) + no_of_files = len(value.get('files', [])) + data.update({reversed_col_id_map[col.get("id")]: no_of_files}) + elif "no show" in reversed_col_id_map[col.get("id")]: + def extract_number_from_text(text): + number_str = '' + + for char in text: + if char.isnumeric(): + number_str += char + elif number_str: + break # stop once a number sequence ends + + return int(number_str) if number_str else None + text = col.get("text") + if text is None: + data.update({ + reversed_col_id_map[col.get("id")]: col.get("text") + }) + else: + data.update({ + reversed_col_id_map[col.get("id")]: extract_number_from_text(text) + }) + else: + data.update({ + reversed_col_id_map[col.get("id")]: col.get("text") + }) + all_records.append(data) + +# Convert to DataFrame +df = pd.DataFrame(all_records) + +filtered_dfs = [] + +def get_df(df, column_name, success_critera, job_name): + _ = df[ + df[column_name].str.lower().isin(success_critera) + ].copy() + _["job_type"] = job_name + return _ + +# RA +ra = get_df(df, "ra", ["completed"], "RA") +filtered_dfs.append(ra) + + +# PRE- ATT +att = get_df(df, "att", ["completed"], "ATT") +filtered_dfs.append(att) + +# V1 Coordination +v1 = get_df(df, "coordination status".lower(), [ + "rc complete", +], "Coordination Stage 1 v1") +filtered_dfs.append(v1) + +# V2 Coordination +# v2 = get_df(df, "mtp v2 status", ["rc v2 complete"], "Coordination Stage 1 v2 remodel") +# filtered_dfs.append(v2) + +# # V3 Coordination +# v3 = get_df(df, "v3 rc status", ["uploaded"], "Coordination Stage 1 v3 remode") +# filtered_dfs.append(v3) + +# v3 = get_df(df, "v3 invoice status", ["to be invoice"], "V3 Coordination") +# filtered_dfs.append(v3) + +# Coordination stage 2 Please complete +cors2 = df[ + df["rc stg. 2"].str.lower().isin(["to invoice"]) +] +cors2["joby_type"] = "Coordination Stage 2" +filtered_dfs.append(cors2) + +# Design Archetype +# design1 = get_df(df, "design invoice status", ["to invoice"], "Design") +# filtered_dfs.append(design1) + + +# Design Repetitive + +# Design Revision +# design2 = get_df(df, "design revision invoice", [ +# "Rev. A to invoice".lower(), +# "Rev. B to invoice".lower(), +# "Rev. C to invoice".lower(), +# "Rev. D to invoice".lower(), +# ], "Design Revision") +# filtered_dfs.append(design2) + +# Lodgement Phase 1 +lodg1 = get_df(df, "lodg. phase 1 invoice status".lower(), ["to invoice"], "Lodgement Phase 1") +filtered_dfs.append(lodg1) + + +# Full Lodgement Phase +full_lodgement = get_df(df, "full lodgement invoice status".lower(), ["to invoice"], "Full lodgement phase 2") +filtered_dfs.append(full_lodgement) + +# POST EPC +post_epc = get_df(df, "lodged epc", ["complete", "complete & lodged",], "Post EPC") +filtered_dfs.append(post_epc) + + +# POST EPR +post_epr = df[ + df["lodged epc"].str.lower().isin(["post epr completed"]) +].copy() +post_epr["job_type"] = "POST EPR" +filtered_dfs.append(post_epr) + +# Post ATT +post_att = get_df(df, "post att", ["done", "post att complete"], "POST ATT") +filtered_dfs.append(post_att) + +# Retrofit Evaluation +retro = get_df(df, "retrofit evaluation", ["done"], "Retrofit Evaluation") +filtered_dfs.append(retro) + +# RA NO Show +ra_ns = df[ + (df["ra no show evidence"].fillna(-9999) != df["ra no show invoice"].fillna(-9999)) & + (df["ra no show evidence"] != 0) +].copy() +ra_ns["job_type"] = "RA NO SHOW" +filtered_dfs.append(ra_ns) + + +# ATT NO Show +att_ns = df[ + (df["att no show evidence"].fillna(-9999) != df["att no show invoice"].fillna(-9999)) & + (df["att no show evidence"] != 0) +].copy() +att_ns["job_type"] = "ATT NO SHOW" +filtered_dfs.append(att_ns) + + +# Post visit no show +# epc_ns = df[ +# df["epc no show evidence"].fillna(-9999) != df["epc no show invoice"].fillna(-9999) +# ].copy() +# epc_ns["job_type"] = "post EPC NO SHOW" +# filtered_dfs.append(epc_ns) + +final_df = pd.concat(filtered_dfs).reset_index(drop=True) + + +final_df["job_type"] = final_df["job_type"].str.lower() +rate_card_df["job_type"] = rate_card_df["job_type"].str.lower() + +# Now perform the merge +combined_with_rates = final_df.merge(rate_card_df, on="job_type", how="left") +import datetime +timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M') + +attribute = ['address', 'client', 'job_type', 'rate'] +combined_with_rates[attribute].to_excel(f'NCHA SHDF Wave 3 On Hold_{timestamp}.xlsx', index=False) \ No newline at end of file diff --git a/etl/month_end_automation_wave_2_no_3.py b/etl/month_end_automation_wave_2_no_3.py new file mode 100644 index 0000000..b349a7a --- /dev/null +++ b/etl/month_end_automation_wave_2_no_3.py @@ -0,0 +1,234 @@ +# Wave 2's month end automation + +from tqdm import tqdm +from monday import MondayClient +from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset_ids +from pprint import pprint +import pandas as pd +import json + +monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY" +monday = MondayClient(monday_key) +# Platform Housing W2 (in use) +board_ids = ["4796290860"] + +rate_card_data = { + "job_type": [ + "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", + "Design Archetype", "Design Repetitive", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", + "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", + "RA no show", "ATT no show", "post EPC no show", "Design Revision" + ], + "rate": [ + 259, 101, 210, 95, 95, + 450, 150, 195, 135, + 120, "(60)) - please confirm with Marianne, EPR", 45, 90.5, 42.4, + 25, 25, 25, "Please ask for Design Revision" + ] +} + +rate_card_df = pd.DataFrame(rate_card_data) + + +for board in tqdm(board_ids): + board_data = monday.boards.fetch_boards_by_id(board) + columns = board_data["data"]["boards"][0]["columns"] + col_id_map = {col["title"].lower(): col["id"] for col in columns} + reversed_col_id_map = {v: k for k, v in col_id_map.items()} + + + items = get_all_items(board, monday) + + all_records = [] + for row in tqdm(items): + data = {} + data.update({"address": row['name']}) + data.update({"client": row['group']['title']}) + for col in row.get("column_values", []): + if col.get("id") in reversed_col_id_map: + if col.get("type") == "file": + value = col.get("value") + no_of_files = 0 + + if value: + value = json.loads(col["value"]) + no_of_files = len(value.get('files', [])) + data.update({reversed_col_id_map[col.get("id")]: no_of_files}) + elif "no show" in reversed_col_id_map[col.get("id")]: + def extract_number_from_text(text): + number_str = '' + + for char in text: + if char.isnumeric(): + number_str += char + elif number_str: + break # stop once a number sequence ends + + return int(number_str) if number_str else None + text = col.get("text") + if text is None: + data.update({ + reversed_col_id_map[col.get("id")]: col.get("text") + }) + else: + data.update({ + reversed_col_id_map[col.get("id")]: extract_number_from_text(text) + }) + else: + data.update({ + reversed_col_id_map[col.get("id")]: col.get("text") + }) + all_records.append(data) + +# Convert to DataFrame +df = pd.DataFrame(all_records) + +filtered_dfs = [] + +# RA +ra = df[ + df["ra"].str.lower().isin(["completed rdsap 10", "completed rdsap 9.9", "completed", "complete"]) +].copy() +ra["job_type"] = "RA" +filtered_dfs.append(ra) + + +# ATT +att = df[ + df["att"].str.lower().isin(["completed"]) +].copy() +att["job_type"] = "ATT" +filtered_dfs.append(att) + +# V1 Coordination +v1 = df[ + df["coordination status"].str.lower().isin(["ima/mtp completed"]) +].copy() +v1["job_type"] = "Coordination Stage 1 v1" +filtered_dfs.append(v1) + +# V2 Coordination +_ = df[df["v2 mtp status"].fillna('').str.lower().isin(['v2 ima-mtp completed', 'v2 completed'])].copy() +_["job_type"] = "Coordination Stage 1 v2 remodel" +filtered_dfs.append(_) + +# V3 Coordination +# v3 = df[ +# df["v3 invoiced"].str.lower().isin(["to be invoiced"]) +# ].copy() +# v3["job_type"] = "Coordination Stage 1 v3 remodel" +# filtered_dfs.append(v3) + +# Coordination stage 2 Please complete +cors2 = df[ + df["rc stg. 2"].str.lower().isin(["to invoice"]) +] +cors2["joby_type"] = "Coordination Stage 2" +filtered_dfs.append(cors2) + +# Design Archetype +design1 = df[ + df["design invoice"].str.lower().isin(["complete pending rc"]) +].copy() +design1 = design1[design1["design type for invoicing"].str.lower().isin(['archetype'])].copy() +design1["job_type"] = "Design Archetype" +filtered_dfs.append(design1) + +# Design Repetitive +design1 = df[ + df["design invoice"].str.lower().isin(["complete pending rc"]) +].copy() +design1 = design1[design1["design type for invoicing"].str.lower().isin(['repetitive'])].copy() +design1["job_type"] = "Design repetitive" +filtered_dfs.append(design1) + +# Design Revision +design_revision = df[ + df["design revision invoice status"].str.lower().isin(["to invoice"]) +].copy() +design_revision["job_type"] = "Design repetitive" +filtered_dfs.append(design_revision) + +# Lodgement Phase 1 +lodg1 = df[ + df["phase 1 invoice status (lodgement)"].str.lower().isin(["done"]) +].copy() +lodg1["job_type"] = "Lodgement Phase 1" +filtered_dfs.append(lodg1) + +# Full Lodgement Phase +lodg2 = df[ + df["lodgement invoice status (lodgement)"].str.lower().isin(["to invoice"]) +].copy() +lodg2["job_type"] = "Full lodgement phase 2" +filtered_dfs.append(lodg2) + +# POST EPC +post_epc = df[ + df["post epc"].str.lower().isin(["success", "pics uploaded"]) +].copy() +post_epc["job_type"] = "POST EPC" +filtered_dfs.append(post_epc) + + +# POST EPR +post_epr = df[ + df["post epc"].str.lower().isin(["post epr completed"]) +].copy() +post_epr["job_type"] = "POST EPR" +filtered_dfs.append(post_epr) + + +# Post ATT +post_att = df[ + df["post att"].str.lower().isin(["uploaded"]) +].copy() +post_att["job_type"] = "POST ATT" +filtered_dfs.append(post_att) + + +# Retrofit Evaluation +retro = df[ + df["retrofit evaluation"].str.lower().isin(["uploaded", "completed", "to invoice"]) +].copy() +retro["job_type"] = "Retrofit Evaluation" +filtered_dfs.append(retro) + +# RA NO Show +ra_ns = df[ + (df["ra no show evidence"].fillna(-9999) != df["ra no show invoice"].fillna(-9999)) & + (df["ra no show evidence"] != 0) +].copy() +ra_ns["job_type"] = "RA NO SHOW" +filtered_dfs.append(ra_ns) + + +# ATT NO Show +att_ns = df[ + (df["att no show evidence"].fillna(-9999) != df["att no show invoice"].fillna(-9999)) & + (df["att no show evidence"] != 0) +].copy() +att_ns["job_type"] = "ATT NO SHOW" +filtered_dfs.append(att_ns) + + +# Post visit no show +epc_ns = df[ + (df["post epc no show evidence"].fillna(-9999) != df["post epc no show invoice"].fillna(-9999)) & + (df["post epc no show evidence"] != 0) +].copy() +epc_ns["job_type"] = "Post EPC NO SHOW" +filtered_dfs.append(epc_ns) + +final_df = pd.concat(filtered_dfs).reset_index(drop=True) + +final_df["job_type"] = final_df["job_type"].str.lower() +rate_card_df["job_type"] = rate_card_df["job_type"].str.lower() + +# Now perform the merge +combined_with_rates = final_df.merge(rate_card_df, on="job_type", how="left") +import datetime +timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M') + +attribute = ['address', 'client', 'job_type', 'rate'] +combined_with_rates[attribute].to_excel(f'Platform Housing W2 (in use)_{timestamp}.xlsx', index=False) diff --git a/etl/month_end_automation_wave_2_no_4.py b/etl/month_end_automation_wave_2_no_4.py new file mode 100644 index 0000000..30b949f --- /dev/null +++ b/etl/month_end_automation_wave_2_no_4.py @@ -0,0 +1,241 @@ +# Wave 2's month end automation + +from tqdm import tqdm +from monday import MondayClient +from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset_ids +from pprint import pprint +import pandas as pd +import json + +monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY" +monday = MondayClient(monday_key) +# Stonewater (in use) +board_ids = ["3584401309"] + +rate_card_data = { + "job_type": [ + "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", + "Design Archetype", "Design Repetitive", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", + "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", + "RA no show", "ATT no show", "post EPC no show", "Design Revision" + ], + "rate": [ + 165.75, 72.25, 174.25, 174.25, 174.25, + 175, 175, 124.25, 135, + 120, "(60) - please check with Marianne", 45, 63.75, 34, + 25, 25, 25, "Please ask marianne or kev for design revision" + ] +} + +rate_card_df = pd.DataFrame(rate_card_data) + +for board in tqdm(board_ids): + board_data = monday.boards.fetch_boards_by_id(board) + columns = board_data["data"]["boards"][0]["columns"] + col_id_map = {col["title"].lower(): col["id"] for col in columns} + reversed_col_id_map = {v: k for k, v in col_id_map.items()} + + + items = get_all_items(board, monday) + + all_records = [] + for row in tqdm(items): + data = {} + data.update({"address": row['name']}) + data.update({"client": row['group']['title']}) + for col in row.get("column_values", []): + if col.get("id") in reversed_col_id_map: + if col.get("type") == "file": + value = col.get("value") + no_of_files = 0 + + if value: + value = json.loads(col["value"]) + no_of_files = len(value.get('files', [])) + data.update({reversed_col_id_map[col.get("id")]: no_of_files}) + elif "no show" in reversed_col_id_map[col.get("id")]: + def extract_number_from_text(text): + number_str = '' + + for char in text: + if char.isnumeric(): + number_str += char + elif number_str: + break # stop once a number sequence ends + + return int(number_str) if number_str else None + text = col.get("text") + if text is None: + data.update({ + reversed_col_id_map[col.get("id")]: col.get("text") + }) + else: + data.update({ + reversed_col_id_map[col.get("id")]: extract_number_from_text(text) + }) + else: + data.update({ + reversed_col_id_map[col.get("id")]: col.get("text") + }) + all_records.append(data) + +# Convert to DataFrame +df = pd.DataFrame(all_records) + +filtered_dfs = [] + +# RA +ra = df[ + df["ra"].str.lower().isin(["completed rdsap 10", "completed rdsap 9.9"]) +].copy() +ra["job_type"] = "RA" +filtered_dfs.append(ra) + + +# ATT +att = df[ + df["att"].str.lower().isin(["completed"]) +].copy() +att["job_type"] = "ATT" +filtered_dfs.append(att) + +# V1 Coordination +v1 = df[ + df["v1 coordination status (ioe,mtp)"].str.lower().isin(["rc complete"]) +].copy() +v1["job_type"] = "Coordination Stage 1 v1" +filtered_dfs.append(v1) + +# V2 Coordination +_ = df[df["mtp v2 status"].str.lower().isin(['rc v2 complete'])].copy() +_["job_type"] = "Coordination Stage 1 v2 remodel" +filtered_dfs.append(_) + +# V2 Coordination +_ = df[df["mtp v2 invoiced"].str.lower().isin(['needs to be invoiced'])].copy() +_["job_type"] = "Coordination Stage 1 v2 remodel" +filtered_dfs.append(_) + +# V3 Coordination +v3 = df[df["v3 rc status"].str.lower().isin(['uploaded'])].copy() +v3["job_type"] = "Coordination Stage 1 v3 remodel" +filtered_dfs.append(_) + +# V3 Coordination +v3 = df[df["v3 invoice status"].str.lower().isin(['to be invoiced'])].copy() +v3["job_type"] = "Coordination Stage 1 v3 remodel" +filtered_dfs.append(_) + +# Coordination stage 2 Please complete +cors2 = df[ + df["rc stg. 2 status"].str.lower().isin(["to invoice", "completed"]) +] +cors2["joby_type"] = "Coordination Stage 2" +filtered_dfs.append(cors2) + +# Design Archetype +design1 = df[ + df["design invoice status"].str.lower().isin(["complete", "to invoice"]) +].copy() +design1 = design1[design1["design type"].str.lower().isin(["archetype"])].copy() +design1["job_type"] = "Design Archetype" +filtered_dfs.append(design1) + +# Design Repetitive +design1 = df[ + df["design invoice status"].str.lower().isin(["complete", "to invoice"]) +].copy() +design1 = design1[design1["design type"].str.lower().isin(["repetitive"])].copy() +design1["job_type"] = "Design Repetitive" +filtered_dfs.append(design1) + +# Design Revision +design_revision = df[ + df["design revision invoice status"].str.lower().isin(["to invoice"]) +].copy() +design_revision["job_type"] = "Design repetitive" +filtered_dfs.append(design_revision) + +# Lodgement Phase 1 +lodg1 = df[ + df["phase 1 invoice status (lodgement)"].str.lower().isin(["done", "to be invoiced"]) +].copy() +lodg1["job_type"] = "Lodgement Phase 1" +filtered_dfs.append(lodg1) + +# Full Lodgement Phase +_ = df[ + df["lodgement invoice status"].str.lower().isin(["to invoice"]) +].copy() +_["job_type"] = "Full lodgement phase 2" +filtered_dfs.append(_) + +# POST EPC +post_epc = df[ + df["post epc"].str.lower().isin(["completed & uploaded"]) +].copy() +post_epc["job_type"] = "POST EPC" +filtered_dfs.append(post_epc) + + +# POST EPR +post_epr = df[ + df["post epc"].str.lower().isin(["post epr completed"]) +].copy() +post_epr["job_type"] = "POST EPR" +filtered_dfs.append(post_epr) + +# Post ATT +post_att = df[ + df["post att"].str.lower().isin(["completed & uploaded"]) +].copy() +post_att["job_type"] = "POST ATT" +filtered_dfs.append(post_att) + + +# Retrofit Evaluation +retro = df[ + df["retrofit evaluation"].str.lower().isin(["completed & uploaded"]) +].copy() +retro["job_type"] = "Retrofit Evaluation" +filtered_dfs.append(retro) + +# RA NO Show +ra_ns = df[ + (df["ra no show evidence"].fillna(-9999) != df["ra no show invoice"].fillna(-9999)) & + (df["ra no show evidence"] != 0) +].copy() +ra_ns["job_type"] = "RA NO SHOW" +filtered_dfs.append(ra_ns) + + +# ATT NO Show +att_ns = df[ + (df["att no show evidence"].fillna(-9999) != df["att no show invoice"].fillna(-9999)) & + (df["att no show evidence"] != 0) +].copy() +att_ns["job_type"] = "ATT NO SHOW" +filtered_dfs.append(att_ns) + + +# Post visit no show +epc_ns = df[ + (df["post epc no show evidence"].fillna(-9999) != df["post epc no show invoice"].fillna(-9999)) & + (df["post epc no show evidence"] != 0) +].copy() +epc_ns["job_type"] = "post EPC NO SHOW" +filtered_dfs.append(epc_ns) + +final_df = pd.concat(filtered_dfs).reset_index(drop=True) + + +final_df["job_type"] = final_df["job_type"].str.lower() +rate_card_df["job_type"] = rate_card_df["job_type"].str.lower() + +# Now perform the merge +combined_with_rates = final_df.merge(rate_card_df, on="job_type", how="left") +import datetime +timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M') + +attribute = ['address', 'client', 'job_type', 'rate'] +combined_with_rates[attribute].to_excel(f'Stonewater - (in use)_{timestamp}.xlsx', index=False) diff --git a/etl/month_end_automation_wave_2_no_5.py b/etl/month_end_automation_wave_2_no_5.py new file mode 100644 index 0000000..f26d3b0 --- /dev/null +++ b/etl/month_end_automation_wave_2_no_5.py @@ -0,0 +1,233 @@ +# Wave 2's month end automation + +from tqdm import tqdm +from monday import MondayClient +from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset_ids +from pprint import pprint +import pandas as pd +import json + +monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY" +monday = MondayClient(monday_key) +# ECO 4 NCHA Almshouses Operations +board_ids = ["9136254638"] + +rate_card_data = { + "job_type": [ + "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", + "Design Archetype", "Design Repetitive", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", + "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", + "RA no show", "ATT no show", "post EPC no show" + ], + "rate": [ + 259, 125, 280, 125, 125, + 650, 195, 175, 135, + 120, "Post EPR Please Marianne", 85, 125, 60, + 25, 25, 25 + ] +} + +rate_card_df = pd.DataFrame(rate_card_data) + + +for board in tqdm(board_ids): + board_data = monday.boards.fetch_boards_by_id(board) + columns = board_data["data"]["boards"][0]["columns"] + col_id_map = {col["title"].lower(): col["id"] for col in columns} + reversed_col_id_map = {v: k for k, v in col_id_map.items()} + + + items = get_all_items(board, monday) + + all_records = [] + for row in tqdm(items): + data = {} + data.update({"address": row['name']}) + data.update({"client": row['group']['title']}) + for col in row.get("column_values", []): + if col.get("id") in reversed_col_id_map: + if col.get("type") == "file": + value = col.get("value") + no_of_files = 0 + + if value: + value = json.loads(col["value"]) + no_of_files = len(value.get('files', [])) + data.update({reversed_col_id_map[col.get("id")]: no_of_files}) + elif "no show" in reversed_col_id_map[col.get("id")]: + def extract_number_from_text(text): + number_str = '' + + for char in text: + if char.isnumeric(): + number_str += char + elif number_str: + break # stop once a number sequence ends + + return int(number_str) if number_str else None + text = col.get("text") + if text is None: + data.update({ + reversed_col_id_map[col.get("id")]: col.get("text") + }) + else: + data.update({ + reversed_col_id_map[col.get("id")]: extract_number_from_text(text) + }) + else: + data.update({ + reversed_col_id_map[col.get("id")]: col.get("text") + }) + all_records.append(data) + +# Convert to DataFrame +df = pd.DataFrame(all_records) + +filtered_dfs = [] + +# RA +ra = df[ + df["retrofit assessment"].str.lower().isin(["completed rdsap 10", "completed rdsap 9.9"]) +].copy() +ra["job_type"] = "RA" +filtered_dfs.append(ra) + + +# ATT +att = df[ + df["pre att"].str.lower().isin(["completed"]) +].copy() +att["job_type"] = "ATT" +filtered_dfs.append(att) + +# V1 Coordination +v1 = df[ + df["coordination status"].str.lower().isin(["ioe/mtp complete"]) +].copy() +v1["job_type"] = "Coordination Stage 1 v1" +filtered_dfs.append(v1) + +# V2 Coordination +# _ = df[df["mtp v2 invoiced"].str.lower().isin(['done', 'needs to be invoiced'])].copy() +# _["job_type"] = "Coordination Stage 1 v2 remodel" +# filtered_dfs.append(_) + +# V3 Coordination +# v3 = df[df["v3 rc status"].str.lower().isin(['uploaded'])].copy() +# v3["job_type"] = "Coordination Stage 1 v3 remodel" +# filtered_dfs.append(_) + +# Coordination stage 2 Please complete +cors2 = df[ + df["rc stage 2"].str.lower().isin(["to invoice",]) +] +cors2["joby_type"] = "Coordination Stage 2" +filtered_dfs.append(cors2) + +# Design stage 1 +design1 = df[ + df["retrofit design status"].str.lower().isin(["to invoice"]) +].copy() +design1 = design1[design1["design type"].str.lower().isin(["archetype"])].copy() +design1["job_type"] = "Design Archetype" +filtered_dfs.append(design1) + +# Design stage 2 +design2 = df[ + df["retrofit design status"].str.lower().isin(["to invoice"]) +].copy() +design2 = design2[design2["design type"].str.lower().isin(["repetitive"])].copy() +design2["job_type"] = "Design Repetitive" +filtered_dfs.append(design2) + +# Design revision +design2 = df[ + df["retrofit design status"].str.lower().isin(["to invoice"]) +].copy() +design2 = design2[design2["design revision"].str.lower().isin(["A", "B", "C"])].copy() +design2["job_type"] = "Design Repetitive" +filtered_dfs.append(design2) + + +# Lodgement Phase 1 +# lodg1 = df[ +# df["phase 1 invoice status (lodgement)"].str.lower().isin(["done", "to be invoiced"]) +# ].copy() +# lodg1["job_type"] = "Lodgement Phase 1" +# filtered_dfs.append(lodg1) + +# Full Lodgement Phase +_ = df[ + df["trustmark lodgement"].str.lower().isin(["done"]) +].copy() +_["job_type"] = "Full lodgement phase 2" +filtered_dfs.append(_) + +# POST EPC +post_epc = df[ + df["post epc status"].str.lower().isin(["done"]) +].copy() +post_epc["job_type"] = "POST EPC" +filtered_dfs.append(post_epc) + + +# POST EPR +post_epr = df[ + df["post epc status"].str.lower().isin(["post epr completed"]) +].copy() +post_epr["job_type"] = "POST epr" +filtered_dfs.append(post_epr) + +# Post ATT +post_att = df[ + df["post att status"].str.lower().isin(["done"]) +].copy() +post_att["job_type"] = "POST ATT" +filtered_dfs.append(post_att) + + +# Retrofit Evaluation +retro = df[ + df["retrofit evaluation"].str.lower().isin(["done"]) +].copy() +retro["job_type"] = "Retrofit Evaluation" +filtered_dfs.append(retro) + +# RA NO Show +ra_ns = df[ + (df["ra no show evidence"].fillna(-9999) != df["ra no show invoice"].fillna(-9999)) & + (df["ra no show evidence"] != 0) +].copy() +ra_ns["job_type"] = "RA NO SHOW" +filtered_dfs.append(ra_ns) + + +# ATT NO Show +att_ns = df[ + (df["pre att no show evidence"].fillna(-9999) != df["pre att no show invoice"].fillna(-9999)) & + (df["pre att no show evidence"] != 0) +].copy() +att_ns["job_type"] = "ATT NO SHOW" +filtered_dfs.append(att_ns) + + +# Post visit no show +epc_ns = df[ + (df["epc no show evidence"].fillna(-9999) != df["epc no show invoice"].fillna(-9999)) & + (df["epc no show evidence"] != 0 ) +].copy() +epc_ns["job_type"] = "post EPC NO SHOW" +filtered_dfs.append(epc_ns) + +final_df = pd.concat(filtered_dfs).reset_index(drop=True) + +final_df["job_type"] = final_df["job_type"].str.lower() +rate_card_df["job_type"] = rate_card_df["job_type"].str.lower() + +# Now perform the merge +combined_with_rates = final_df.merge(rate_card_df, on="job_type", how="left") +import datetime +timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M') + +attribute = ['address', 'client', 'job_type', 'rate'] +combined_with_rates[attribute].to_excel(f'ECO 4 NCHA Almshouses Operations_{timestamp}.xlsx', index=False) diff --git a/etl/month_end_automation_wave_2_no_6.py b/etl/month_end_automation_wave_2_no_6.py new file mode 100644 index 0000000..0ad996f --- /dev/null +++ b/etl/month_end_automation_wave_2_no_6.py @@ -0,0 +1,207 @@ +# Wave 2's month end automation + +from tqdm import tqdm +from monday import MondayClient +from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset_ids +from pprint import pprint +import pandas as pd +import json + +monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY" +monday = MondayClient(monday_key) +#ECO 4 Wates Operations +board_ids = ["9520779048"] + +rate_card_data = { + "job_type": [ + "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", + "Design Archetype", "Design Repetitive", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", + "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", + "RA no show", "ATT no show", "post EPC no show" + ], + "rate": [ + 259, 125, 280, 125, 125, + 650, 195, 175, 135, + 120, "Post EPR Please Marianne", 85, 125, 60, + 25, 25, 25 + ] +} + +rate_card_df = pd.DataFrame(rate_card_data) + +for board in tqdm(board_ids): + board_data = monday.boards.fetch_boards_by_id(board) + columns = board_data["data"]["boards"][0]["columns"] + col_id_map = {col["title"].lower(): col["id"] for col in columns} + reversed_col_id_map = {v: k for k, v in col_id_map.items()} + + + items = get_all_items(board, monday) + + all_records = [] + for row in tqdm(items): + data = {} + data.update({"address": row['name']}) + data.update({"client": row['group']['title']}) + for col in row.get("column_values", []): + if col.get("id") in reversed_col_id_map: + if col.get("type") == "file": + value = col.get("value") + no_of_files = 0 + + if value: + value = json.loads(col["value"]) + no_of_files = len(value.get('files', [])) + data.update({reversed_col_id_map[col.get("id")]: no_of_files}) + elif "no show" in reversed_col_id_map[col.get("id")]: + def extract_number_from_text(text): + number_str = '' + + for char in text: + if char.isnumeric(): + number_str += char + elif number_str: + break # stop once a number sequence ends + + return int(number_str) if number_str else None + text = col.get("text") + if text is None: + data.update({ + reversed_col_id_map[col.get("id")]: col.get("text") + }) + else: + data.update({ + reversed_col_id_map[col.get("id")]: extract_number_from_text(text) + }) + else: + data.update({ + reversed_col_id_map[col.get("id")]: col.get("text") + }) + all_records.append(data) + +# Convert to DataFrame +df = pd.DataFrame(all_records) + +filtered_dfs = [] + +def get_df(df, column_name, success_critera, job_name): + _ = df[ + df[column_name].str.lower().isin(success_critera) + ].copy() + _["job_type"] = job_name + return _ + +# RA +ra = get_df(df, "ra status", ["completed & uploaded"], "RA") +filtered_dfs.append(ra) + + +# ATT +att = get_df(df, "pre att status", ["completed & uploaded"], "ATT") +filtered_dfs.append(att) + +# V1 Coordination +v1 = get_df(df, "Coordination Status IOE/MTP".lower(), [ + "(V1) IOE/MTP Complete".lower(), + "(V2) IOE/MTP Complete".lower(), + "(V3) IOE/MTP Complete".lower(), +], "Coordination Stage 1 v1") +filtered_dfs.append(v1) + +# V2 Coordination +# _ = df[df["mtp v2 invoiced"].str.lower().isin(['done', 'needs to be invoiced'])].copy() +# _["job_type"] = "Coordination Stage 1 v2 remodel" +# filtered_dfs.append(_) + +# V3 Coordination +# v3 = df[df["v3 rc status"].str.lower().isin(['uploaded'])].copy() +# v3["job_type"] = "Coordination Stage 1 v3 remodel" +# filtered_dfs.append(_) + +# Coordination stage 2 Please complete +cors2 = df[ + df["rc stage 2 status"].str.lower().isin(["to invoice"]) +] +cors2["joby_type"] = "Coordination Stage 2" +filtered_dfs.append(cors2) + +# Design stage 1 +design1 = get_df(df, "retrofit design status", ["completed"], "Design") +filtered_dfs.append(design1) + +# Design stage 2 +design2 = get_df(df, "design revision invoice", [ + "Rev. A to invoice".lower(), + "Rev. B to invoice".lower(), + "Rev. C to invoice".lower(), + "Rev. D to invoice".lower(), +], "Design Repetitive") +filtered_dfs.append(design2) + +# Lodgement Phase 1 +# lodg1 = df[ +# df["phase 1 invoice status (lodgement)"].str.lower().isin(["done", "to be invoiced"]) +# ].copy() +# lodg1["job_type"] = "Lodgement Phase 1" +# filtered_dfs.append(lodg1) + +# Full Lodgement Phase +full_lodgement = get_df(df, "full lodgement", ["completed"], "Full lodgement phase 2") +filtered_dfs.append(full_lodgement) + +# POST EPC +post_epc = get_df(df, "post epc & evaluation status", ["uploaded"], "POST EPC") +filtered_dfs.append(post_epc) + + +# POST EPR +post_epr = df[ + df["post epc & evaluation status"].str.lower().isin(["post epr completed"]) +].copy() +post_epr["job_type"] = "POST epr" +filtered_dfs.append(post_epr) + +# Post ATT +post_att = get_df(df, "post att status", ["uploaded"], "POST ATT") +filtered_dfs.append(post_att) + + +# Retrofit Evaluation +retro = get_df(df, "post epc & evaluation status", ["uploaded"], "Retrofit Evaluation") +filtered_dfs.append(retro) + +# RA NO Show +# ra_ns = df[ +# df["ra no show evidence"].fillna(-9999) != df["ra no show invoice"].fillna(-9999) +# ].copy() +# ra_ns["job_type"] = "RA NO SHOW" +# filtered_dfs.append(ra_ns) + + +# # ATT NO Show +# att_ns = df[ +# df["pre att no show evidence"].fillna(-9999) != df["pre att no show invoice"].fillna(-9999) +# ].copy() +# att_ns["job_type"] = "ATT NO SHOW" +# filtered_dfs.append(att_ns) + + +# # Post visit no show +# epc_ns = df[ +# df["epc no show evidence"].fillna(-9999) != df["epc no show invoice"].fillna(-9999) +# ].copy() +# epc_ns["job_type"] = "post EPC NO SHOW" +# filtered_dfs.append(epc_ns) + +final_df = pd.concat(filtered_dfs).reset_index(drop=True) + +final_df["job_type"] = final_df["job_type"].str.lower() +rate_card_df["job_type"] = rate_card_df["job_type"].str.lower() + +# Now perform the merge +combined_with_rates = final_df.merge(rate_card_df, on="job_type", how="left") +import datetime +timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M') + +attribute = ['address', 'client', 'job_type', 'rate'] +combined_with_rates[attribute].to_excel(f'ECO 4 Wates Operations_{timestamp}.xlsx', index=False) diff --git a/etl/month_end_automation_wave_2_no_7.py b/etl/month_end_automation_wave_2_no_7.py new file mode 100644 index 0000000..214c233 --- /dev/null +++ b/etl/month_end_automation_wave_2_no_7.py @@ -0,0 +1,257 @@ +# Wave 2's month end automation + +from tqdm import tqdm +from monday import MondayClient +from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset_ids +from pprint import pprint +import pandas as pd +import json + +monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY" +monday = MondayClient(monday_key) +#Home Group Wave 2SP+ +board_ids = ["4254419092"] + +rate_card_data_sp_plus = { + "job_type": [ + "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", + "Design Archetype", "Design Repetitive", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", + "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", + "RA no show", "ATT no show", "post EPC no show" + ], + "rate": [ + 170, 70, 200, "check with Kevin", "check with Kevin", + 470, 155, 165, 135, + 120, "60 but check with Kevin as EPR", 45, 70, 40, + 30, 30, 30 + ] +} + +emp_msg = "was empty in rate card - ask Marianne/Kevin" +rate_card_data_net_zero = { + "job_type": [ + "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", + "Design Archetype", "Design Repetitive", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", + "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", + "RA no show", "ATT no show", "post EPC no show" + ], + "rate": [ + 170, 70, 200, emp_msg, emp_msg, + 325, 140, 165, 135, + 120, "60 but check with Kevin as EPR", 45, 70, 40, + 30, 30, 30 + ] +} + +error_message = "Unsure which client this one is - sorry!" +rate_card_data_error_msg= { + "job_type": [ + "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", + "Design Archetype", "Design Repetitive", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", + "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", + "RA no show", "ATT no show", "post EPC no show" + ], + "rate": [ + error_message, error_message, error_message, error_message, error_message, + error_message, error_message, error_message, error_message, + error_message, error_message, error_message, error_message, error_message, + error_message, error_message, error_message + ] +} + +rate_card_df_sp_plus = pd.DataFrame(rate_card_data_sp_plus) +rate_card_df_net_zero = pd.DataFrame(rate_card_data_net_zero) +rate_card_df_error_message = pd.DataFrame(rate_card_data_error_msg) + + +for board in tqdm(board_ids): + board_data = monday.boards.fetch_boards_by_id(board) + columns = board_data["data"]["boards"][0]["columns"] + col_id_map = {col["title"].lower(): col["id"] for col in columns} + reversed_col_id_map = {v: k for k, v in col_id_map.items()} + + + items = get_all_items(board, monday) + + all_records = [] + for row in tqdm(items): + data = {} + data.update({"address": row['name']}) + data.update({"client": row['group']['title']}) + for col in row.get("column_values", []): + if col.get("id") in reversed_col_id_map: + if col.get("type") == "file": + value = col.get("value") + no_of_files = 0 + + if value: + value = json.loads(col["value"]) + no_of_files = len(value.get('files', [])) + data.update({reversed_col_id_map[col.get("id")]: no_of_files}) + elif "no show" in reversed_col_id_map[col.get("id")]: + def extract_number_from_text(text): + number_str = '' + + for char in text: + if char.isnumeric(): + number_str += char + elif number_str: + break # stop once a number sequence ends + + return int(number_str) if number_str else None + text = col.get("text") + if text is None: + data.update({ + reversed_col_id_map[col.get("id")]: col.get("text") + }) + else: + data.update({ + reversed_col_id_map[col.get("id")]: extract_number_from_text(text) + }) + else: + data.update({ + reversed_col_id_map[col.get("id")]: col.get("text") + }) + all_records.append(data) + +# Convert to DataFrame +df = pd.DataFrame(all_records) + +filtered_dfs = [] + +def get_df(df, column_name, success_critera, job_name=None): + _ = df[ + df[column_name].str.lower().isin(success_critera) + ].copy() + if job_name: + _["job_type"] = job_name + return _ + + +# RA +ra = get_df(df, "ra", ["completed rdsap 9.9", "completed rdsap 10"], "RA") +filtered_dfs.append(ra) + + +# ATT +att = get_df(df, "pre-att", ["completed"], "ATT") +filtered_dfs.append(att) + +# V1 Coordination +v1 = get_df(df, "osmosis rc status".lower(), [ + "rc completed", +], "Coordination Stage 1 v1") +filtered_dfs.append(v1) + +# V2 Coordination +v2 = get_df(df, "v2 ioe mtp", ["completed"], "Coordination Stage 1 v2 remodel") +filtered_dfs.append(v2) + +# V3 Coordination +v3 = get_df(df, "v3 rc status", ["rc completed"], "Coordination Stage 1 v3 remodel") +filtered_dfs.append(v3) + +# Coordination stage 2 Please complete +cors2 = df[ + df["rc stage 2"].str.lower().isin(["to invoice"]) +] +cors2["joby_type"] = "Coordination Stage 2" +filtered_dfs.append(cors2) + +# Design stage Archetype +design1 = get_df(df, "design invoice status", ["to invoice"]) +design1 = get_df(design1, "design type for invoicing", ["archetype"], "Design Archetype") +filtered_dfs.append(design1) + +# Design stage Repetitive +design1 = get_df(df, "design invoice status", ["to invoice"]) +design1 = get_df(design1, "design type for invoicing", ["repetitive"], "Design Repetitive") +filtered_dfs.append(design1) + +# Design revision +design2 = get_df(df, "design revision invoice status", [ + "to invoice" +], "Design Revision") +filtered_dfs.append(design2) + +# Lodgement Phase 1 +lodg1 = get_df(df, "TM Phase 1 Invoicing Status".lower(), ["done", "to invoice"], "Lodgement phase 1") +filtered_dfs.append(lodg1) + +# Full Lodgement Phase +full_lodgement = get_df(df, "Jun-te TM Phase 2 Invoicing Status".lower(), ["to invoice"], "Full lodgement phase 2") +filtered_dfs.append(full_lodgement) + +# POST EPC +post_epc = get_df(df, "post-epc status", ["complete & uploaded"], "POST EPC") +filtered_dfs.append(post_epc) + + +# POST EPR +post_epr = df[ + df["post-epc status"].str.lower().isin(["post epr completed"]) +].copy() +post_epr["job_type"] = "POST epr" +filtered_dfs.append(post_epr) + +# Post ATT +post_att = get_df(df, "post att invoicing status ", ["to invoice"], "POST ATT") +filtered_dfs.append(post_att) + + +# Retrofit Evaluation +retro = get_df(df, "retrofit evaluation", ["completed & uploaded"], "Retrofit Evaluation") +filtered_dfs.append(retro) + +# RA NO Show +ra_ns = df[ + (df["ra no show evidence"].fillna(-9999) != df["ra no show invoice"].fillna(-9999)) & + (df["ra no show evidence"] != 0) +].copy() +ra_ns["job_type"] = "RA NO SHOW" +filtered_dfs.append(ra_ns) + + +# ATT NO Show +att_ns = df[ + (df["att no show evidence"].fillna(-9999) != df["att no show invoice"].fillna(-9999)) & + (df["att no show evidence"] != 0 ) +].copy() +att_ns["job_type"] = "ATT NO SHOW" +filtered_dfs.append(att_ns) + + +# Post visit no show +epc_ns = df[ + (df["epc no show evidence"].fillna(-9999) != df["epc no show invoice"].fillna(-9999)) & + (df["epc no show evidence"] != 0 ) +].copy() +epc_ns["job_type"] = "post EPC NO SHOW" +filtered_dfs.append(epc_ns) + +final_df = pd.concat(filtered_dfs).reset_index(drop=True) + +final_df["job_type"] = final_df["job_type"].str.lower() +rate_card_df_net_zero["job_type"] = rate_card_df_net_zero["job_type"].str.lower() +rate_card_df_sp_plus["job_type"] = rate_card_df_sp_plus["job_type"].str.lower() +rate_card_df_error_message["job_type"] = rate_card_df_error_message["job_type"].str.lower() + +# Now perform the merge +net_zero_df = final_df[final_df['client'].str.contains('shdf net zero'.lower(), case=False, na=False)] +sp_plus_df = final_df[final_df['client'].str.contains('SHDF 2.0 SP+'.lower(), case=False, na=False)] +other_df = final_df[~final_df.index.isin(net_zero_df.index) & ~final_df.index.isin(sp_plus_df.index)] + +combined_with_rates_net_zero_df = net_zero_df.merge(rate_card_df_net_zero, on="job_type", how="left") +combined_with_rates_sp_plus = sp_plus_df.merge(rate_card_df_sp_plus, on="job_type", how="left") +combined_with_rates_other_from_home_group = other_df.merge(rate_card_df_error_message, on="job_type", how="left") +import datetime +timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M') + +attribute = ['address', 'client', 'job_type', 'rate'] +combined_with_rates_sp_plus[attribute].to_excel(f'HomeGroup Wave 2SP+_{timestamp}.xlsx', index=False) +combined_with_rates_net_zero_df[attribute].to_excel(f'HomeGroup Wave NetZero_{timestamp}.xlsx', index=False) +combined_with_rates_other_from_home_group[attribute].to_excel(f'HomeGroup Wave Unsure_who_to_bill_{timestamp}.xlsx', index=False) + + +# TO DO check everything in excel +# make logic for seperation \ No newline at end of file diff --git a/etl/month_end_automation_wave_2_no_8.py b/etl/month_end_automation_wave_2_no_8.py new file mode 100644 index 0000000..0aff6fc --- /dev/null +++ b/etl/month_end_automation_wave_2_no_8.py @@ -0,0 +1,218 @@ +# Wave 2's month end automation +from tqdm import tqdm +from monday import MondayClient +from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset_ids +from pprint import pprint +import pandas as pd +import json + +monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY" +monday = MondayClient(monday_key) +# NCHA SHDF 2.1 SBS +board_ids = ["8668578700"] + +rate_card_data = { + "job_type": [ + "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", + "Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", + "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", + "RA no show", "ATT no show", "post EPC no show", "Design Revision" + ], + "rate": [ + 259, 125, 280, 125, 125, + 650, 415, 195, 175, 135, + 120, "60 - Double check with Kevin/Marianne EPR", 85, 125, 60, + 45, 45, 45, "Design Revision check with Kevin/Marianne" + ] +} + +rate_card_df = pd.DataFrame(rate_card_data) + + +for board in tqdm(board_ids): + board_data = monday.boards.fetch_boards_by_id(board) + columns = board_data["data"]["boards"][0]["columns"] + col_id_map = {col["title"].lower(): col["id"] for col in columns} + reversed_col_id_map = {v: k for k, v in col_id_map.items()} + + + items = get_all_items(board, monday) + + all_records = [] + for row in tqdm(items): + data = {} + data.update({"address": row['name']}) + data.update({"client": row['group']['title']}) + for col in row.get("column_values", []): + if col.get("id") in reversed_col_id_map: + if col.get("type") == "file": + value = col.get("value") + no_of_files = 0 + + if value: + value = json.loads(col["value"]) + no_of_files = len(value.get('files', [])) + data.update({reversed_col_id_map[col.get("id")]: no_of_files}) + elif "no show" in reversed_col_id_map[col.get("id")]: + def extract_number_from_text(text): + number_str = '' + + for char in text: + if char.isnumeric(): + number_str += char + elif number_str: + break # stop once a number sequence ends + + return int(number_str) if number_str else None + text = col.get("text") + if text is None: + data.update({ + reversed_col_id_map[col.get("id")]: col.get("text") + }) + else: + data.update({ + reversed_col_id_map[col.get("id")]: extract_number_from_text(text) + }) + else: + data.update({ + reversed_col_id_map[col.get("id")]: col.get("text") + }) + all_records.append(data) + +# Convert to DataFrame +df = pd.DataFrame(all_records) + +filtered_dfs = [] + +def get_df(df, column_name, success_critera, job_name=None): + _ = df[ + df[column_name].str.lower().isin(success_critera) + ].copy() + if job_name: + _["job_type"] = job_name + return _ + + +# RA +ra = get_df(df, "ra", ["completed rdsap 9.9", "completed rdsap 10"], "RA") +filtered_dfs.append(ra) + + +# ATT +att = get_df(df, "att", ["completed"], "ATT") +filtered_dfs.append(att) + +# V1 Coordination +v1 = get_df(df, "v1 coordination status".lower(), [ + "rc complete", +], "Coordination Stage 1 v1") +filtered_dfs.append(v1) + +# V2 Coordination +v2 = get_df(df, "v2 coordination status", ["rc v2 complete", "uploaded"], "Coordination Stage 1 v2 remodel") +filtered_dfs.append(v2) + +# # V3 Coordination +# v3 = get_df(df, "") +# # filtered_dfs.append(v3) + +# Corodination stage 2 +cors2 = get_df(df, "rc stg. 2", ["to invoice"], "Coordination Stage 2") +filtered_dfs.append(cors2) + +# Design Archtype Complex +design1 = get_df(df, "rd invoiced", ["to invoice"]) +design1 = get_df(design1, "design type", ["archetype (complex)"]) +design1 = get_df(design1, "design upload to sharepoint", ["done"], "Design Archetype Complex") +filtered_dfs.append(design1) + +# Design Archtype Simple +design1 = get_df(df, "rd invoiced", ["to invoice"]) +design1 = get_df(design1, "design type", ["archetype (simple)"]) +design1 = get_df(design1, "design upload to sharepoint", ["done"], "Design Archetype Simple") +filtered_dfs.append(design1) + +# Design Repitive Simple +design1 = get_df(df, "rd invoiced", ["to invoice"]) +design1 = get_df(design1, "design type", ["Design Repetitive Simple"]) +design1 = get_df(design1, "design upload to sharepoint", ["done"], "Design Repetitive Simple") +filtered_dfs.append(design1) + +# Design revision +design2 = get_df(df, "design revision invoice status", [ + "to invoice" +], "Design Revision") +filtered_dfs.append(design2) + +# Lodgement Phase 1 +lodg1 = get_df(df, "Lodg. Phase 1 Invoice Status".lower(), ["to be invoiced"], "Lodgement Phase 1") +filtered_dfs.append(lodg1) + +# Lodgement Phase 1 +lodg1 = get_df(df, "phase 1 to be invoiced".lower(), ["phase 1 to be invoiced"], "Lodgement Phase 1") +filtered_dfs.append(lodg1) + +# Full Lodgement Phase +full_lodgement = get_df(df, "full lodgement invoice status".lower(), ["to be invoice"], "Full Lodgement phase 2") +filtered_dfs.append(full_lodgement) + +# POST EPC +post_epc = get_df(df, "post epc status", ["uploaded"], "POST EPC") +filtered_dfs.append(post_epc) + + +# POST EPR +post_epr = df[ + df["post epc status"].str.lower().isin(["post epr completed"]) +].copy() +post_epr["job_type"] = "POST epr" +filtered_dfs.append(post_epr) + +# Post ATT +post_att = get_df(df, "post att", ["post att uploaded"], "POST ATT") +filtered_dfs.append(post_att) + + +# Retrofit Evaluation +retro = get_df(df, "retrofit evaluation", ["done"], "Retrofit Evaluation") +filtered_dfs.append(retro) + +# RA NO Show +ra_ns = df[ + (df["ra no show evidence"].fillna(-9999) != df["ra no show invoice"].fillna(-9999)) & + (df["ra no show evidence"] != 0 ) +].copy() +ra_ns["job_type"] = "RA NO SHOW" +filtered_dfs.append(ra_ns) + + +# ATT NO Show +att_ns = df[ + (df["att no show evidence"].fillna(-9999) != df["att no show invoice"].fillna(-9999)) & + (df["att no show evidence"] !=0 ) +].copy() +att_ns["job_type"] = "ATT NO SHOW" +filtered_dfs.append(att_ns) + + +# Post visit no show +epc_ns = df[ + df["post works no show evidence"].fillna(-9999) != df["post works no show invoice"].fillna(-9999) +].copy() +epc_ns["job_type"] = "post EPC NO SHOW" +filtered_dfs.append(epc_ns) + +final_df = pd.concat(filtered_dfs).reset_index(drop=True) + +final_df[['address', 'client', 'job_type']] + +final_df["job_type"] = final_df["job_type"].str.lower() +rate_card_df["job_type"] = rate_card_df["job_type"].str.lower() + +# Now perform the merge +combined_with_rates = final_df.merge(rate_card_df, on="job_type", how="left") +import datetime +timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M') + +attribute = ['address', 'client', 'job_type', 'rate'] +combined_with_rates[attribute].to_excel(f'NCHA SHDF 2.1 SBS_{timestamp}.xlsx', index=False) diff --git a/etl/month_end_automation_wave_2_no_9.py b/etl/month_end_automation_wave_2_no_9.py new file mode 100644 index 0000000..15bf2fb --- /dev/null +++ b/etl/month_end_automation_wave_2_no_9.py @@ -0,0 +1,207 @@ +# Wave 2's month end automation +from tqdm import tqdm +from monday import MondayClient +from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset_ids +from pprint import pprint +import pandas as pd +import json + +monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY" +monday = MondayClient(monday_key) +# NCHA Almshouses +board_ids = ["5423364294"] + +rate_card_data = { + "job_type": [ + "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", + "Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", + "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", + "RA no show", "ATT no show", "post EPC no show" + ], + "rate": [ + 259, 125, 280, 125, 125, + 650, 415, 195, 175, 135, + 120, "Post EPR Please Marianne", 85, 125, 60, + 25, 25, 25 + ] +} + +rate_card_df = pd.DataFrame(rate_card_data) + + +for board in tqdm(board_ids): + board_data = monday.boards.fetch_boards_by_id(board) + columns = board_data["data"]["boards"][0]["columns"] + col_id_map = {col["title"].lower(): col["id"] for col in columns} + reversed_col_id_map = {v: k for k, v in col_id_map.items()} + + + items = get_all_items(board, monday) + + all_records = [] + for row in tqdm(items): + data = {} + data.update({"address": row['name']}) + data.update({"client": row['group']['title']}) + for col in row.get("column_values", []): + if col.get("id") in reversed_col_id_map: + if col.get("type") == "file": + value = col.get("value") + no_of_files = 0 + + if value: + value = json.loads(col["value"]) + no_of_files = len(value.get('files', [])) + data.update({reversed_col_id_map[col.get("id")]: no_of_files}) + elif "no show" in reversed_col_id_map[col.get("id")]: + def extract_number_from_text(text): + number_str = '' + + for char in text: + if char.isnumeric(): + number_str += char + elif number_str: + break # stop once a number sequence ends + + return int(number_str) if number_str else None + text = col.get("text") + if text is None: + data.update({ + reversed_col_id_map[col.get("id")]: col.get("text") + }) + else: + data.update({ + reversed_col_id_map[col.get("id")]: extract_number_from_text(text) + }) + else: + data.update({ + reversed_col_id_map[col.get("id")]: col.get("text") + }) + all_records.append(data) + +# Convert to DataFrame +df = pd.DataFrame(all_records) + +filtered_dfs = [] + +def get_df(df, column_name, success_critera, job_name): + _ = df[ + df[column_name].str.lower().isin(success_critera) + ].copy() + _["job_type"] = job_name + return _ + + +# RA +ra = get_df(df, "ra", ["completed rdsap 9.9", "completed rdsap 10"], "RA") +filtered_dfs.append(ra) + + +# ATT +att = get_df(df, "att", ["completed"], "ATT") +filtered_dfs.append(att) + +# V1 Coordination +v1 = get_df(df, "coordination status (mtp)".lower(), [ + "rc complete", +], "Coordination Stage 1 v1") +filtered_dfs.append(v1) + +# V2 Coordination +# v2 = get_df(df, "v2 coordination status", ["rc v2 complete", "uploaded"], "Coordination Stage 1 v2 remodel") +# filtered_dfs.append(v2) + +# # V3 Coordination +# Coordination Stage 1 v3 remode +# v3 = get_df(df, "") +# # filtered_dfs.append(v3) + +# Coordination stage 2 Please complete +cors2 = df[ + df["rc stage 2"].str.lower().isin(["to invoice"]) +].copy() +cors2["joby_type"] = "Coordination Stage 2" +filtered_dfs.append(cors2) + +# Design stage 1 +# design1 = get_df(df, "rd invoiced", ["to invoice"], "Design") +# filtered_dfs.append(design1) + +# Design revision +# design2 = get_df(df, "design revision invoice", [ +# "Rev. A to invoice".lower(), +# "Rev. B to invoice".lower(), +# "Rev. C to invoice".lower(), +# "Rev. D to invoice".lower(), +# ], "Design Revision") +# filtered_dfs.append(design2) + +# Lodgement Phase 1 +# lodg1 = get_df(df, "Lodg. Phase 1 Invoice Status".lower(), ["to be invoiced"], "Lodgement Phase 1") +# filtered_dfs.append(lodg1) + + +# Full Lodgement Phase +full_lodgement = get_df(df, "trustmark lodgement".lower(), ["done"], "Full lodgement phase 2") +filtered_dfs.append(full_lodgement) + +# POST EPC +# post_epc = get_df(df, "post epc status", ["uploaded"], "POST EPC") +# filtered_dfs.append(post_epc) + + +# # POST EPR +# post_epr = df[ +# df["post-epc status"].str.lower().isin(["post epr completed"]) +# ].copy() +# post_epr["job_type"] = "POST epr" +# filtered_dfs.append(post_epr) + +# Post ATT +# post_att = get_df(df, "post att", ["post att uploaded"], "POST ATT") +# filtered_dfs.append(post_att) + + +# Retrofit Evaluation +retro = get_df(df, "retrofit evaluation", ["done"], "Retrofit Evaluation") +filtered_dfs.append(retro) + +# RA NO Show +ra_ns = df[ + (df["ra no show evidence"].fillna(-9999) != df["ra no show invoice"].fillna(-9999)) & + (df["ra no show evidence"] != 0) +].copy() +ra_ns["job_type"] = "RA NO SHOW" +filtered_dfs.append(ra_ns) + + +# ATT NO Show +att_ns = df[ + (df["att no show evidence"].fillna(-9999) != df["att no show invoice"].fillna(-9999)) & + (df["att no show evidence"] != 0) +].copy() +att_ns["job_type"] = "ATT NO SHOW" +filtered_dfs.append(att_ns) + + +# Post visit no show +# epc_ns = df[ +# df["post epc no show evidence"].fillna(-9999) != df["post epc no show invoice"].fillna(-9999) +# ].copy() +# epc_ns["job_type"] = "post EPC no show" +# filtered_dfs.append(epc_ns) + +final_df = pd.concat(filtered_dfs).reset_index(drop=True) + +final_df[['address', 'client', 'job_type']] + +final_df["job_type"] = final_df["job_type"].str.lower() +rate_card_df["job_type"] = rate_card_df["job_type"].str.lower() + +# Now perform the merge +combined_with_rates = final_df.merge(rate_card_df, on="job_type", how="left") +import datetime +timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M') + +attribute = ['address', 'client', 'job_type', 'rate'] +combined_with_rates[attribute].to_excel(f'NCHA Almshouses_{timestamp}.xlsx', index=False) diff --git a/etl/month_end_automation_wave_3_layout.py b/etl/month_end_automation_wave_3_layout.py new file mode 100644 index 0000000..673ea66 --- /dev/null +++ b/etl/month_end_automation_wave_3_layout.py @@ -0,0 +1,267 @@ +# Wave 3's month end automation + +from tqdm import tqdm +from monday import MondayClient +from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset_ids +from pprint import pprint +import pandas as pd +import json + +monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY" +monday = MondayClient(monday_key) +board_ids = [ +# "9349630181", # WCHG Walkups-Operations + # "8829428746", # 2502 Accent Housing + # "8830772914", # "L&Q London" + # "9601691730", # Cardo Wales & West - Wave 3 + "9660895490", # Northumberland County SHDF Wave 3 +] + +empty = "Rate card info missing" +rate_card_data_example = { + "job_type": [ + "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", + "Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", + "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", + "RA no show", "ATT no show", "post EPC no show", "Full cost MTP", "measure modelling" + ], + "rate": [ + 259, 125, 280, 125, 125, + 650, 415, 195, 175, 135, + 120, "Post EPR Please Marianne", 85, 125, 60, + 25, 25, 25, "Mariann please input full cost mtp", "Marianne please input measure modelling" + ] +} + + +rate_card_data_2502_accent_housing = { + "job_type": [ + "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", + "Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", + "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", + "RA no show", "ATT no show", "post EPC no show", "Full cost MTP", "measure modelling" + ], + "rate": [ + empty, empty, empty, empty, empty, + empty, empty, empty, empty, empty, + empty, empty, empty, empty, empty, + empty, empty, empty, 280, 150 + ] +} + +rate_card_data_l_and_q_london = { + "job_type": [ + "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", + "Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", + "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", + "RA no show", "ATT no show", "post EPC no show", "Full cost MTP", "measure modelling" + ], + "rate": [ + empty, empty, empty, empty, empty, + empty, empty, empty, empty, empty, + empty, empty, empty, empty, empty, + empty, empty, empty, 280, 150 + ] +} + + +rate_card_data_northhumberland_country_shdf_wave_3 = { + "job_type": [ + "RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel", + "Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2", + "Post EPR", "Post EPC", "Post ATT", "retrofit evaluation", + "RA no show", "ATT no show", "post EPC no show", "Full cost MTP", "measure modelling" + ], + "rate": [ + empty, empty, empty, empty, empty, + empty, empty, empty, empty, empty, + empty, empty, empty, empty, empty, + empty, empty, empty, 280, 150 + ] +} + + + +# rate_card_df = pd.DataFrame(rate_card_data_example) +# rate_card_df = pd.DataFrame(rate_card_data_2502_accent_housing) +# rate_card_df = pd.DataFrame(rate_card_data_l_and_q_london) +rate_card = pd.DataFrame(rate_card_data_northhumberland_country_shdf_wave_3) + + +for board in tqdm(board_ids): + print(f"working on board {board}") + board_data = monday.boards.fetch_boards_by_id(board) + columns = board_data["data"]["boards"][0]["columns"] + col_id_map = {col["title"].lower(): col["id"] for col in columns} + reversed_col_id_map = {v: k for k, v in col_id_map.items()} + + + items = get_all_items(board, monday) + + all_records = [] + for row in tqdm(items): + data = {} + data.update({"address": row['name']}) + data.update({"client": row['group']['title']}) + for col in row.get("column_values", []): + if col.get("id") in reversed_col_id_map: + if col.get("type") == "file": + value = col.get("value") + no_of_files = 0 + + if value: + value = json.loads(col["value"]) + no_of_files = len(value.get('files', [])) + data.update({reversed_col_id_map[col.get("id")]: no_of_files}) + else: + data.update({ + reversed_col_id_map[col.get("id")]: col.get("text") + }) + all_records.append(data) + +# Convert to DataFrame +df = pd.DataFrame(all_records) + +filtered_dfs = [] + + +def get_df(df, column_name, success_critera, job_name=None): + _ = pd.DataFrame() + if column_name in col_id_map: + _ = df[ + df[column_name].str.lower().isin(success_critera) + ].copy() + if job_name: + _["job_type"] = job_name + + + return _ + + +# RA +ra = get_df(df, "ra invoicing status", ["to invoice"], "RA") +if not ra.empty: + filtered_dfs.append(ra) + + +att = get_df(df, "post att invoicing status", ["to invoice"], "ATT") +if not att.empty: + filtered_dfs.append(att) + +modeling = get_df(df, "mtp invoicing status", ["modelling to invoice"], "Measure Modelling") +if not modeling.empty: + filtered_dfs.append(modeling) + +try: + # Only needed for one board in wave 3 + full_cost = get_df(df, "mtp invoicing status", ["(V1) Full cost MTP to invoice (no previous modelling)".lower()], "full cost mtp") + if not full_cost.empty: + filtered_dfs(full_cost) +except Exception as e: + print(e) + +v1 = get_df(df, "mtp invoicing status", ["(v1) ioe/mtp to invoice"], "Coordination Stage 1 v1") +if not v1.empty: + filtered_dfs.append(v1) + +v2 = get_df(df, "mtp invoicing status", ["(v2) ioe/mtp to invoice"], "Coordination Stage 1 v2 remodel") +if not v2.empty: + filtered_dfs.append(v2) + +v3 = get_df(df, "mtp invoicing status", ["(v3) ioe/mtp to invoice"], "Coordination Stage 1 v3 remodel") +if not v3.empty: + filtered_dfs.append(v3) + +# Coordination stage 2 Please complete +cors2 = get_df(df, "rc stage 2", ["to invoice"], "Coordination Stage 2") +if not cors2.empty: + filtered_dfs.append(cors2) + +# Design archetype complex +design = get_df(df, "design invoicing status", ["to invoice"]) +design1 = get_df(design, "design invoice type", ["archetype (complex)"], "Design Archetype Complex") +if not design1.empty : + filtered_dfs.append(design1) + +# Design archetype simple +design1 = get_df(design, "design invoice type", ["archetype (simple)"], "Design Archetype Simple") +if not design1.empty: + filtered_dfs.append(design1) + +# Design repetitive simple +design1 = get_df(design, "design invoice type", ["archetype (simple)"], "Design Archetype repetitive") +if not design1.empty: + filtered_dfs.append(design1) + +# Design repetitive complex +design1 = get_df(design, "design invoice type", ["archetype (complex)"], "Design Archetype complex") +if not design1.empty: + filtered_dfs.append(design1) + +# Design Revision +revision_letter = ['a', 'b', 'c', 'd'] +for letter in revision_letter: + design = get_df(df, "design revision invoice", [f"rev. {letter} to invoice"], "Design Revision") + if not design.empty: + filtered_dfs.append(design) + +# Lodgement Phase 1 +lodg1 = get_df(df, "lodgement phase 1 invoicing status", ["to invoice"], "Lodgement Phase 1") +if not lodg1.empty: + filtered_dfs(lodg1) + +# Full Lodgement Phase +lodg2 = get_df(df, "full lodgement invoicing status", ["to invoice"], "Full lodgement phase 2") +if not lodg2.empty: + filtered_dfs.append(lodg2) + +# POST EPC +post_epc = get_df(df, "post epc & eval. invoicing status", ["epc to invoice"], "POST EPC") +if not post_epc.empty: + filtered_dfs.append(post_epc) + + +# POST EPR +post_epr = get_df(df, "post epc & eval. invoicing status", ["epr to invoice"], "POST EPR") +if not post_epr.empty: + filtered_dfs.append(post_epr) + +# post att +post_att = get_df(df, "post att invoicing status", ["to invoice"], "POST ATT") +if not post_att.empty: + filtered_dfs.append(post_epc) + +# Retrofit Evaluation +rc = get_df(df, "rc stage 2 invoicing status", ["to invoice"], "retrofit evaluation") +if not rc.empty: + filtered_dfs.append(rc) + +# RA NO Show +ra_ns = get_df(df,"ra no show invoice", ["to invoice","to invoice (+1 previous no show)", "to invoice (+2 previous no shows)"], "RA NO SHOW") +if not ra_ns.empty: + filtered_dfs.append(ra_ns) + + +# ATT NO Show +att_ns = get_df(df, "pre att no show invoice", ["to invoice","to invoice (+1 previous no show)", "to invoice (+2 previous no shows)"], "ATT NO SHOW") +if not att_ns.empty: + filtered_dfs.append(att_ns) + + +# Post visit no show +epc_ns = get_df(df, "post works no show invoice", ["to invoice","to invoice (+1 previous no show)", "to invoice (+2 previous no shows)"], "post EPC NO SHOW") +if not epc_ns.empty: + filtered_dfs.append(epc_ns) + +final_df = pd.concat(filtered_dfs).reset_index(drop=True) + +final_df["job_type"] = final_df["job_type"].str.lower() +rate_card_df["job_type"] = rate_card_df["job_type"].str.lower() + +# Now perform the merge +combined_with_rates = final_df.merge(rate_card_df, on="job_type", how="left") +import datetime +timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M') + +attribute = ['address', 'client', 'job_type', 'rate'] +combined_with_rates[attribute].to_excel(f'L&Q London {timestamp}.xlsx', index=False) diff --git a/etl/osmosis_complaince_address_to_files.py b/etl/osmosis_complaince_address_to_files.py index 0ab6ed9..c48e4eb 100644 --- a/etl/osmosis_complaince_address_to_files.py +++ b/etl/osmosis_complaince_address_to_files.py @@ -91,7 +91,7 @@ def get_all_items(board_id, monday): limit = 25 # Adjust the limit based on how many items you want per request all_items = [] # List to store all fetched items cursor = None # Start without a cursor for the first page - + print(f"Connecting to Monday API and retrieving data for board {board_id}") # Loop through pages while True: # Fetch items for the current page @@ -116,8 +116,7 @@ def get_all_items(board_id, monday): # If there's no cursor, we've reached the last page if not cursor: break - print(f"cursor {cursor}") - print(f"len all_itemms {len(all_items)}") + print("Loading...") return all_items def upload_to_sharepoint(to_upload, master_folder_name): @@ -128,36 +127,37 @@ def upload_to_sharepoint(to_upload, master_folder_name): print(f"Uploading {file_name} to sharepoint") osmosis.upload_file(file_path, parent_folder + f"/{master_folder_name}", file_name) -# Step 1: Fetch column IDs -board_data = monday.boards.fetch_boards_by_id(board_id) -columns = board_data["data"]["boards"][0]["columns"] -col_id_map = {col["title"].lower(): col["id"] for col in columns} +if __name__ == "__main__": + # Step 1: Fetch column IDs + board_data = monday.boards.fetch_boards_by_id(board_id) + columns = board_data["data"]["boards"][0]["columns"] + col_id_map = {col["title"].lower(): col["id"] for col in columns} -name_id = col_id_map.get("name") # Replace with actual title if different -files_id = col_id_map.get("file(s)") # Replace with actual title if different + name_id = col_id_map.get("name") # Replace with actual title if different + files_id = col_id_map.get("file(s)") # Replace with actual title if different -if not name_id or not files_id: - raise Exception("Could not find 'name' or 'file(s)' columns") + if not name_id or not files_id: + raise Exception("Could not find 'name' or 'file(s)' columns") -items = get_all_items(board_id, monday) -for i,item in enumerate(tqdm(items)): - if i>329: - item_name = item["name"] - item_name = sanitize_name(item_name, ignore_dot=True) - print(f"Item name is {item_name}") - asset_ids = extract_asset_ids(item, files_id) + items = get_all_items(board_id, monday) + for i,item in enumerate(tqdm(items)): + if i>329: + item_name = item["name"] + item_name = sanitize_name(item_name, ignore_dot=True) + print(f"Item name is {item_name}") + asset_ids = extract_asset_ids(item, files_id) - to_upload = [] - for asset_id in asset_ids: - try: - public_url, file_name = get_public_url(asset_id) - print(f"Downloading {file_name}") - file_path = download_file_from_public_url(public_url, file_name) - to_upload.append(file_path) - except Exception as e: - print(f"Failed to download/upload asset {asset_id}: {e}") + to_upload = [] + for asset_id in asset_ids: + try: + public_url, file_name = get_public_url(asset_id) + print(f"Downloading {file_name}") + file_path = download_file_from_public_url(public_url, file_name) + to_upload.append(file_path) + except Exception as e: + print(f"Failed to download/upload asset {asset_id}: {e}") - if to_upload: - upload_to_sharepoint(to_upload, item_name) + if to_upload: + upload_to_sharepoint(to_upload, item_name) -# Liv green # Cocuun # Wates \ No newline at end of file + # Liv green # Cocuun # Wates \ No newline at end of file diff --git a/etl/osmosis_data/asset_list.xlsx b/etl/osmosis_data/asset_list.xlsx index ceada9c..822b436 100644 Binary files a/etl/osmosis_data/asset_list.xlsx and b/etl/osmosis_data/asset_list.xlsx differ diff --git a/etl/osmosis_monday_to_sharepoint_automation.py b/etl/osmosis_monday_to_sharepoint_automation.py index 4e83612..40118f4 100644 --- a/etl/osmosis_monday_to_sharepoint_automation.py +++ b/etl/osmosis_monday_to_sharepoint_automation.py @@ -8,57 +8,81 @@ from etl.scraper.scraper import SharePointInstaller from etl.scraper.scraper import SharePointScraper import pandas as pd from tqdm import tqdm +import time -osmosis = SharePointScraper(SharePointInstaller.OSMOSIS_WAVE_2) +osmosis = SharePointScraper(SharePointInstaller.OSMOSIS_WAVE_3) -parent_folder = "/Osmosis ACD/Osmosis ACD Projects/WCHG/WCHG Walkups/Property Folders" +parent_folder = "/Osmosis-ACD Projects/Cardo/Cardo (Wales & West)/2506 Cardo Property Folders" -asset_list = pd.read_excel("osmosis_data/asset_list.xlsx", sheet_name="Sheet1") +asset_list = pd.read_excel("osmosis_data/asset_list.xlsx", sheet_name="Sheet 1") new_asset_list = [] # Create asset list and location for index, address in tqdm(asset_list.iterrows()): - folder_name = address['Name'] + " " + address['Postcode'] - webUrl = osmosis.create_dir(folder_name, parent_folder) - - first_folder = "1. Retrofit Assessment" - osmosis.create_dir(first_folder, parent_folder + f"/{folder_name}") - osmosis.create_dir("A. Assessment", parent_folder + f"/{folder_name}/{first_folder}") - osmosis.create_dir("B. Air Tightness Tests", parent_folder + f"/{folder_name}/{first_folder}") + if index > 39: + folder_name = address['Name'] + " " + address['Postcode'] + webUrl = osmosis.create_dir(folder_name, parent_folder) + time.sleep(1) + print(f"building folders insidea {folder_name}") - second_folder = "2. RC Mid-Term Plan" - osmosis.create_dir(second_folder, parent_folder + f"/{folder_name}") - osmosis.create_dir("SAP", parent_folder + f"/{folder_name}/{second_folder}") + print("building retrofit assessment") + first_folder = "1. Retrofit Assessment" + osmosis.create_dir(first_folder, parent_folder + f"/{folder_name}") + osmosis.create_dir("A. Assessment", parent_folder + f"/{folder_name}/{first_folder}") + osmosis.create_dir("B. Air Tightness Tests", parent_folder + f"/{folder_name}/{first_folder}") - third_folder = "3. Retrofit Design" - osmosis.create_dir(third_folder, parent_folder + f"/{folder_name}") + print("building RC MID Term plan") + second_folder = "2. RC Mid-Term Plan" + osmosis.create_dir(second_folder, parent_folder + f"/{folder_name}") + osmosis.create_dir("SAP", parent_folder + f"/{folder_name}/{second_folder}") - fourth_folder = "4. Post EPC" - osmosis.create_dir(fourth_folder, parent_folder + f"/{folder_name}") - osmosis.create_dir(f"{address['Name']} - POST EPC Photos", parent_folder + f"/{folder_name}/{fourth_folder}") + print("building Retrofit Design") + third_folder = "3. Retrofit Design" + osmosis.create_dir(third_folder, parent_folder + f"/{folder_name}") - fifth_folder = "5. Trustmark Lodgement" - osmosis.create_dir(fifth_folder, parent_folder + f"/{folder_name}") - osmosis.create_dir("1. Works", parent_folder + f"/{folder_name}/{fifth_folder}") + print("building post epc") + fourth_folder = "4. Post EPC" + osmosis.create_dir(fourth_folder, parent_folder + f"/{folder_name}") + osmosis.create_dir(f"{address['Name']} - POST EPC Photos", parent_folder + f"/{folder_name}/{fourth_folder}") - osmosis.create_dir("2. Required Documents", parent_folder + f"/{folder_name}/{fifth_folder}") - osmosis.create_dir("3. Additional Documents", parent_folder + f"/{folder_name}/{fifth_folder}") - - asset_data = { - "Name": address['Name'], - "Postcode": address['Postcode'], - "Sharepoint": webUrl, - } + print("Building Trust mark Lodgement") + fifth_folder = "5. Trustmark Lodgement" + osmosis.create_dir(fifth_folder, parent_folder + f"/{folder_name}") + osmosis.create_dir("1. Works", parent_folder + f"/{folder_name}/{fifth_folder}") - new_asset_list.append(asset_data) + osmosis.create_dir("2. Required Documents", parent_folder + f"/{folder_name}/{fifth_folder}") + osmosis.create_dir("3. Additional Documents", parent_folder + f"/{folder_name}/{fifth_folder}") + + asset_data = { + "Name": address['Name'], + "Postcode": address['Postcode'], + "Sharepoint": webUrl, + } + print(asset_data) + + new_asset_list.append(asset_data) -# Osmosist File strucutre +# Run this is you just want to get url +def just_url(asset_list): + new_asset_list = [] + for index, address in tqdm(asset_list.iterrows()): + folder_name = address['Name'] + " " + address['Postcode'] + webUrl = osmosis.create_dir(folder_name, parent_folder) + asset_data = { + "Name": address['Name'], + "Postcode": address['Postcode'], + "Sharepoint": webUrl, + } + print(asset_data) + new_asset_list.append(asset_data) + return new_asset_list +new_asset_list = just_url(asset_list=asset_list) df = pd.DataFrame(new_asset_list) df.to_csv("output.csv", index=False) diff --git a/etl/quick_script_to_pull_data_from_hubspot_and_ideally_push.py b/etl/quick_script_to_pull_data_from_hubspot_and_ideally_push.py new file mode 100644 index 0000000..fdb15fe --- /dev/null +++ b/etl/quick_script_to_pull_data_from_hubspot_and_ideally_push.py @@ -0,0 +1,53 @@ +import os +from pprint import pprint + +os.environ["SHAREPOINT_CLIENT_ID"] = "895e3b77-b1d7-43ec-b18f-dcfe07cdfeaf" +os.environ["SHAREPOINT_CLIENT_SECRET"] = "SOf8Q~-is4wdQiqvEEm9FlJQRAY9ELGaj5Qz-a6E" +os.environ["SHAREPOINT_TENANT_ID"] = "c3f7519c-2719-4547-af04-6da6cbfd8f8f" +os.environ["SOUTH_COAST_INSULATION_SERVICE_SHAREPOINT_ID"] = "b5a51507-9427-4ee0-b03e-90ec7681e2d3" +os.environ["JJC_SERVICE_SHAREPOINT_ID"] = "7fdd0485-bbf3-4b29-b30f-98c81c2a6284" + +from etl.hubSpotClient.hubspot import DealStage, HubSpotClient +from etl.surveyedData.surveryedData import surveyedDataProcessor +from etl.scraper.scraper import SharePointScraper, SharePointInstaller +from etl.utils.utils import get_sharepoint_path + +def string_to_installer(installer): + if installer.upper() == "J & J CRUMP": + return SharePointInstaller.JJC + elif installer.upper() == "SCIS": + return SharePointInstaller.SOUTH_COAST_INSULATION + elif installer.upper() == "SGEC": + return SharePointInstaller.JJC + else: + return None + +# Local development +os.environ["DATABASE_URL"] = "postgresql://postgres:makingwarmhomes@db:5432/postgres" + +hubspotClient = HubSpotClient() + +# Gets all deals and puts it into a SubmissionInfoFromDeal class +# KHALIM - I ADDED A SCRIPT TO ONLY DOWNLOAD 1 deal for speed sake +deals = hubspotClient.get_deals_from_deal_stage(DealStage.SURVEYED_COMPLETE_NEEDS_SIGN_OFF) + + +for deal in deals: + sharepoint_url = deal.submission_folder_path + installer = string_to_installer(deal.installer) + sp = SharePointScraper(installer) + path = get_sharepoint_path(sharepoint_url) + + files = sp.downloadt_files_from_path(path) + sdp = surveyedDataProcessor("fake address", files) + + # Class Object for EPR Summary Informaiton ( Transform ) + sdp.epr_summary_information + + # File path to epr + sdp.epr_summary_information_file_path + + break + + + diff --git a/etl/read_stuff_from_s3_example.py b/etl/read_stuff_from_s3_example.py new file mode 100644 index 0000000..6ffdbcc --- /dev/null +++ b/etl/read_stuff_from_s3_example.py @@ -0,0 +1,44 @@ +import boto3 +import os + + +def print_hello_from_etl_module(): + print("You are printing from a etl module we made in poetry") + +def split_s3_url(s3_url): + if not s3_url.startswith("s3://"): + raise ValueError("Invalid S3 URL. Must start with 's3://'") + + path = s3_url[5:] + parts = path.split('/', 1) + + if len(parts) != 2: + raise ValueError("S3 URL must include a key after the bucket name") + return parts[0], parts[1] + +def create_temp_file(content_bytes, relative_path): + # Save under /tmp/s3/ + full_path = os.path.join("/tmp/s3", relative_path) + + # Make sure the directory exists + os.makedirs(os.path.dirname(full_path), exist_ok=True) + + # Write content to file + with open(full_path, 'wb') as temp_file: + temp_file.write(content_bytes) + + print(f"Temporary file created at: {full_path}") + return full_path + +def download_data_from_s3(s3_uri): + s3 = boto3.resource('s3') + bucket_name, file_name = split_s3_url(s3_uri) + + obj = s3.Object(bucket_name, file_name) + data = obj.get()['Body'].read() + + # Save using full S3 key as relative path + return create_temp_file(data, file_name) + +# Example usage +# download_data_from_s3("s3://retrofit-energy-assessments-dev/JAFFERSONS ENERGY CONSULTANTS/VDE001/12103116/docs & plans/77 Perryn Road, W3 7LT EPR.pdf") diff --git a/etl/surveyedData/surveryedData.py b/etl/surveyedData/surveryedData.py index f75b112..a88b8da 100644 --- a/etl/surveyedData/surveryedData.py +++ b/etl/surveyedData/surveryedData.py @@ -43,6 +43,7 @@ class surveyedDataProcessor(): self.hubspot_deal_id = None self.epr_with_data = None self.epr_summary_information = None + self.epr_summary_information_file_path = None self.full_sap_xml = None self.lig_sap_xml = None self.rd_sap_xml = None @@ -69,6 +70,7 @@ class surveyedDataProcessor(): self.epr_with_data = pdf.get_reader() elif pdf.type == ReportType.ENERGY_PERFORMANCE_REPORT_SUMMARY_INFORMATION: self.epr_summary_information = pdf.get_reader() + self.epr_summary_information_file_path = file elif file.lower().endswith('.xml'): xml = xmlReader(file) diff --git a/poetry.lock b/poetry.lock index 62d14e9..6c58987 100644 --- a/poetry.lock +++ b/poetry.lock @@ -84,6 +84,46 @@ charset-normalizer = ["charset-normalizer"] html5lib = ["html5lib"] lxml = ["lxml"] +[[package]] +name = "boto3" +version = "1.39.6" +description = "The AWS SDK for Python" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "boto3-1.39.6-py3-none-any.whl", hash = "sha256:db965dc9019df7b1d20e8d8ab7a653956f275865175a8652419ebfd03de03d83"}, + {file = "boto3-1.39.6.tar.gz", hash = "sha256:e75bfcd444e199767642f28ef8dc4f972846dc3118e48a7e09f9c458dae2021e"}, +] + +[package.dependencies] +botocore = ">=1.39.6,<1.40.0" +jmespath = ">=0.7.1,<2.0.0" +s3transfer = ">=0.13.0,<0.14.0" + +[package.extras] +crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] + +[[package]] +name = "botocore" +version = "1.39.6" +description = "Low-level, data-driven core of boto 3." +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "botocore-1.39.6-py3-none-any.whl", hash = "sha256:9c002724e9b97cec610dbbb3bb019b3248ff6bf58407835621f0461e740af90b"}, + {file = "botocore-1.39.6.tar.gz", hash = "sha256:d3a6c207d233ddee3289c1d56646047bef18b21a1faebb3d83a6fca149fd0f59"}, +] + +[package.dependencies] +jmespath = ">=0.7.1,<2.0.0" +python-dateutil = ">=2.1,<3.0.0" +urllib3 = {version = ">=1.25.4,<2.2.0 || >2.2.0,<3", markers = "python_version >= \"3.10\""} + +[package.extras] +crt = ["awscrt (==0.23.8)"] + [[package]] name = "certifi" version = "2025.4.26" @@ -717,6 +757,18 @@ docs = ["Jinja2 (==2.11.3)", "MarkupSafe (==1.1.1)", "Pygments (==2.8.1)", "alab qa = ["flake8 (==5.0.4)", "mypy (==0.971)", "types-setuptools (==67.2.0.1)"] testing = ["Django", "attrs", "colorama", "docopt", "pytest (<9.0.0)"] +[[package]] +name = "jmespath" +version = "1.0.1" +description = "JSON Matching Expressions" +optional = false +python-versions = ">=3.7" +groups = ["main"] +files = [ + {file = "jmespath-1.0.1-py3-none-any.whl", hash = "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980"}, + {file = "jmespath-1.0.1.tar.gz", hash = "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe"}, +] + [[package]] name = "jupyter-client" version = "8.6.3" @@ -1191,23 +1243,81 @@ dev = ["abi3audit", "black (==24.10.0)", "check-manifest", "coverage", "packagin test = ["pytest", "pytest-xdist", "setuptools"] [[package]] -name = "psycopg2" +name = "psycopg2-binary" version = "2.9.10" description = "psycopg2 - Python-PostgreSQL Database Adapter" optional = false python-versions = ">=3.8" groups = ["main"] files = [ - {file = "psycopg2-2.9.10-cp310-cp310-win32.whl", hash = "sha256:5df2b672140f95adb453af93a7d669d7a7bf0a56bcd26f1502329166f4a61716"}, - {file = "psycopg2-2.9.10-cp310-cp310-win_amd64.whl", hash = "sha256:c6f7b8561225f9e711a9c47087388a97fdc948211c10a4bccbf0ba68ab7b3b5a"}, - {file = "psycopg2-2.9.10-cp311-cp311-win32.whl", hash = "sha256:47c4f9875125344f4c2b870e41b6aad585901318068acd01de93f3677a6522c2"}, - {file = "psycopg2-2.9.10-cp311-cp311-win_amd64.whl", hash = "sha256:0435034157049f6846e95103bd8f5a668788dd913a7c30162ca9503fdf542cb4"}, - {file = "psycopg2-2.9.10-cp312-cp312-win32.whl", hash = "sha256:65a63d7ab0e067e2cdb3cf266de39663203d38d6a8ed97f5ca0cb315c73fe067"}, - {file = "psycopg2-2.9.10-cp312-cp312-win_amd64.whl", hash = "sha256:4a579d6243da40a7b3182e0430493dbd55950c493d8c68f4eec0b302f6bbf20e"}, - {file = "psycopg2-2.9.10-cp313-cp313-win_amd64.whl", hash = "sha256:91fd603a2155da8d0cfcdbf8ab24a2d54bca72795b90d2a3ed2b6da8d979dee2"}, - {file = "psycopg2-2.9.10-cp39-cp39-win32.whl", hash = "sha256:9d5b3b94b79a844a986d029eee38998232451119ad653aea42bb9220a8c5066b"}, - {file = "psycopg2-2.9.10-cp39-cp39-win_amd64.whl", hash = "sha256:88138c8dedcbfa96408023ea2b0c369eda40fe5d75002c0964c78f46f11fa442"}, - {file = "psycopg2-2.9.10.tar.gz", hash = "sha256:12ec0b40b0273f95296233e8750441339298e6a572f7039da5b260e3c8b60e11"}, + {file = "psycopg2-binary-2.9.10.tar.gz", hash = "sha256:4b3df0e6990aa98acda57d983942eff13d824135fe2250e6522edaa782a06de2"}, + {file = "psycopg2_binary-2.9.10-cp310-cp310-macosx_12_0_x86_64.whl", hash = "sha256:0ea8e3d0ae83564f2fc554955d327fa081d065c8ca5cc6d2abb643e2c9c1200f"}, + {file = "psycopg2_binary-2.9.10-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:3e9c76f0ac6f92ecfc79516a8034a544926430f7b080ec5a0537bca389ee0906"}, + {file = "psycopg2_binary-2.9.10-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2ad26b467a405c798aaa1458ba09d7e2b6e5f96b1ce0ac15d82fd9f95dc38a92"}, + {file = "psycopg2_binary-2.9.10-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:270934a475a0e4b6925b5f804e3809dd5f90f8613621d062848dd82f9cd62007"}, + {file = "psycopg2_binary-2.9.10-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:48b338f08d93e7be4ab2b5f1dbe69dc5e9ef07170fe1f86514422076d9c010d0"}, + {file = "psycopg2_binary-2.9.10-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7f4152f8f76d2023aac16285576a9ecd2b11a9895373a1f10fd9db54b3ff06b4"}, + {file = "psycopg2_binary-2.9.10-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:32581b3020c72d7a421009ee1c6bf4a131ef5f0a968fab2e2de0c9d2bb4577f1"}, + {file = "psycopg2_binary-2.9.10-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:2ce3e21dc3437b1d960521eca599d57408a695a0d3c26797ea0f72e834c7ffe5"}, + {file = "psycopg2_binary-2.9.10-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:e984839e75e0b60cfe75e351db53d6db750b00de45644c5d1f7ee5d1f34a1ce5"}, + {file = "psycopg2_binary-2.9.10-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:3c4745a90b78e51d9ba06e2088a2fe0c693ae19cc8cb051ccda44e8df8a6eb53"}, + {file = "psycopg2_binary-2.9.10-cp310-cp310-win32.whl", hash = "sha256:e5720a5d25e3b99cd0dc5c8a440570469ff82659bb09431c1439b92caf184d3b"}, + {file = "psycopg2_binary-2.9.10-cp310-cp310-win_amd64.whl", hash = "sha256:3c18f74eb4386bf35e92ab2354a12c17e5eb4d9798e4c0ad3a00783eae7cd9f1"}, + {file = "psycopg2_binary-2.9.10-cp311-cp311-macosx_12_0_x86_64.whl", hash = "sha256:04392983d0bb89a8717772a193cfaac58871321e3ec69514e1c4e0d4957b5aff"}, + {file = "psycopg2_binary-2.9.10-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:1a6784f0ce3fec4edc64e985865c17778514325074adf5ad8f80636cd029ef7c"}, + {file = "psycopg2_binary-2.9.10-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b5f86c56eeb91dc3135b3fd8a95dc7ae14c538a2f3ad77a19645cf55bab1799c"}, + {file = "psycopg2_binary-2.9.10-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2b3d2491d4d78b6b14f76881905c7a8a8abcf974aad4a8a0b065273a0ed7a2cb"}, + {file = "psycopg2_binary-2.9.10-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2286791ececda3a723d1910441c793be44625d86d1a4e79942751197f4d30341"}, + {file = "psycopg2_binary-2.9.10-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:512d29bb12608891e349af6a0cccedce51677725a921c07dba6342beaf576f9a"}, + {file = "psycopg2_binary-2.9.10-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:5a507320c58903967ef7384355a4da7ff3f28132d679aeb23572753cbf2ec10b"}, + {file = "psycopg2_binary-2.9.10-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:6d4fa1079cab9018f4d0bd2db307beaa612b0d13ba73b5c6304b9fe2fb441ff7"}, + {file = "psycopg2_binary-2.9.10-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:851485a42dbb0bdc1edcdabdb8557c09c9655dfa2ca0460ff210522e073e319e"}, + {file = "psycopg2_binary-2.9.10-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:35958ec9e46432d9076286dda67942ed6d968b9c3a6a2fd62b48939d1d78bf68"}, + {file = "psycopg2_binary-2.9.10-cp311-cp311-win32.whl", hash = "sha256:ecced182e935529727401b24d76634a357c71c9275b356efafd8a2a91ec07392"}, + {file = "psycopg2_binary-2.9.10-cp311-cp311-win_amd64.whl", hash = "sha256:ee0e8c683a7ff25d23b55b11161c2663d4b099770f6085ff0a20d4505778d6b4"}, + {file = "psycopg2_binary-2.9.10-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:880845dfe1f85d9d5f7c412efea7a08946a46894537e4e5d091732eb1d34d9a0"}, + {file = "psycopg2_binary-2.9.10-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:9440fa522a79356aaa482aa4ba500b65f28e5d0e63b801abf6aa152a29bd842a"}, + {file = "psycopg2_binary-2.9.10-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e3923c1d9870c49a2d44f795df0c889a22380d36ef92440ff618ec315757e539"}, + {file = "psycopg2_binary-2.9.10-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7b2c956c028ea5de47ff3a8d6b3cc3330ab45cf0b7c3da35a2d6ff8420896526"}, + {file = "psycopg2_binary-2.9.10-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f758ed67cab30b9a8d2833609513ce4d3bd027641673d4ebc9c067e4d208eec1"}, + {file = "psycopg2_binary-2.9.10-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8cd9b4f2cfab88ed4a9106192de509464b75a906462fb846b936eabe45c2063e"}, + {file = "psycopg2_binary-2.9.10-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:6dc08420625b5a20b53551c50deae6e231e6371194fa0651dbe0fb206452ae1f"}, + {file = "psycopg2_binary-2.9.10-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:d7cd730dfa7c36dbe8724426bf5612798734bff2d3c3857f36f2733f5bfc7c00"}, + {file = "psycopg2_binary-2.9.10-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:155e69561d54d02b3c3209545fb08938e27889ff5a10c19de8d23eb5a41be8a5"}, + {file = "psycopg2_binary-2.9.10-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c3cc28a6fd5a4a26224007712e79b81dbaee2ffb90ff406256158ec4d7b52b47"}, + {file = "psycopg2_binary-2.9.10-cp312-cp312-win32.whl", hash = "sha256:ec8a77f521a17506a24a5f626cb2aee7850f9b69a0afe704586f63a464f3cd64"}, + {file = "psycopg2_binary-2.9.10-cp312-cp312-win_amd64.whl", hash = "sha256:18c5ee682b9c6dd3696dad6e54cc7ff3a1a9020df6a5c0f861ef8bfd338c3ca0"}, + {file = "psycopg2_binary-2.9.10-cp313-cp313-macosx_12_0_x86_64.whl", hash = "sha256:26540d4a9a4e2b096f1ff9cce51253d0504dca5a85872c7f7be23be5a53eb18d"}, + {file = "psycopg2_binary-2.9.10-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:e217ce4d37667df0bc1c397fdcd8de5e81018ef305aed9415c3b093faaeb10fb"}, + {file = "psycopg2_binary-2.9.10-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:245159e7ab20a71d989da00f280ca57da7641fa2cdcf71749c193cea540a74f7"}, + {file = "psycopg2_binary-2.9.10-cp313-cp313-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3c4ded1a24b20021ebe677b7b08ad10bf09aac197d6943bfe6fec70ac4e4690d"}, + {file = "psycopg2_binary-2.9.10-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3abb691ff9e57d4a93355f60d4f4c1dd2d68326c968e7db17ea96df3c023ef73"}, + {file = "psycopg2_binary-2.9.10-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8608c078134f0b3cbd9f89b34bd60a943b23fd33cc5f065e8d5f840061bd0673"}, + {file = "psycopg2_binary-2.9.10-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:230eeae2d71594103cd5b93fd29d1ace6420d0b86f4778739cb1a5a32f607d1f"}, + {file = "psycopg2_binary-2.9.10-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:bb89f0a835bcfc1d42ccd5f41f04870c1b936d8507c6df12b7737febc40f0909"}, + {file = "psycopg2_binary-2.9.10-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:f0c2d907a1e102526dd2986df638343388b94c33860ff3bbe1384130828714b1"}, + {file = "psycopg2_binary-2.9.10-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:f8157bed2f51db683f31306aa497311b560f2265998122abe1dce6428bd86567"}, + {file = "psycopg2_binary-2.9.10-cp313-cp313-win_amd64.whl", hash = "sha256:27422aa5f11fbcd9b18da48373eb67081243662f9b46e6fd07c3eb46e4535142"}, + {file = "psycopg2_binary-2.9.10-cp38-cp38-macosx_12_0_x86_64.whl", hash = "sha256:eb09aa7f9cecb45027683bb55aebaaf45a0df8bf6de68801a6afdc7947bb09d4"}, + {file = "psycopg2_binary-2.9.10-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b73d6d7f0ccdad7bc43e6d34273f70d587ef62f824d7261c4ae9b8b1b6af90e8"}, + {file = "psycopg2_binary-2.9.10-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ce5ab4bf46a211a8e924d307c1b1fcda82368586a19d0a24f8ae166f5c784864"}, + {file = "psycopg2_binary-2.9.10-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:056470c3dc57904bbf63d6f534988bafc4e970ffd50f6271fc4ee7daad9498a5"}, + {file = "psycopg2_binary-2.9.10-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:73aa0e31fa4bb82578f3a6c74a73c273367727de397a7a0f07bd83cbea696baa"}, + {file = "psycopg2_binary-2.9.10-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:8de718c0e1c4b982a54b41779667242bc630b2197948405b7bd8ce16bcecac92"}, + {file = "psycopg2_binary-2.9.10-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:5c370b1e4975df846b0277b4deba86419ca77dbc25047f535b0bb03d1a544d44"}, + {file = "psycopg2_binary-2.9.10-cp38-cp38-musllinux_1_2_ppc64le.whl", hash = "sha256:ffe8ed017e4ed70f68b7b371d84b7d4a790368db9203dfc2d222febd3a9c8863"}, + {file = "psycopg2_binary-2.9.10-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:8aecc5e80c63f7459a1a2ab2c64df952051df196294d9f739933a9f6687e86b3"}, + {file = "psycopg2_binary-2.9.10-cp39-cp39-macosx_12_0_x86_64.whl", hash = "sha256:7a813c8bdbaaaab1f078014b9b0b13f5de757e2b5d9be6403639b298a04d218b"}, + {file = "psycopg2_binary-2.9.10-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d00924255d7fc916ef66e4bf22f354a940c67179ad3fd7067d7a0a9c84d2fbfc"}, + {file = "psycopg2_binary-2.9.10-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7559bce4b505762d737172556a4e6ea8a9998ecac1e39b5233465093e8cee697"}, + {file = "psycopg2_binary-2.9.10-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e8b58f0a96e7a1e341fc894f62c1177a7c83febebb5ff9123b579418fdc8a481"}, + {file = "psycopg2_binary-2.9.10-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6b269105e59ac96aba877c1707c600ae55711d9dcd3fc4b5012e4af68e30c648"}, + {file = "psycopg2_binary-2.9.10-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:79625966e176dc97ddabc142351e0409e28acf4660b88d1cf6adb876d20c490d"}, + {file = "psycopg2_binary-2.9.10-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:8aabf1c1a04584c168984ac678a668094d831f152859d06e055288fa515e4d30"}, + {file = "psycopg2_binary-2.9.10-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:19721ac03892001ee8fdd11507e6a2e01f4e37014def96379411ca99d78aeb2c"}, + {file = "psycopg2_binary-2.9.10-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:7f5d859928e635fa3ce3477704acee0f667b3a3d3e4bb109f2b18d4005f38287"}, + {file = "psycopg2_binary-2.9.10-cp39-cp39-win32.whl", hash = "sha256:3216ccf953b3f267691c90c6fe742e45d890d8272326b4a8b20850a03d05b7b8"}, + {file = "psycopg2_binary-2.9.10-cp39-cp39-win_amd64.whl", hash = "sha256:30e34c4e97964805f715206c7b789d54a78b70f3ff19fbe590104b71c45600e5"}, ] [[package]] @@ -1682,6 +1792,24 @@ urllib3 = ">=1.21.1,<3" socks = ["PySocks (>=1.5.6,!=1.5.7)"] use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] +[[package]] +name = "s3transfer" +version = "0.13.0" +description = "An Amazon S3 Transfer Manager" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "s3transfer-0.13.0-py3-none-any.whl", hash = "sha256:0148ef34d6dd964d0d8cf4311b2b21c474693e57c2e069ec708ce043d2b527be"}, + {file = "s3transfer-0.13.0.tar.gz", hash = "sha256:f5e6db74eb7776a37208001113ea7aa97695368242b364d73e91c981ac522177"}, +] + +[package.dependencies] +botocore = ">=1.37.4,<2.0a.0" + +[package.extras] +crt = ["botocore[crt] (>=1.37.4,<2.0a.0)"] + [[package]] name = "six" version = "1.17.0" @@ -1969,4 +2097,4 @@ files = [ [metadata] lock-version = "2.1" python-versions = ">=3.12" -content-hash = "1d5c1e0bfc12e88ca9b4c46141c848064a45e9cc4b60990fa3ec7ecb5ef71209" +content-hash = "dfda98ea4e00851a83a2f67c231b59476d407a1e38006610722c64842976e736" diff --git a/pyproject.toml b/pyproject.toml index 56b1bb8..b22d450 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,7 +15,6 @@ dependencies = [ "openpyxl (>=3.1.5,<4.0.0)", "fuzzywuzzy (>=0.18.0,<0.19.0)", "sqlmodel (>=0.0.24,<0.0.25)", - "psycopg2 (>=2.9.10,<3.0.0)", "pydantic-settings (>=2.8.1,<3.0.0)", "alembic (>=1.15.1,<2.0.0)", "pytest (>=8.3.5,<9.0.0)", @@ -23,6 +22,8 @@ dependencies = [ "beautifulsoup4 (>=4.13.4,<5.0.0)", "tqdm (>=4.67.1,<5.0.0)", "hubspot-api-client (>=12.0.0,<13.0.0)", + "boto3 (>=1.39.6,<2.0.0)", + "psycopg2-binary (>=2.9.10,<3.0.0)", ] [tool.poetry]