mirror of
https://github.com/Hestia-Homes/survey-extraction.git
synced 2026-06-30 13:10:56 +00:00
Merge pull request #68 from Hestia-Homes/feature/month_end_automation
Feature/month end automation
This commit is contained in:
Jun-te Kim
GitHub
commit
2664dcbf83
67 changed files with 4582 additions and 284 deletions
|
|
@ -3,6 +3,12 @@ FROM library/python:3.12-bullseye
|
|||
ARG USER=vscode
|
||||
ARG DEBIAN_FRONTEND=noninteractive
|
||||
|
||||
# DO NOT PUSH IMAGE TO ECR!!! as anyone with access to image can log on to our aws
|
||||
# Will log on as aws Jun-te account, change in the future to development account
|
||||
ENV AWS_ACCESS_KEY_ID=AKIAU5A36PPNK7RXX52V
|
||||
ENV AWS_SECRET_ACCESS_KEY=KRTjzoGVestZ0ifDwaAVqiPoXXZAvQKAjY5sVBtP
|
||||
ENV AWS_DEFAULT_REGION=eu-west-2
|
||||
|
||||
# Install system dependencies in a single layer
|
||||
RUN apt update && apt install -y --no-install-recommends \
|
||||
sudo jq vim curl\
|
||||
|
|
|
|||
|
|
@ -6,9 +6,8 @@
|
|||
"workspaceFolder": "/workspaces/survey-extractor",
|
||||
"postStartCommand": "bash .devcontainer/post-install.sh",
|
||||
"mounts": [
|
||||
"source=${localEnv:HOME},target=/workspaces/home,type=bind",
|
||||
// Make sure you aws credentials are saved at ~/.aws
|
||||
"source=${localEnv:HOME}/.aws/,target=/home/vscode/.aws/,type=bind"
|
||||
// Optional, just makes getting from Downloads (local env) easier
|
||||
"source=${localEnv:HOME},target=/workspaces/home,type=bind"
|
||||
],
|
||||
"customizations": {
|
||||
"vscode": {
|
||||
|
|
|
|||
86
.github/workflows/actions/lambda-deploy/action.yml
vendored
Normal file
86
.github/workflows/actions/lambda-deploy/action.yml
vendored
Normal file
|
|
@ -0,0 +1,86 @@
|
|||
name: "Build and Push Lambda Image to ECR"
|
||||
description: "Reusable action for building and pushing lambda Docker image to ECR"
|
||||
|
||||
inputs:
|
||||
lambda_name:
|
||||
description: "Lambda name / ECR repo name"
|
||||
required: true
|
||||
dockerfile_path:
|
||||
description: "Path to Dockerfile"
|
||||
required: true
|
||||
ecr_tf_dir:
|
||||
description: "Path to ECR terraform directory"
|
||||
required: true
|
||||
lambda_tf_dir:
|
||||
description: "Path to Lambda terraform directory"
|
||||
required: true
|
||||
aws-access-key-id:
|
||||
description: "AWS access key"
|
||||
required: true
|
||||
aws-secret-access-key:
|
||||
description: "AWS secret key"
|
||||
required: true
|
||||
aws-region:
|
||||
description: "AWS region"
|
||||
required: true
|
||||
git-sha:
|
||||
description: "Git commit SHA"
|
||||
required: true
|
||||
git-ref:
|
||||
description: "Git ref name"
|
||||
required: true
|
||||
|
||||
runs:
|
||||
using: "composite"
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Configure AWS credentials
|
||||
uses: aws-actions/configure-aws-credentials@v4
|
||||
with:
|
||||
aws-access-key-id: ${{ inputs.aws-access-key-id }}
|
||||
aws-secret-access-key: ${{ inputs.aws-secret-access-key }}
|
||||
aws-region: ${{ inputs.aws-region }}
|
||||
|
||||
- name: Log in to Amazon ECR
|
||||
id: login-ecr
|
||||
uses: aws-actions/amazon-ecr-login@v2
|
||||
|
||||
- name: Deploy ECR
|
||||
uses: ./.github/workflows/actions/terraform-deploy
|
||||
with:
|
||||
working_directory: ${{ inputs.ecr_tf_dir }}
|
||||
aws-access-key-id: ${{ inputs.aws-access-key-id }}
|
||||
aws-secret-access-key: ${{ inputs.aws-secret-access-key }}
|
||||
aws-region: ${{ inputs.aws-region }}
|
||||
- name: Set Docker image tag
|
||||
id: set_tag
|
||||
shell: bash
|
||||
run: |
|
||||
SHORT_SHA=$(echo "${{ inputs.git-sha }}" | cut -c1-7)
|
||||
BRANCH=$(echo "${{ inputs.git-ref }}" | tr '/' '-')
|
||||
TAG="${BRANCH}-${SHORT_SHA}"
|
||||
echo "IMAGE_TAG=${TAG}" >> $GITHUB_ENV
|
||||
echo "tag=$TAG" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: Build and push Docker image
|
||||
shell: bash
|
||||
run: |
|
||||
IMAGE_URI=${{ steps.login-ecr.outputs.registry }}/${{ inputs.lambda_name }}:${{ steps.set_tag.outputs.tag }}
|
||||
echo "Building Docker image for ${{ inputs.lambda_name }}..."
|
||||
docker build -t $IMAGE_URI -f ${{ inputs.dockerfile_path }} .
|
||||
|
||||
echo "Pushing to ECR..."
|
||||
docker push $IMAGE_URI
|
||||
|
||||
- name: Deploy Lambda
|
||||
uses: ./.github/workflows/actions/terraform-deploy
|
||||
with:
|
||||
working_directory: ${{ inputs.lambda_tf_dir }}
|
||||
aws-access-key-id: ${{ inputs.aws-access-key-id }}
|
||||
aws-secret-access-key: ${{ inputs.aws-secret-access-key }}
|
||||
aws-region: ${{ inputs.aws-region }}
|
||||
lambda-image-tag: ${{ steps.set_tag.outputs.tag }}
|
||||
|
||||
|
||||
|
||||
54
.github/workflows/actions/terraform-deploy/action.yml
vendored
Normal file
54
.github/workflows/actions/terraform-deploy/action.yml
vendored
Normal file
|
|
@ -0,0 +1,54 @@
|
|||
name: "Terraform Plan Shared Config"
|
||||
description: "Plans shared Terraform config for Lambdas"
|
||||
|
||||
inputs:
|
||||
working_directory:
|
||||
description: "Directory containing Terraform config"
|
||||
required: true
|
||||
aws-access-key-id:
|
||||
description: "AWS access key"
|
||||
required: true
|
||||
aws-secret-access-key:
|
||||
description: "AWS secret key"
|
||||
required: true
|
||||
aws-region:
|
||||
description: "AWS region"
|
||||
required: true
|
||||
lambda-image-tag:
|
||||
description: "Tag of the Lambda image (e.g., GitHub SHA)"
|
||||
required: false
|
||||
|
||||
runs:
|
||||
using: "composite"
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Configure AWS credentials
|
||||
uses: aws-actions/configure-aws-credentials@v4
|
||||
with:
|
||||
aws-access-key-id: ${{ inputs.aws-access-key-id }}
|
||||
aws-secret-access-key: ${{ inputs.aws-secret-access-key }}
|
||||
aws-region: ${{ inputs.aws-region }}
|
||||
|
||||
- name: Setup Terraform
|
||||
uses: hashicorp/setup-terraform@v3
|
||||
|
||||
- name: Terraform Init
|
||||
working-directory: ${{ inputs.working_directory }}
|
||||
shell: bash
|
||||
run: terraform init -reconfigure
|
||||
|
||||
- name: Terraform Plan
|
||||
working-directory: ${{ inputs.working_directory }}
|
||||
shell: bash
|
||||
run: |
|
||||
if [ -n "${{ inputs.lambda-image-tag }}" ]; then
|
||||
terraform plan -out=tfplan -var="lambda_image_tag=${{ inputs.lambda-image-tag }}"
|
||||
else
|
||||
terraform plan -out=tfplan
|
||||
fi
|
||||
|
||||
- name: Terraform Apply
|
||||
working-directory: ${{ inputs.working_directory }}
|
||||
shell: bash
|
||||
run: terraform apply -auto-approve tfplan
|
||||
69
.github/workflows/lambda_main.yml
vendored
Normal file
69
.github/workflows/lambda_main.yml
vendored
Normal file
|
|
@ -0,0 +1,69 @@
|
|||
name: Lambda Main Workflow
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main, feature/seperate_terraform_with_different_states]
|
||||
|
||||
env:
|
||||
AWS_REGION: eu-west-2
|
||||
|
||||
jobs:
|
||||
shared-lambda-terraform:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout repo
|
||||
uses: actions/checkout@v4
|
||||
- name: Deploy shared Lambda Config Terraform
|
||||
uses: ./.github/workflows/actions/terraform-deploy
|
||||
with:
|
||||
working_directory: ./deployment/lambda/lambda_shared
|
||||
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
|
||||
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
||||
aws-region: ${{ secrets.AWS_REGION }}
|
||||
|
||||
lambda-ecr-example:
|
||||
runs-on: ubuntu-latest
|
||||
needs: shared-lambda-terraform
|
||||
permissions:
|
||||
id-token: write
|
||||
contents: read
|
||||
|
||||
steps:
|
||||
- name: Checkout repo
|
||||
uses: actions/checkout@v4
|
||||
- name: Build and deploy Lambda example
|
||||
uses: ./.github/workflows/actions/lambda-deploy
|
||||
with:
|
||||
lambda_name: lambda_example
|
||||
dockerfile_path: ./deployment/lambda/lambda_example/docker/Dockerfile
|
||||
ecr_tf_dir: ./deployment/lambda/lambda_example/docker/
|
||||
lambda_tf_dir: ./deployment/lambda/lambda_example/
|
||||
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
|
||||
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
||||
aws-region: ${{ secrets.AWS_REGION }}
|
||||
git-sha: ${{ github.sha }}
|
||||
git-ref: ${{ github.ref_name }}
|
||||
|
||||
extractor-and-loader:
|
||||
runs-on: ubuntu-latest
|
||||
needs: shared-lambda-terraform
|
||||
permissions:
|
||||
id-token: write
|
||||
contents: read
|
||||
|
||||
steps:
|
||||
- name: Checkout repo
|
||||
uses: actions/checkout@v4
|
||||
- name: Build and deploy Extractor & Loader Lambda
|
||||
uses: ./.github/workflows/actions/lambda-deploy
|
||||
with:
|
||||
lambda_name: extractor_and_loader
|
||||
dockerfile_path: ./deployment/lambda/extractor_and_loader/docker/Dockerfile
|
||||
ecr_tf_dir: ./deployment/lambda/extractor_and_loader/docker/
|
||||
lambda_tf_dir: ./deployment/lambda/extractor_and_loader/
|
||||
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
|
||||
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
||||
aws-region: ${{ secrets.AWS_REGION }}
|
||||
git-sha: ${{ github.sha }}
|
||||
git-ref: ${{ github.ref_name }}
|
||||
|
||||
49
.github/workflows/push_docker_image_to_ecr.yml
vendored
49
.github/workflows/push_docker_image_to_ecr.yml
vendored
|
|
@ -1,49 +0,0 @@
|
|||
name: Build and Push Docker Image to ECR lambda example
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [feature/energy_report_etl, main]
|
||||
|
||||
env:
|
||||
AWS_REGION: eu-west-2
|
||||
ECR_REPOSITORY: lambda_example
|
||||
|
||||
jobs:
|
||||
build-and-push-to-elastic-container-registry:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
permissions:
|
||||
id-token: write
|
||||
contents: read
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: AWS credentials
|
||||
uses: aws-actions/configure-aws-credentials@v4
|
||||
with:
|
||||
# as of 14/07/2025 it'll be using user:Junte's keys
|
||||
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
|
||||
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
||||
aws-region: ${{ secrets.AWS_REGION }}
|
||||
|
||||
- name: Log in to Amazon ECR
|
||||
id: login-ecr
|
||||
uses: aws-actions/amazon-ecr-login@v2
|
||||
|
||||
- name: Build, tag, and push Docker image to ECR
|
||||
env:
|
||||
ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
|
||||
IMAGE_TAG: latest
|
||||
run: |
|
||||
IMAGE_URI=${{ env.ECR_REGISTRY }}/${{ env.ECR_REPOSITORY }}:${{ env.IMAGE_TAG }}
|
||||
echo "pwd"
|
||||
pwd
|
||||
ls -la
|
||||
echo "Building Docker image..."
|
||||
docker build -t $IMAGE_URI -f deployment/lambda_example/Dockerfile .
|
||||
|
||||
echo "Pushing Docker image to ECR..."
|
||||
docker push $IMAGE_URI
|
||||
|
||||
|
|
@ -2,14 +2,13 @@ terraform {
|
|||
required_providers {
|
||||
aws = {
|
||||
source = "hashicorp/aws"
|
||||
version = "~> 4.16"
|
||||
version = "~> 6.3.0"
|
||||
}
|
||||
}
|
||||
backend "s3" {
|
||||
bucket = "survey-extractor-tf-state"
|
||||
region = "eu-west-2"
|
||||
profile = "domna.dev" # /home/vscode/aws/credentials
|
||||
key = "terraform.tfstate"
|
||||
key = "env:/dev/terraform.tfstate"
|
||||
}
|
||||
|
||||
required_version = ">= 1.2.0"
|
||||
|
|
@ -14,4 +14,4 @@ variable allocated_storage {
|
|||
description = "The allocated storage in gigabytes"
|
||||
type = number
|
||||
default = 20
|
||||
}
|
||||
}
|
||||
21
deployment/lambda/extractor_and_loader/docker/.dockerignore
Normal file
21
deployment/lambda/extractor_and_loader/docker/.dockerignore
Normal file
|
|
@ -0,0 +1,21 @@
|
|||
# Ignore junk and large files
|
||||
*.pdf
|
||||
*.csv
|
||||
*.xml
|
||||
*.parquet
|
||||
*.ipynb
|
||||
*.mp4
|
||||
*.mov
|
||||
*.jpg
|
||||
*.png
|
||||
*.zip
|
||||
*.tar.gz
|
||||
__pycache__/
|
||||
*.pyc
|
||||
*.pyo
|
||||
*.pyd
|
||||
build/
|
||||
dist/
|
||||
.etl_cache/
|
||||
tests/
|
||||
docs/
|
||||
25
deployment/lambda/extractor_and_loader/docker/Dockerfile
Normal file
25
deployment/lambda/extractor_and_loader/docker/Dockerfile
Normal file
|
|
@ -0,0 +1,25 @@
|
|||
FROM public.ecr.aws/lambda/python:3.12
|
||||
|
||||
# Install Poetry (you could pin a version if you like)
|
||||
RUN curl -sSL https://install.python-poetry.org | python3 -
|
||||
|
||||
# Add Poetry to PATH
|
||||
ENV PATH="/root/.local/bin:$PATH"
|
||||
|
||||
# Set working directory
|
||||
WORKDIR /var/task
|
||||
|
||||
# Copy Poetry files first to leverage Docker layer caching
|
||||
COPY pyproject.toml poetry.lock README.md ./
|
||||
COPY etl/ etl/
|
||||
|
||||
|
||||
# Install dependencies into /var/task
|
||||
RUN poetry config virtualenvs.create false \
|
||||
&& poetry install --only main --no-interaction --no-ansi
|
||||
|
||||
# Copy app code
|
||||
COPY deployment/lambda/extractor_and_loader/docker/app.py ./
|
||||
|
||||
# Set Lambda handler
|
||||
CMD ["app.handler"]
|
||||
30
deployment/lambda/extractor_and_loader/docker/app.py
Normal file
30
deployment/lambda/extractor_and_loader/docker/app.py
Normal file
|
|
@ -0,0 +1,30 @@
|
|||
"""
|
||||
A quick example of lambda working a function in python
|
||||
"""
|
||||
from etl.read_stuff_from_s3_example import print_hello_from_etl_module
|
||||
|
||||
def handler(event, context):
|
||||
print("Outside try statment")
|
||||
print_hello_from_etl_module()
|
||||
try:
|
||||
print("show me something.. anything...")
|
||||
s3_uri = event.get("file_location")
|
||||
if not s3_uri:
|
||||
print("failed to get s3_uri")
|
||||
return {
|
||||
"statusCode": 400,
|
||||
"body": "Missing 'file_location' in event"
|
||||
}
|
||||
print(f"s3 uri is {s3_uri}")
|
||||
|
||||
return {
|
||||
"statusCode": 200,
|
||||
"body": f"s3 uri {s3_uri}"
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
print(f"❌ Error: {e}")
|
||||
return {
|
||||
"statusCode": 500,
|
||||
"body": str(e)
|
||||
}
|
||||
62
deployment/lambda/extractor_and_loader/docker/ecr.tf
Normal file
62
deployment/lambda/extractor_and_loader/docker/ecr.tf
Normal file
|
|
@ -0,0 +1,62 @@
|
|||
# ECR repo
|
||||
resource "aws_ecr_repository" "extractor_and_loader" {
|
||||
name = "extractor_and_loader"
|
||||
}
|
||||
|
||||
# ECR policy to allow Lambda access
|
||||
resource "aws_ecr_repository_policy" "extractor_loader_ecr_access" {
|
||||
repository = aws_ecr_repository.extractor_and_loader.name
|
||||
|
||||
policy = jsonencode({
|
||||
Version = "2008-10-17",
|
||||
Statement = [{
|
||||
Sid = "AllowLambdaPull",
|
||||
Effect = "Allow",
|
||||
Principal = {
|
||||
Service = "lambda.amazonaws.com"
|
||||
},
|
||||
Action = [
|
||||
"ecr:GetDownloadUrlForLayer",
|
||||
"ecr:BatchGetImage",
|
||||
"ecr:BatchCheckLayerAvailability"
|
||||
]
|
||||
}]
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
# ECR lifecycle policy to delete tagged images older than 14 days
|
||||
resource "aws_ecr_lifecycle_policy" "extractor_loader_lifecycle" {
|
||||
repository = aws_ecr_repository.extractor_and_loader.name
|
||||
|
||||
policy = jsonencode({
|
||||
"rules": [
|
||||
{
|
||||
"rulePriority": 2,
|
||||
"description": "Expire images older than 14 days",
|
||||
"selection": {
|
||||
"tagStatus": "untagged",
|
||||
"countType": "sinceImagePushed",
|
||||
"countUnit": "days",
|
||||
"countNumber": 1
|
||||
},
|
||||
"action": {
|
||||
"type": "expire"
|
||||
}
|
||||
},
|
||||
{
|
||||
"rulePriority": 1,
|
||||
"description": "Keep last 5 images",
|
||||
"selection": {
|
||||
"tagStatus": "tagged",
|
||||
"tagPrefixList": ["feature"],
|
||||
"countType": "imageCountMoreThan",
|
||||
"countNumber": 5
|
||||
},
|
||||
"action": {
|
||||
"type": "expire"
|
||||
}
|
||||
}
|
||||
]
|
||||
})
|
||||
}
|
||||
0
deployment/lambda/extractor_and_loader/docker/main.tf
Normal file
0
deployment/lambda/extractor_and_loader/docker/main.tf
Normal file
15
deployment/lambda/extractor_and_loader/docker/provider.tf
Normal file
15
deployment/lambda/extractor_and_loader/docker/provider.tf
Normal file
|
|
@ -0,0 +1,15 @@
|
|||
terraform {
|
||||
required_providers {
|
||||
aws = {
|
||||
source = "hashicorp/aws"
|
||||
version = "~> 6.3.0"
|
||||
}
|
||||
}
|
||||
backend "s3" {
|
||||
bucket = "survey-extractor-tf-state"
|
||||
region = "eu-west-2"
|
||||
key = "env:/dev/lambda/ecr/extractor_and_loader.tfstate"
|
||||
}
|
||||
|
||||
required_version = ">= 1.2.0"
|
||||
}
|
||||
|
|
@ -0,0 +1,71 @@
|
|||
# Reference existing IAM role
|
||||
data "aws_iam_role" "lambda_exec_role" {
|
||||
name = "lambda-exec-role"
|
||||
}
|
||||
|
||||
# Reference existing ECR repository
|
||||
data "aws_ecr_repository" "extractor_and_loader" {
|
||||
name = "extractor_and_loader"
|
||||
}
|
||||
|
||||
# SQS queue for extractor_and_loader
|
||||
resource "aws_sqs_queue" "extractor_and_loader_queue" {
|
||||
name = "extractor-loader-queue"
|
||||
}
|
||||
|
||||
|
||||
# IAM policy specific to this Lambda
|
||||
resource "aws_iam_policy" "extractor_loader_policy" {
|
||||
name = "extractor-loader-policy"
|
||||
|
||||
policy = jsonencode({
|
||||
Version = "2012-10-17",
|
||||
Statement = [
|
||||
{
|
||||
Effect = "Allow",
|
||||
Action = [
|
||||
"sqs:ReceiveMessage",
|
||||
"sqs:DeleteMessage",
|
||||
"sqs:GetQueueAttributes"
|
||||
],
|
||||
Resource = aws_sqs_queue.extractor_and_loader_queue.arn
|
||||
},
|
||||
{
|
||||
Effect = "Allow",
|
||||
Action = [
|
||||
"ecr:GetDownloadUrlForLayer",
|
||||
"ecr:BatchGetImage",
|
||||
"ecr:BatchCheckLayerAvailability"
|
||||
],
|
||||
Resource = data.aws_ecr_repository.extractor_and_loader.arn
|
||||
},
|
||||
{
|
||||
Effect = "Allow",
|
||||
Action = ["ecr:GetAuthorizationToken"],
|
||||
Resource = "*"
|
||||
}
|
||||
]
|
||||
})
|
||||
}
|
||||
|
||||
resource "aws_iam_role_policy_attachment" "extractor_loader_policy_attach" {
|
||||
role = data.aws_iam_role.lambda_exec_role.name
|
||||
policy_arn = aws_iam_policy.extractor_loader_policy.arn
|
||||
}
|
||||
|
||||
# Lambda function
|
||||
resource "aws_lambda_function" "extractor_and_loader" {
|
||||
function_name = "extractor-and-loader"
|
||||
role = data.aws_iam_role.lambda_exec_role.arn
|
||||
package_type = "Image"
|
||||
image_uri = "${data.aws_ecr_repository.extractor_and_loader.repository_url}:${var.lambda_image_tag}"
|
||||
timeout = 30
|
||||
}
|
||||
|
||||
# SQS trigger
|
||||
resource "aws_lambda_event_source_mapping" "extractor_and_loader_trigger" {
|
||||
event_source_arn = aws_sqs_queue.extractor_and_loader_queue.arn
|
||||
function_name = aws_lambda_function.extractor_and_loader.arn
|
||||
batch_size = 1
|
||||
}
|
||||
|
||||
0
deployment/lambda/extractor_and_loader/main.tf
Normal file
0
deployment/lambda/extractor_and_loader/main.tf
Normal file
15
deployment/lambda/extractor_and_loader/provider.tf
Normal file
15
deployment/lambda/extractor_and_loader/provider.tf
Normal file
|
|
@ -0,0 +1,15 @@
|
|||
terraform {
|
||||
required_providers {
|
||||
aws = {
|
||||
source = "hashicorp/aws"
|
||||
version = "~> 6.3.0"
|
||||
}
|
||||
}
|
||||
backend "s3" {
|
||||
bucket = "survey-extractor-tf-state"
|
||||
region = "eu-west-2"
|
||||
key = "env:/dev/lambda/eachlambda/extractor_and_loader_lambda.tfstate"
|
||||
}
|
||||
|
||||
required_version = ">= 1.2.0"
|
||||
}
|
||||
5
deployment/lambda/extractor_and_loader/vars.tf
Normal file
5
deployment/lambda/extractor_and_loader/vars.tf
Normal file
|
|
@ -0,0 +1,5 @@
|
|||
variable "lambda_image_tag" {
|
||||
description = "Docker image tag (e.g. GitHub SHA)"
|
||||
type = string
|
||||
default = "local-dev-latest"
|
||||
}
|
||||
|
|
@ -2,7 +2,7 @@
|
|||
FROM public.ecr.aws/lambda/python:3.11
|
||||
|
||||
# Copy function code
|
||||
COPY deployment/lambda_example/app.py ./
|
||||
COPY deployment/lambda/lambda_example/docker/app.py ./
|
||||
|
||||
# Set the CMD to your handler
|
||||
CMD ["app.handler"]
|
||||
19
deployment/lambda/lambda_example/docker/app.py
Normal file
19
deployment/lambda/lambda_example/docker/app.py
Normal file
|
|
@ -0,0 +1,19 @@
|
|||
"""
|
||||
A quick example of lambda working a function in python
|
||||
"""
|
||||
|
||||
def handler(event, context):
|
||||
try:
|
||||
print("Printing from lambda example")
|
||||
|
||||
return {
|
||||
"statusCode": 200,
|
||||
"body": f"s3 uri {s3_uri}"
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
print(f"❌ Error: {e}")
|
||||
return {
|
||||
"statusCode": 500,
|
||||
"body": str(e)
|
||||
}
|
||||
61
deployment/lambda/lambda_example/docker/ecr.tf
Normal file
61
deployment/lambda/lambda_example/docker/ecr.tf
Normal file
|
|
@ -0,0 +1,61 @@
|
|||
# ECR repo for lambda_example
|
||||
resource "aws_ecr_repository" "lambda_example" {
|
||||
name = "lambda_example"
|
||||
}
|
||||
|
||||
# ECR policy to allow Lambda access
|
||||
resource "aws_ecr_repository_policy" "lambda_example_ecr_access" {
|
||||
repository = aws_ecr_repository.lambda_example.name
|
||||
|
||||
policy = jsonencode({
|
||||
Version = "2008-10-17",
|
||||
Statement = [{
|
||||
Sid = "AllowLambdaPull",
|
||||
Effect = "Allow",
|
||||
Principal = {
|
||||
Service = "lambda.amazonaws.com"
|
||||
},
|
||||
Action = [
|
||||
"ecr:GetDownloadUrlForLayer",
|
||||
"ecr:BatchGetImage",
|
||||
"ecr:BatchCheckLayerAvailability"
|
||||
]
|
||||
}]
|
||||
})
|
||||
}
|
||||
|
||||
# ECR lifecycle policy to delete tagged images older than 14 days
|
||||
resource "aws_ecr_lifecycle_policy" "lambda_example_ecr_lifecycle" {
|
||||
repository = aws_ecr_repository.lambda_example.name
|
||||
|
||||
policy = jsonencode({
|
||||
"rules": [
|
||||
{
|
||||
"rulePriority": 2,
|
||||
"description": "Expire images older than 14 days",
|
||||
"selection": {
|
||||
"tagStatus": "untagged",
|
||||
"countType": "sinceImagePushed",
|
||||
"countUnit": "days",
|
||||
"countNumber": 1
|
||||
},
|
||||
"action": {
|
||||
"type": "expire"
|
||||
}
|
||||
},
|
||||
{
|
||||
"rulePriority": 1,
|
||||
"description": "Keep last 5 images",
|
||||
"selection": {
|
||||
"tagStatus": "tagged",
|
||||
"tagPrefixList": ["feature"],
|
||||
"countType": "imageCountMoreThan",
|
||||
"countNumber": 5
|
||||
},
|
||||
"action": {
|
||||
"type": "expire"
|
||||
}
|
||||
}
|
||||
]
|
||||
})
|
||||
}
|
||||
0
deployment/lambda/lambda_example/docker/main.tf
Normal file
0
deployment/lambda/lambda_example/docker/main.tf
Normal file
3
deployment/lambda/lambda_example/docker/output.tf
Normal file
3
deployment/lambda/lambda_example/docker/output.tf
Normal file
|
|
@ -0,0 +1,3 @@
|
|||
output "ecr_repo_url" {
|
||||
value = aws_ecr_repository.lambda_example.repository_url
|
||||
}
|
||||
15
deployment/lambda/lambda_example/docker/provider.tf
Normal file
15
deployment/lambda/lambda_example/docker/provider.tf
Normal file
|
|
@ -0,0 +1,15 @@
|
|||
terraform {
|
||||
required_providers {
|
||||
aws = {
|
||||
source = "hashicorp/aws"
|
||||
version = "~> 6.3.0"
|
||||
}
|
||||
}
|
||||
backend "s3" {
|
||||
bucket = "survey-extractor-tf-state"
|
||||
region = "eu-west-2"
|
||||
key = "env:/dev/lambda/ecr/lambda_example_ecr.tfstate"
|
||||
}
|
||||
|
||||
required_version = ">= 1.2.0"
|
||||
}
|
||||
|
|
@ -0,0 +1,69 @@
|
|||
# Reference existing IAM role
|
||||
data "aws_iam_role" "lambda_exec_role" {
|
||||
name = "lambda-exec-role"
|
||||
}
|
||||
|
||||
# Reference existing ECR repository
|
||||
data "aws_ecr_repository" "lambda_example" {
|
||||
name = "lambda_example"
|
||||
}
|
||||
|
||||
# SQS queue for lambda_example
|
||||
resource "aws_sqs_queue" "lambda_example_queue" {
|
||||
name = "lambda-example-queue"
|
||||
}
|
||||
|
||||
# Custom IAM policy specific to lambda_example
|
||||
resource "aws_iam_policy" "lambda_example_policy" {
|
||||
name = "lambda-example-policy"
|
||||
|
||||
policy = jsonencode({
|
||||
Version = "2012-10-17",
|
||||
Statement = [
|
||||
{
|
||||
Effect = "Allow",
|
||||
Action = [
|
||||
"sqs:ReceiveMessage",
|
||||
"sqs:DeleteMessage",
|
||||
"sqs:GetQueueAttributes"
|
||||
],
|
||||
Resource = aws_sqs_queue.lambda_example_queue.arn
|
||||
},
|
||||
{
|
||||
Effect = "Allow",
|
||||
Action = [
|
||||
"ecr:GetDownloadUrlForLayer",
|
||||
"ecr:BatchGetImage",
|
||||
"ecr:BatchCheckLayerAvailability"
|
||||
],
|
||||
Resource = data.aws_ecr_repository.lambda_example.arn
|
||||
},
|
||||
{
|
||||
Effect = "Allow",
|
||||
Action = ["ecr:GetAuthorizationToken"],
|
||||
Resource = "*"
|
||||
}
|
||||
]
|
||||
})
|
||||
}
|
||||
|
||||
resource "aws_iam_role_policy_attachment" "lambda_example_policy_attach" {
|
||||
role = data.aws_iam_role.lambda_exec_role.name
|
||||
policy_arn = aws_iam_policy.lambda_example_policy.arn
|
||||
}
|
||||
|
||||
# Lambda function
|
||||
resource "aws_lambda_function" "lambda_example" {
|
||||
function_name = "lambda-example"
|
||||
role = data.aws_iam_role.lambda_exec_role.arn
|
||||
package_type = "Image"
|
||||
image_uri = "${data.aws_ecr_repository.lambda_example.repository_url}:${var.lambda_image_tag}"
|
||||
timeout = 10
|
||||
}
|
||||
|
||||
# SQS trigger
|
||||
resource "aws_lambda_event_source_mapping" "lambda_example_trigger" {
|
||||
event_source_arn = aws_sqs_queue.lambda_example_queue.arn
|
||||
function_name = aws_lambda_function.lambda_example.arn
|
||||
batch_size = 1
|
||||
}
|
||||
0
deployment/lambda/lambda_example/main.tf
Normal file
0
deployment/lambda/lambda_example/main.tf
Normal file
15
deployment/lambda/lambda_example/provider.tf
Normal file
15
deployment/lambda/lambda_example/provider.tf
Normal file
|
|
@ -0,0 +1,15 @@
|
|||
terraform {
|
||||
required_providers {
|
||||
aws = {
|
||||
source = "hashicorp/aws"
|
||||
version = "~> 6.3.0"
|
||||
}
|
||||
}
|
||||
backend "s3" {
|
||||
bucket = "survey-extractor-tf-state"
|
||||
region = "eu-west-2"
|
||||
key = "env:/dev/lambda/eachlambda/lambda_example.tfstate"
|
||||
}
|
||||
|
||||
required_version = ">= 1.2.0"
|
||||
}
|
||||
5
deployment/lambda/lambda_example/vars.tf
Normal file
5
deployment/lambda/lambda_example/vars.tf
Normal file
|
|
@ -0,0 +1,5 @@
|
|||
variable "lambda_image_tag" {
|
||||
description = "Docker image tag (e.g. GitHub SHA)"
|
||||
type = string
|
||||
default = "local-dev-latest"
|
||||
}
|
||||
21
deployment/lambda/lambda_shared/lambda_shared_config.tf
Normal file
21
deployment/lambda/lambda_shared/lambda_shared_config.tf
Normal file
|
|
@ -0,0 +1,21 @@
|
|||
# IAM role for both Lambdas (can be shared)
|
||||
resource "aws_iam_role" "lambda_exec_role" {
|
||||
name = "lambda-exec-role"
|
||||
|
||||
assume_role_policy = jsonencode({
|
||||
Version = "2012-10-17",
|
||||
Statement = [{
|
||||
Effect = "Allow",
|
||||
Principal = {
|
||||
Service = "lambda.amazonaws.com"
|
||||
},
|
||||
Action = "sts:AssumeRole"
|
||||
}]
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
resource "aws_iam_role_policy_attachment" "lambda_basic_execution" {
|
||||
role = aws_iam_role.lambda_exec_role.name
|
||||
policy_arn = "arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole"
|
||||
}
|
||||
0
deployment/lambda/lambda_shared/main.tf
Normal file
0
deployment/lambda/lambda_shared/main.tf
Normal file
9
deployment/lambda/lambda_shared/output.tf
Normal file
9
deployment/lambda/lambda_shared/output.tf
Normal file
|
|
@ -0,0 +1,9 @@
|
|||
output "lambda_exec_role_arn" {
|
||||
description = "The ARN of the IAM role used by the Lambda functions"
|
||||
value = aws_iam_role.lambda_exec_role.arn
|
||||
}
|
||||
|
||||
output "lambda_exec_role_name" {
|
||||
description = "The ARN of the IAM role used by the Lambda functions"
|
||||
value = aws_iam_role.lambda_exec_role.name
|
||||
}
|
||||
15
deployment/lambda/lambda_shared/provider.tf
Normal file
15
deployment/lambda/lambda_shared/provider.tf
Normal file
|
|
@ -0,0 +1,15 @@
|
|||
terraform {
|
||||
required_providers {
|
||||
aws = {
|
||||
source = "hashicorp/aws"
|
||||
version = "~> 6.3.0"
|
||||
}
|
||||
}
|
||||
backend "s3" {
|
||||
bucket = "survey-extractor-tf-state"
|
||||
region = "eu-west-2"
|
||||
key = "env:/dev/lambda/lambda_share_configuration.tfstate"
|
||||
}
|
||||
|
||||
required_version = ">= 1.2.0"
|
||||
}
|
||||
|
|
@ -1,121 +0,0 @@
|
|||
# This is an example file to setup a lamda function with a sqs and cloudwatch.
|
||||
# Please us this as a template for future lambda.
|
||||
# Be sure to push the image you are using to ECR or it won't deploy properly
|
||||
|
||||
# Create an SQS queue that will trigger the Lambda
|
||||
resource "aws_sqs_queue" "my_queue" {
|
||||
name = "my-lambda-queue"
|
||||
}
|
||||
|
||||
# Create an ECR repository to store the Docker image for the Lambda function
|
||||
resource "aws_ecr_repository" "lambda_repo" {
|
||||
name = "lambda_example"
|
||||
}
|
||||
|
||||
# IAM role that the Lambda function will assume
|
||||
resource "aws_iam_role" "lambda_exec_role" {
|
||||
name = "lambda-exec-role"
|
||||
|
||||
assume_role_policy = jsonencode({
|
||||
Version = "2012-10-17",
|
||||
Statement = [
|
||||
{
|
||||
Action = "sts:AssumeRole",
|
||||
Effect = "Allow",
|
||||
Principal = {
|
||||
Service = "lambda.amazonaws.com"
|
||||
}
|
||||
}
|
||||
]
|
||||
})
|
||||
}
|
||||
|
||||
# Attach AWS-managed policy for basic Lambda execution (CloudWatch logging)
|
||||
resource "aws_iam_role_policy_attachment" "lambda_basic_execution" {
|
||||
role = aws_iam_role.lambda_exec_role.name
|
||||
policy_arn = "arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole"
|
||||
}
|
||||
|
||||
# Custom policy: SQS access + ECR image pull permissions
|
||||
resource "aws_iam_policy" "lambda_custom_policy" {
|
||||
name = "lambda-sqs-ecr-policy"
|
||||
|
||||
policy = jsonencode({
|
||||
Version = "2012-10-17",
|
||||
Statement = [
|
||||
# Allow Lambda to read from SQS
|
||||
{
|
||||
Effect = "Allow",
|
||||
Action = [
|
||||
"sqs:ReceiveMessage",
|
||||
"sqs:DeleteMessage",
|
||||
"sqs:GetQueueAttributes"
|
||||
],
|
||||
Resource = aws_sqs_queue.my_queue.arn
|
||||
},
|
||||
# Allow Lambda to pull images from ECR
|
||||
{
|
||||
Effect = "Allow",
|
||||
Action = [
|
||||
"ecr:GetDownloadUrlForLayer",
|
||||
"ecr:BatchGetImage",
|
||||
"ecr:BatchCheckLayerAvailability"
|
||||
],
|
||||
Resource = aws_ecr_repository.lambda_repo.arn
|
||||
},
|
||||
# Needed to authenticate to ECR (pulling the image)
|
||||
{
|
||||
Effect = "Allow",
|
||||
Action = [
|
||||
"ecr:GetAuthorizationToken"
|
||||
],
|
||||
Resource = "*"
|
||||
}
|
||||
]
|
||||
})
|
||||
}
|
||||
|
||||
# Attach the custom policy to the Lambda role
|
||||
resource "aws_iam_role_policy_attachment" "lambda_custom_policy_attach" {
|
||||
role = aws_iam_role.lambda_exec_role.name
|
||||
policy_arn = aws_iam_policy.lambda_custom_policy.arn
|
||||
}
|
||||
|
||||
# Define the Lambda function using a Docker image from ECR
|
||||
resource "aws_lambda_function" "lambda_docker" {
|
||||
function_name = "docker-hello-world-python-example"
|
||||
role = aws_iam_role.lambda_exec_role.arn
|
||||
package_type = "Image"
|
||||
image_uri = "${aws_ecr_repository.lambda_repo.repository_url}:latest"
|
||||
timeout = 10
|
||||
}
|
||||
|
||||
# Connect the SQS queue to the Lambda so it gets triggered by incoming messages
|
||||
resource "aws_lambda_event_source_mapping" "sqs_trigger" {
|
||||
event_source_arn = aws_sqs_queue.my_queue.arn
|
||||
function_name = aws_lambda_function.lambda_docker.arn
|
||||
batch_size = 1
|
||||
}
|
||||
|
||||
|
||||
resource "aws_ecr_repository_policy" "lambda_ecr_access" {
|
||||
repository = aws_ecr_repository.lambda_repo.name
|
||||
|
||||
policy = jsonencode({
|
||||
Version = "2008-10-17",
|
||||
Statement = [
|
||||
{
|
||||
Sid = "AllowLambdaPull",
|
||||
Effect = "Allow",
|
||||
Principal = {
|
||||
Service = "lambda.amazonaws.com"
|
||||
},
|
||||
Action = [
|
||||
"ecr:GetDownloadUrlForLayer",
|
||||
"ecr:BatchGetImage",
|
||||
"ecr:BatchCheckLayerAvailability"
|
||||
]
|
||||
}
|
||||
]
|
||||
})
|
||||
}
|
||||
|
|
@ -1,11 +0,0 @@
|
|||
"""
|
||||
A quick example of lambda working a function in python
|
||||
"""
|
||||
|
||||
def handler(event, context):
|
||||
print("Hello from Python function. This shold be running from a dockerfile env and executed on a aws lambda!")
|
||||
return {
|
||||
'statusCode': 200,
|
||||
'body': 'Hello World'
|
||||
}
|
||||
|
||||
|
|
@ -1,7 +1,7 @@
|
|||
from etl.surveyedData.surveryedData import surveyedDataProcessor
|
||||
|
||||
files = [
|
||||
"/tmp/sharepoint/Sandwell/SANDWELL-001/26 Willow close B64 6EG/Content (13).pdf",
|
||||
# "/tmp/sharepoint/Sandwell/SANDWELL-001/26 Willow close B64 6EG/Content (13).pdf",
|
||||
"/tmp/sharepoint/Livewest/Livewest-001/12 Birch End/Summary Information 12 Birch End.pdf"
|
||||
]
|
||||
|
||||
|
|
|
|||
|
|
@ -33,7 +33,6 @@ class pdfReaderToText():
|
|||
self.all_text += text
|
||||
|
||||
self.text_list = self.all_text.split('\n')
|
||||
pprint(self.text_list)
|
||||
|
||||
def get_list_of_text(self):
|
||||
return self.text_list
|
||||
|
|
|
|||
|
|
@ -298,7 +298,6 @@ class WarmHomesConditionReport(SiteNotesExtractor):
|
|||
|
||||
# Gable 2
|
||||
data = self.get_data_between("2.2.4. External Elevation - Gable 2", "2.3. Conservatory or Outbuilding")
|
||||
pprint(self.raw_data)
|
||||
state = True if self.get_next_value(data, "Is there a 4th external elevation?").lower() == "yes" else False
|
||||
if state is False:
|
||||
gable_two = ExternalElevationGableTwo(is_there_a_fourth_external_elevation=state)
|
||||
|
|
@ -1601,9 +1600,9 @@ class EnergyPerformanceReportWithData(SiteNotesExtractor):
|
|||
|
||||
class EnergyPerformanceReportSummaryInformation(SiteNotesExtractor):
|
||||
def __init__(self, data_list):
|
||||
super().__init__(data_list)
|
||||
self.raw_data = data_list
|
||||
self.type = ReportType.ENERGY_PERFORMANCE_REPORT_SUMMARY_INFORMATION
|
||||
self.master_obj = self.setup()
|
||||
self.setup()
|
||||
|
||||
def setup(self):
|
||||
pass
|
||||
|
|
@ -18,7 +18,7 @@ class DealStage(Enum):
|
|||
SURVEYED_NO_ACCESS_NEED_SIGN_OFF = "1617223915"
|
||||
CUSTOMER_CONTACTED = "888730834"
|
||||
SURVEYED_COMPLETED_SIGNED_OFF = "1617223916"
|
||||
NEEDS_ADDITIONAL_INFORMATION_FROM_ASSESSOR = "1887736000"
|
||||
FILES_MISSING_FROM_ASSESSOR = "1887736000"
|
||||
|
||||
class HubSpotClient():
|
||||
def __init__(self):
|
||||
|
|
@ -206,7 +206,7 @@ class HubSpotClient():
|
|||
after = response.paging.next.after
|
||||
|
||||
all_deals = []
|
||||
for deal in found_deals:
|
||||
for i,deal in enumerate(found_deals):
|
||||
domna_id, landlord_id, uprn = self.get_domna_and_landlord_id(deal.id)
|
||||
try:
|
||||
deal_name = deal.properties['dealname']
|
||||
|
|
@ -263,10 +263,8 @@ class HubSpotClient():
|
|||
self.add_note_to_deal(deal_id, format_error_note(e))
|
||||
else:
|
||||
self.logger.error(f"Non-validation error occurred: {str(e)}", exc_info=True)
|
||||
|
||||
|
||||
self.logger.info(f"Deal name <{deal_name}> moving to 'needs additional information'")
|
||||
self.move_deals_to_different_stage([deal_id], DealStage.NEEDS_ADDITIONAL_INFORMATION_FROM_ASSESSOR.value)
|
||||
self.move_deals_to_different_stage([deal_id], DealStage.FILES_MISSING_FROM_ASSESSOR.value)
|
||||
return all_deals
|
||||
|
||||
def print_all_pipeline_ids(self):
|
||||
|
|
|
|||
|
|
@ -60,14 +60,20 @@ class SubmissionInfoFromDeal(BaseModel):
|
|||
raise ValueError(f"Error accessing SharePoint path: {self.submission_folder_path}. Error: {str(e)}")
|
||||
|
||||
try:
|
||||
# Check if sharepoint link is reachable and has any contents
|
||||
files = sp.get_folders_in_path(path)
|
||||
if "value" in files and len(files["value"]) > 0:
|
||||
pass
|
||||
else:
|
||||
raise ValueError(f"SharePoint folder is empty: {self.submission_folder_path}")
|
||||
try:
|
||||
files = sp.get_folders_in_path(path)
|
||||
if files.get("value"):
|
||||
pass
|
||||
except Exception as e:
|
||||
print("Trying SGEC")
|
||||
sp = SharePointScraper(SharePointInstaller.SGEC)
|
||||
files = sp.get_folders_in_path(path)
|
||||
if files.get("value"):
|
||||
pass
|
||||
else:
|
||||
raise ValueError(f"[SharePoint Folder Empty] Folder has no contents after multiple attempts: {self.submission_folder_path}")
|
||||
except Exception as e:
|
||||
raise ValueError(str(e))
|
||||
raise ValueError(f"[Folder Access Error] {str(e)}")
|
||||
|
||||
# download files in url and check files are there:
|
||||
try:
|
||||
|
|
@ -80,8 +86,8 @@ class SubmissionInfoFromDeal(BaseModel):
|
|||
if sdp.condition_report is None:
|
||||
missing_items.append("Condition Report")
|
||||
|
||||
if sdp.epr_summary_information is None:
|
||||
missing_items.append("EPR Energy report with data is missing")
|
||||
if sdp.epr_with_data is None:
|
||||
missing_items.append("EPR Energy report with data")
|
||||
|
||||
if sdp.rd_sap_xml is None:
|
||||
missing_items.append("RDSAP XML")
|
||||
|
|
@ -90,7 +96,7 @@ class SubmissionInfoFromDeal(BaseModel):
|
|||
missing_items.append("LIG SAP XML")
|
||||
|
||||
if sdp.epr_summary_information is None:
|
||||
missing_items.append("EPR Summary information is missing")
|
||||
missing_items.append("EPR Summary information")
|
||||
|
||||
if missing_items:
|
||||
raise ValueError(f"Missing required items: {', '.join(missing_items)}")
|
||||
|
|
|
|||
|
|
@ -10,6 +10,8 @@ os.environ["SHAREPOINT_CLIENT_SECRET"] = "SOf8Q~-is4wdQiqvEEm9FlJQRAY9ELGaj5Qz-a
|
|||
os.environ["SHAREPOINT_TENANT_ID"] = "c3f7519c-2719-4547-af04-6da6cbfd8f8f"
|
||||
os.environ["SOUTH_COAST_INSULATION_SERVICE_SHAREPOINT_ID"] = "b5a51507-9427-4ee0-b03e-90ec7681e2d3"
|
||||
os.environ["JJC_SERVICE_SHAREPOINT_ID"] = "7fdd0485-bbf3-4b29-b30f-98c81c2a6284"
|
||||
os.environ["SGEC_SERVICE_SHAREPOINT_ID"] = "52018e5c-3215-4fe4-a4e3-bbf0d0aa7cd9"
|
||||
|
||||
|
||||
from etl.hubSpotClient.hubspot import DealStage, HubSpotClient
|
||||
# Local development
|
||||
|
|
@ -18,7 +20,7 @@ os.environ["DATABASE_URL"] = "postgresql://postgres:makingwarmhomes@db:5432/post
|
|||
hubspotClient = HubSpotClient()
|
||||
|
||||
# files missing from assessor column
|
||||
deals = hubspotClient.get_deals_from_deal_stage(DealStage.NEEDS_ADDITIONAL_INFORMATION_FROM_ASSESSOR)
|
||||
deals = hubspotClient.get_deals_from_deal_stage(DealStage.FILES_MISSING_FROM_ASSESSOR)
|
||||
|
||||
|
||||
for deal in deals:
|
||||
|
|
|
|||
236
etl/month_end_automation_wave_2_layout.py
Normal file
236
etl/month_end_automation_wave_2_layout.py
Normal file
|
|
@ -0,0 +1,236 @@
|
|||
# Wave 2's month end automation
|
||||
|
||||
from tqdm import tqdm
|
||||
from monday import MondayClient
|
||||
from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset_ids
|
||||
from pprint import pprint
|
||||
import pandas as pd
|
||||
import json
|
||||
|
||||
monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY"
|
||||
monday = MondayClient(monday_key)
|
||||
# NCHA SHDF Westville Wave 1 & 2
|
||||
board_ids = ["3900434153"]
|
||||
|
||||
rate_card_data = {
|
||||
"job_type": [
|
||||
"RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel",
|
||||
"Design Archetype", "Design Repetitive", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2",
|
||||
"Post EPR", "Post EPC", "Post ATT", "retrofit evaluation",
|
||||
"RA no show", "ATT no show", "post EPC no show", "Design Revision"
|
||||
],
|
||||
"rate": [
|
||||
207.65, 101, 186.4, 98, 98,
|
||||
450, 150, 163, 135, 120,
|
||||
"60 - Needs to be verified (Post EPR)", 45, 90.5, 40,
|
||||
25, 25, 25, "check with Kevin"
|
||||
]
|
||||
}
|
||||
|
||||
rate_card_df = pd.DataFrame(rate_card_data)
|
||||
|
||||
|
||||
for board in tqdm(board_ids):
|
||||
board_data = monday.boards.fetch_boards_by_id(board)
|
||||
columns = board_data["data"]["boards"][0]["columns"]
|
||||
col_id_map = {col["title"].lower(): col["id"] for col in columns}
|
||||
reversed_col_id_map = {v: k for k, v in col_id_map.items()}
|
||||
|
||||
|
||||
items = get_all_items(board, monday)
|
||||
|
||||
all_records = []
|
||||
for row in tqdm(items):
|
||||
data = {}
|
||||
data.update({"address": row['name']})
|
||||
data.update({"client": row['group']['title']})
|
||||
for col in row.get("column_values", []):
|
||||
if col.get("id") in reversed_col_id_map:
|
||||
if col.get("type") == "file":
|
||||
value = col.get("value")
|
||||
no_of_files = 0
|
||||
|
||||
if value:
|
||||
value = json.loads(col["value"])
|
||||
no_of_files = len(value.get('files', []))
|
||||
data.update({reversed_col_id_map[col.get("id")]: no_of_files})
|
||||
elif "no show" in reversed_col_id_map[col.get("id")]:
|
||||
def extract_number_from_text(text):
|
||||
number_str = ''
|
||||
|
||||
for char in text:
|
||||
if char.isnumeric():
|
||||
number_str += char
|
||||
elif number_str:
|
||||
break # stop once a number sequence ends
|
||||
|
||||
return int(number_str) if number_str else None
|
||||
text = col.get("text")
|
||||
if text is None:
|
||||
data.update({
|
||||
reversed_col_id_map[col.get("id")]: col.get("text")
|
||||
})
|
||||
else:
|
||||
data.update({
|
||||
reversed_col_id_map[col.get("id")]: extract_number_from_text(text)
|
||||
})
|
||||
else:
|
||||
data.update({
|
||||
reversed_col_id_map[col.get("id")]: col.get("text")
|
||||
})
|
||||
all_records.append(data)
|
||||
|
||||
# Convert to DataFrame
|
||||
df = pd.DataFrame(all_records)
|
||||
|
||||
filtered_dfs = []
|
||||
|
||||
# RA
|
||||
ra = df[
|
||||
df["ra"].str.lower().isin(["completed rdsap 10", "completed rdsap 9.9"])
|
||||
].copy()
|
||||
ra["job_type"] = "RA"
|
||||
filtered_dfs.append(ra)
|
||||
|
||||
|
||||
# ATT
|
||||
att = df[
|
||||
df["att"].str.lower().isin(["completed"])
|
||||
].copy()
|
||||
att["job_type"] = "ATT"
|
||||
filtered_dfs.append(att)
|
||||
|
||||
# V1 Coordination
|
||||
v1 = df[
|
||||
df["v1 coordination status"].str.lower().isin(["rc complete"])
|
||||
].copy()
|
||||
v1["job_type"] = "Coordination Stage 1 v1"
|
||||
filtered_dfs.append(v1)
|
||||
|
||||
# V2 Coordination
|
||||
_ = df[df["v2 invoiced"].fillna('').str.lower().isin(['to be invoiced'])]
|
||||
v2 = _[_["v2 dc/ima/pas"] > 0].copy()
|
||||
v2["job_type"] = "Coordination Stage 1 v2 remodel"
|
||||
filtered_dfs.append(v2)
|
||||
|
||||
# V3 Coordination
|
||||
v3 = df[
|
||||
df["v3 invoiced"].str.lower().isin(["to be invoiced"])
|
||||
].copy()
|
||||
v3["job_type"] = "Coordination Stage 1 v3 remodel"
|
||||
filtered_dfs.append(v3)
|
||||
|
||||
# Coordination stage 2 Please complete
|
||||
cors2 = df[
|
||||
df["rc stg. 2"].str.lower().isin(["to invoice"])
|
||||
].copy()
|
||||
cors2["job_type"] = "Coordination Stage 2"
|
||||
filtered_dfs.append(cors2)
|
||||
|
||||
# Design type archietype
|
||||
design1 = df[
|
||||
(df["design type for invoicing"].str.lower().isin(["archetype"])) & (df["design invoice status"].str.lower().isin(["to invoice"]))
|
||||
].copy()
|
||||
design1["job_type"] = "Design Archetype"
|
||||
filtered_dfs.append(design1)
|
||||
|
||||
# design type reptitive
|
||||
design1 = df[
|
||||
(df["design type for invoicing"].str.lower().isin(["repetitive"])) & df["design invoice status"].str.lower().isin(["to invoice"])
|
||||
].copy()
|
||||
design1["job_type"] = "Design Repetitive"
|
||||
filtered_dfs.append(design1)
|
||||
|
||||
# Design stage revisions
|
||||
design2 = df[
|
||||
df["design revision invoice status"].str.lower().isin(["to invoice"])
|
||||
].copy()
|
||||
design2["job_type"] = "Design Revision"
|
||||
filtered_dfs.append(design2)
|
||||
|
||||
# Lodgement Phase 1
|
||||
lodg1 = df[
|
||||
df["lodg. phase 1 invoice status"].str.lower().isin(["to invoice"])
|
||||
].copy()
|
||||
lodg1["job_type"] = "Lodgement phase 1"
|
||||
filtered_dfs.append(lodg1)
|
||||
|
||||
# Full Lodgement Phase
|
||||
lodg2 = df[
|
||||
df["full lodgement invoice status"].str.lower().isin(["to invoice"])
|
||||
].copy()
|
||||
lodg2["job_type"] = "Full Lodgement phase 2"
|
||||
filtered_dfs.append(lodg2)
|
||||
|
||||
# POST EPC
|
||||
post_epc = df[
|
||||
df["post-epc status"].str.lower().isin(["epc files uploaded"])
|
||||
].copy()
|
||||
post_epc["job_type"] = "Post EPC"
|
||||
filtered_dfs.append(post_epc)
|
||||
|
||||
|
||||
# POST EPR
|
||||
post_epr = df[
|
||||
df["post-epc status"].str.lower().isin(["post epr completed"])
|
||||
].copy()
|
||||
post_epr["job_type"] = "Post EPR"
|
||||
filtered_dfs.append(post_epr)
|
||||
|
||||
|
||||
|
||||
# Post ATT
|
||||
post_att = df[
|
||||
df["post-att"].str.lower().isin(["post-att uploaded"])
|
||||
].copy()
|
||||
post_att["job_type"] = "POST ATT"
|
||||
filtered_dfs.append(post_att)
|
||||
|
||||
|
||||
# Retrofit Evaluation
|
||||
retro = df[
|
||||
df["retrofit evaluation"].str.lower().isin(["complete"])
|
||||
].copy()
|
||||
retro["job_type"] = "Retrofit Evaluation"
|
||||
filtered_dfs.append(retro)
|
||||
|
||||
# RA NO Show
|
||||
ra_ns = df[
|
||||
(df["ra no show evidence"].fillna(-9999) != df["ra no show invoice"].fillna(-9999)) &
|
||||
(df["ra no show evidence"] != 0)
|
||||
].copy()
|
||||
ra_ns["job_type"] = "RA NO SHOW"
|
||||
filtered_dfs.append(ra_ns)
|
||||
|
||||
|
||||
|
||||
# ATT NO Show
|
||||
att_ns = df[
|
||||
(df["att no show evidence"].fillna(-9999) != df["att no show invoice"].fillna(-9999)) &
|
||||
(df["att no show evidence"] != 0)
|
||||
].copy()
|
||||
att_ns["job_type"] = "ATT NO SHOW"
|
||||
filtered_dfs.append(att_ns)
|
||||
|
||||
|
||||
# Post visit no show
|
||||
epc_ns = df[
|
||||
(df["epc no show evidence"].fillna(-9999) != df["epc no show invoice"].fillna(-9999)) &
|
||||
(df["epc no show evidence"] != 0)
|
||||
].copy()
|
||||
epc_ns["job_type"] = "post EPC NO SHOW"
|
||||
filtered_dfs.append(epc_ns)
|
||||
|
||||
final_df = pd.concat(filtered_dfs).reset_index(drop=True)
|
||||
|
||||
|
||||
final_df["job_type"] = final_df["job_type"].str.lower()
|
||||
rate_card_df["job_type"] = rate_card_df["job_type"].str.lower()
|
||||
|
||||
# Now perform the merge
|
||||
combined_with_rates = final_df.merge(rate_card_df, on="job_type", how="left")
|
||||
import datetime
|
||||
timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M')
|
||||
|
||||
attribute = ['address', 'client', 'job_type', 'rate']
|
||||
combined_with_rates[attribute].to_excel(f'NCHA SHDF Westville Wave 1 & 2_{timestamp}.xlsx', index=False)
|
||||
208
etl/month_end_automation_wave_2_no_10.py
Normal file
208
etl/month_end_automation_wave_2_no_10.py
Normal file
|
|
@ -0,0 +1,208 @@
|
|||
# Wave 2's month end automation
|
||||
from tqdm import tqdm
|
||||
from monday import MondayClient
|
||||
from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset_ids
|
||||
from pprint import pprint
|
||||
import pandas as pd
|
||||
import json
|
||||
|
||||
monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY"
|
||||
monday = MondayClient(monday_key)
|
||||
# NCHA Derbyshire Dales (DDDCC) SHDF
|
||||
board_ids = ["6947307148"]
|
||||
|
||||
rate_card_data = {
|
||||
"job_type": [
|
||||
"RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel",
|
||||
"Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2",
|
||||
"Post EPR", "Post EPC", "Post ATT", "retrofit evaluation",
|
||||
"RA no show", "ATT no show", "post EPC no show"
|
||||
],
|
||||
"rate": [
|
||||
259, 125, 280, 125, 125,
|
||||
650, 415, 195, 175, 135,
|
||||
120, "Post EPR Please Marianne", 85, 125, 60,
|
||||
25, 25, 25
|
||||
]
|
||||
}
|
||||
|
||||
rate_card_df = pd.DataFrame(rate_card_data)
|
||||
|
||||
|
||||
for board in tqdm(board_ids):
|
||||
board_data = monday.boards.fetch_boards_by_id(board)
|
||||
columns = board_data["data"]["boards"][0]["columns"]
|
||||
col_id_map = {col["title"].lower(): col["id"] for col in columns}
|
||||
reversed_col_id_map = {v: k for k, v in col_id_map.items()}
|
||||
|
||||
|
||||
items = get_all_items(board, monday)
|
||||
|
||||
all_records = []
|
||||
for row in tqdm(items):
|
||||
data = {}
|
||||
data.update({"address": row['name']})
|
||||
data.update({"client": row['group']['title']})
|
||||
for col in row.get("column_values", []):
|
||||
if col.get("id") in reversed_col_id_map:
|
||||
if col.get("type") == "file":
|
||||
value = col.get("value")
|
||||
no_of_files = 0
|
||||
|
||||
if value:
|
||||
value = json.loads(col["value"])
|
||||
no_of_files = len(value.get('files', []))
|
||||
data.update({reversed_col_id_map[col.get("id")]: no_of_files})
|
||||
elif "no show" in reversed_col_id_map[col.get("id")]:
|
||||
def extract_number_from_text(text):
|
||||
number_str = ''
|
||||
|
||||
for char in text:
|
||||
if char.isnumeric():
|
||||
number_str += char
|
||||
elif number_str:
|
||||
break # stop once a number sequence ends
|
||||
|
||||
return int(number_str) if number_str else None
|
||||
text = col.get("text")
|
||||
if text is None:
|
||||
data.update({
|
||||
reversed_col_id_map[col.get("id")]: col.get("text")
|
||||
})
|
||||
else:
|
||||
data.update({
|
||||
reversed_col_id_map[col.get("id")]: extract_number_from_text(text)
|
||||
})
|
||||
else:
|
||||
data.update({
|
||||
reversed_col_id_map[col.get("id")]: col.get("text")
|
||||
})
|
||||
all_records.append(data)
|
||||
|
||||
# Convert to DataFrame
|
||||
df = pd.DataFrame(all_records)
|
||||
|
||||
filtered_dfs = []
|
||||
|
||||
def get_df(df, column_name, success_critera, job_name):
|
||||
_ = df[
|
||||
df[column_name].str.lower().isin(success_critera)
|
||||
].copy()
|
||||
_["job_type"] = job_name
|
||||
return _
|
||||
|
||||
|
||||
# RA
|
||||
ra = get_df(df, "ra", ["completed rdsap 9.9", "completed rd sap 10"], "RA")
|
||||
filtered_dfs.append(ra)
|
||||
|
||||
|
||||
# ATT
|
||||
att = get_df(df, "att", ["completed"], "ATT")
|
||||
filtered_dfs.append(att)
|
||||
|
||||
# V1 Coordination
|
||||
v1 = get_df(df, "coordination status".lower(), [
|
||||
"rc complete",
|
||||
], "Coordination Stage 1 v1")
|
||||
filtered_dfs.append(v1)
|
||||
|
||||
# V2 Coordination
|
||||
# v2 = get_df(df, "v2 coordination status", ["rc v2 complete", "uploaded"], "V2 Coordination")
|
||||
# filtered_dfs.append(v2)
|
||||
|
||||
# # V3 Coordination
|
||||
# v3 = get_df(df, "")
|
||||
# # filtered_dfs.append(v3)
|
||||
|
||||
# Coordination stage 2 Please complete
|
||||
cors2 = df[
|
||||
df["rc stg. 2"].str.lower().isin(["to invoice"])
|
||||
]
|
||||
cors2["joby_type"] = "Coordination Stage 2"
|
||||
filtered_dfs.append(cors2)
|
||||
|
||||
# # Design stage 1
|
||||
# design1 = get_df(df, "design upload to sharepoint", ["done"], "Design")
|
||||
# filtered_dfs.append(design1)
|
||||
|
||||
# Design revision
|
||||
# design2 = get_df(df, "design revision invoice", [
|
||||
# "Rev. A to invoice".lower(),
|
||||
# "Rev. B to invoice".lower(),
|
||||
# "Rev. C to invoice".lower(),
|
||||
# "Rev. D to invoice".lower(),
|
||||
# ], "Design Revision")
|
||||
# filtered_dfs.append(design2)
|
||||
|
||||
# Lodgement Phase 1
|
||||
lodg1 = get_df(df, "Lodg. Phase 1 Invoice Status".lower(), ["to invoice"], "Lodgement Phase 1")
|
||||
filtered_dfs.append(lodg1)
|
||||
|
||||
|
||||
# Full Lodgement Phase
|
||||
full_lodgement = get_df(df, "full lodgement invoice status".lower(), ["to invoice"], "Full lodgement phase 2")
|
||||
filtered_dfs.append(full_lodgement)
|
||||
|
||||
# POST EPC
|
||||
post_epc = get_df(df, "lodged epc", ["complete"], "POST EPC")
|
||||
filtered_dfs.append(post_epc)
|
||||
|
||||
|
||||
# POST EPR
|
||||
post_epr = df[
|
||||
df["lodged epc"].str.lower().isin(["post epr completed"])
|
||||
].copy()
|
||||
post_epr["job_type"] = "Post EPR"
|
||||
filtered_dfs.append(post_epr)
|
||||
|
||||
# Post ATT
|
||||
post_att = get_df(df, "post att", ["done"], "POST ATT")
|
||||
filtered_dfs.append(post_att)
|
||||
|
||||
post_att = get_df(df, "post-test status", ["complete"], "POST ATT")
|
||||
filtered_dfs.append(post_att)
|
||||
|
||||
# Retrofit Evaluation
|
||||
retro = get_df(df, "retrofit evaluation", ["done"], "Retrofit Evaluation")
|
||||
filtered_dfs.append(retro)
|
||||
|
||||
# RA NO Show
|
||||
ra_ns = df[
|
||||
(df["ra no show evidence"].fillna(-9999) != df["ra no show invoice"].fillna(-9999)) &
|
||||
(df["ra no show evidence"] !=0 )
|
||||
].copy()
|
||||
ra_ns["job_type"] = "RA NO SHOW"
|
||||
filtered_dfs.append(ra_ns)
|
||||
|
||||
|
||||
# ATT NO Show
|
||||
att_ns = df[
|
||||
(df["att no show evidence"].fillna(-9999) != df["att no show invoice"].fillna(-9999)) &
|
||||
(df["att no show evidence"] != 0)
|
||||
].copy()
|
||||
att_ns["job_type"] = "ATT NO SHOW"
|
||||
filtered_dfs.append(att_ns)
|
||||
|
||||
|
||||
# Post visit no show
|
||||
# epc_ns = df[
|
||||
# df["post epc no show evidence"].fillna(-9999) != df["post epc no show invoice"].fillna(-9999)
|
||||
# ].copy()
|
||||
# epc_ns["job_type"] = "post EPC NO SHOW"
|
||||
# filtered_dfs.append(epc_ns)
|
||||
|
||||
final_df = pd.concat(filtered_dfs).reset_index(drop=True)
|
||||
|
||||
final_df[['address', 'client', 'job_type']]
|
||||
|
||||
final_df["job_type"] = final_df["job_type"].str.lower()
|
||||
rate_card_df["job_type"] = rate_card_df["job_type"].str.lower()
|
||||
|
||||
# Now perform the merge
|
||||
combined_with_rates = final_df.merge(rate_card_df, on="job_type", how="left")
|
||||
import datetime
|
||||
timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M')
|
||||
|
||||
attribute = ['address', 'client', 'job_type', 'rate']
|
||||
combined_with_rates[attribute].to_excel(f'NCHA Derbyshire Dales (DDDCC) SHDF_{timestamp}.xlsx', index=False)
|
||||
201
etl/month_end_automation_wave_2_no_11.py
Normal file
201
etl/month_end_automation_wave_2_no_11.py
Normal file
|
|
@ -0,0 +1,201 @@
|
|||
# Wave 2's month end automation
|
||||
from tqdm import tqdm
|
||||
from monday import MondayClient
|
||||
from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset_ids
|
||||
from pprint import pprint
|
||||
import pandas as pd
|
||||
import json
|
||||
|
||||
monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY"
|
||||
monday = MondayClient(monday_key)
|
||||
# Northumberland LAD2 & HUG2
|
||||
board_ids = ["5121300882"]
|
||||
|
||||
rate_card_data = {
|
||||
"job_type": [
|
||||
"RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel",
|
||||
"Design Archetype", "Design Repetitive", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2",
|
||||
"Post EPR", "Post EPC", "Post ATT", "retrofit evaluation",
|
||||
"RA no show", "ATT no show", "post EPC no show"
|
||||
],
|
||||
"rate": [
|
||||
259, 125, 280, 125, 125,
|
||||
650, 195, 175, 135,
|
||||
120, "Post EPR Please Marianne", 85, 125, 60,
|
||||
25, 25, 25
|
||||
]
|
||||
}
|
||||
|
||||
rate_card_df = pd.DataFrame(rate_card_data)
|
||||
|
||||
for board in tqdm(board_ids):
|
||||
board_data = monday.boards.fetch_boards_by_id(board)
|
||||
columns = board_data["data"]["boards"][0]["columns"]
|
||||
col_id_map = {col["title"].lower(): col["id"] for col in columns}
|
||||
reversed_col_id_map = {v: k for k, v in col_id_map.items()}
|
||||
|
||||
|
||||
items = get_all_items(board, monday)
|
||||
|
||||
all_records = []
|
||||
for row in tqdm(items):
|
||||
data = {}
|
||||
data.update({"address": row['name']})
|
||||
data.update({"client": row['group']['title']})
|
||||
for col in row.get("column_values", []):
|
||||
if col.get("id") in reversed_col_id_map:
|
||||
if col.get("type") == "file":
|
||||
value = col.get("value")
|
||||
no_of_files = 0
|
||||
|
||||
if value:
|
||||
value = json.loads(col["value"])
|
||||
no_of_files = len(value.get('files', []))
|
||||
data.update({reversed_col_id_map[col.get("id")]: no_of_files})
|
||||
elif "no show" in reversed_col_id_map[col.get("id")]:
|
||||
def extract_number_from_text(text):
|
||||
number_str = ''
|
||||
|
||||
for char in text:
|
||||
if char.isnumeric():
|
||||
number_str += char
|
||||
elif number_str:
|
||||
break # stop once a number sequence ends
|
||||
|
||||
return int(number_str) if number_str else None
|
||||
text = col.get("text")
|
||||
if text is None:
|
||||
data.update({
|
||||
reversed_col_id_map[col.get("id")]: col.get("text")
|
||||
})
|
||||
else:
|
||||
data.update({
|
||||
reversed_col_id_map[col.get("id")]: extract_number_from_text(text)
|
||||
})
|
||||
else:
|
||||
data.update({
|
||||
reversed_col_id_map[col.get("id")]: col.get("text")
|
||||
})
|
||||
all_records.append(data)
|
||||
|
||||
# Convert to DataFrame
|
||||
df = pd.DataFrame(all_records)
|
||||
|
||||
filtered_dfs = []
|
||||
|
||||
def get_df(df, column_name, success_critera, job_name=None):
|
||||
_ = df[
|
||||
df[column_name].str.lower().isin(success_critera)
|
||||
].copy()
|
||||
_["job_type"] = job_name
|
||||
return _
|
||||
|
||||
|
||||
# RA
|
||||
ra = get_df(df, "ra", ["completed rdsap 9.9", "completed rd sap 10"], "RA")
|
||||
filtered_dfs.append(ra)
|
||||
|
||||
|
||||
# ATT
|
||||
att = get_df(df, "att", ["completed"], "ATT")
|
||||
filtered_dfs.append(att)
|
||||
|
||||
# V1 Coordination
|
||||
v1 = get_df(df, "lite ima status".lower(), [
|
||||
"rc complete",
|
||||
], "Coordination Stage 1 v1")
|
||||
filtered_dfs.append(v1)
|
||||
|
||||
# V2 Coordination
|
||||
v2 = get_df(df, "ima-mtp status", ["ima-mtp completed"], "Coordination Stage 1 v2 remodel")
|
||||
filtered_dfs.append(v2)
|
||||
|
||||
# # V3 Coordination
|
||||
# v3 = get_df(df, "")
|
||||
# filtered_dfs.append(v3)
|
||||
|
||||
# Coordination stage 2 Please complete
|
||||
cors2 = df[
|
||||
df["rc stg. 2"].str.lower().isin(["to invoice"])
|
||||
]
|
||||
cors2["joby_type"] = "Coordination Stage 2"
|
||||
filtered_dfs.append(cors2)
|
||||
|
||||
# Design stage 1
|
||||
# design1 = get_df(df, "", ["done"], "Design")
|
||||
# filtered_dfs.append(design1)
|
||||
|
||||
# Design revision
|
||||
# design2 = get_df(df, "design revision invoice", [
|
||||
# "Rev. A to invoice".lower(),
|
||||
# "Rev. B to invoice".lower(),
|
||||
# "Rev. C to invoice".lower(),
|
||||
# "Rev. D to invoice".lower(),
|
||||
# ], "Design Revision")
|
||||
# filtered_dfs.append(design2)
|
||||
|
||||
# Lodgement Phase 1
|
||||
lodg1 = get_df(df, "tm phase 1 invoiced".lower(), ["to invoice"], "Lodgement Phase 1")
|
||||
filtered_dfs.append(lodg1)
|
||||
|
||||
|
||||
# Full Lodgement Phase
|
||||
full_lodgement = get_df(df, "trustmark lodgement".lower(), ["done"], "Full lodgement phase 2")
|
||||
filtered_dfs.append(full_lodgement)
|
||||
|
||||
# POST EPC
|
||||
post_epc = get_df(df, "post-epc status", ["uploaded & completed", "to invoice"], "POST EPC")
|
||||
filtered_dfs.append(post_epc)
|
||||
|
||||
|
||||
# POST EPR
|
||||
post_epr = df[
|
||||
df["post-epc status"].str.lower().isin(["post epr completed"])
|
||||
].copy()
|
||||
post_epr["job_type"] = "POST epr"
|
||||
filtered_dfs.append(post_epr)
|
||||
|
||||
# Post ATT
|
||||
post_att = get_df(df, "post att status", ["uploaded & completed", "to invoice"], "POST ATT")
|
||||
filtered_dfs.append(post_att)
|
||||
|
||||
# Retrofit Evaluation
|
||||
retro = get_df(df, "retrofit evaluation", ["done", "to invoice"], "Retrofit Evaluation")
|
||||
filtered_dfs.append(retro)
|
||||
|
||||
# RA NO Show
|
||||
# ra_ns = df[
|
||||
# df["ra no show evidence"].fillna(-9999) != df["ra no show invoice"].fillna(-9999)
|
||||
# ].copy()
|
||||
# ra_ns["job_type"] = "RA NO SHOW"
|
||||
# filtered_dfs.append(ra_ns)
|
||||
|
||||
|
||||
# ATT NO Show
|
||||
att_ns = df[
|
||||
(df["att no show evidence"].fillna(-9999) != df["att no show invoice"].fillna(-9999)) &
|
||||
(df["att no show evidence"] != 0)
|
||||
].copy()
|
||||
att_ns["job_type"] = "ATT NO SHOW"
|
||||
filtered_dfs.append(att_ns)
|
||||
|
||||
|
||||
# Post visit no show
|
||||
# epc_ns = df[
|
||||
# df["post epc no show evidence"].fillna(-9999) != df["post epc no show invoice"].fillna(-9999)
|
||||
# ].copy()
|
||||
# epc_ns["job_type"] = "post EPC NO SHOW"
|
||||
# filtered_dfs.append(epc_ns)
|
||||
|
||||
final_df = pd.concat(filtered_dfs).reset_index(drop=True)
|
||||
|
||||
final_df["job_type"] = final_df["job_type"].str.lower()
|
||||
rate_card_df["job_type"] = rate_card_df["job_type"].str.lower()
|
||||
|
||||
# Now perform the merge
|
||||
combined_with_rates = final_df.merge(rate_card_df, on="job_type", how="left")
|
||||
import datetime
|
||||
timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M')
|
||||
|
||||
attribute = ['address', 'client', 'job_type', 'rate']
|
||||
combined_with_rates[attribute].to_excel(f'Northumberland LAD2 & HUG2_{timestamp}.xlsx', index=False)
|
||||
207
etl/month_end_automation_wave_2_no_12.py
Normal file
207
etl/month_end_automation_wave_2_no_12.py
Normal file
|
|
@ -0,0 +1,207 @@
|
|||
# Wave 2's month end automation
|
||||
from tqdm import tqdm
|
||||
from monday import MondayClient
|
||||
from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset_ids
|
||||
from pprint import pprint
|
||||
import pandas as pd
|
||||
import json
|
||||
|
||||
monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY"
|
||||
monday = MondayClient(monday_key)
|
||||
# Shropshire Council HUG2
|
||||
board_ids = ["4718185486"]
|
||||
|
||||
empty = "nothing on rate card"
|
||||
|
||||
rate_card_data = {
|
||||
"job_type": [
|
||||
"RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel",
|
||||
"Design Archetype", "Design Repetitive", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2",
|
||||
"Post EPR", "Post EPC", "Post ATT", "retrofit evaluation",
|
||||
"RA no show", "ATT no show", "post EPC no show", "Design Sign off"
|
||||
],
|
||||
"rate": [
|
||||
"(185) - Kevin Check as this depends on property size", 115, 200, 125, 125,
|
||||
empty, "(135)- Mariane said 'Remaining RC & design sign off", 185, 135,
|
||||
120, "(60) Post EPR, please verify", 65, 115, 60,
|
||||
50, 50, 50, "(60) - Please add price for design sign off, check with kevin and marianne"
|
||||
]
|
||||
}
|
||||
|
||||
rate_card_df = pd.DataFrame(rate_card_data)
|
||||
|
||||
for board in tqdm(board_ids):
|
||||
board_data = monday.boards.fetch_boards_by_id(board)
|
||||
columns = board_data["data"]["boards"][0]["columns"]
|
||||
col_id_map = {col["title"].lower(): col["id"] for col in columns}
|
||||
reversed_col_id_map = {v: k for k, v in col_id_map.items()}
|
||||
|
||||
|
||||
items = get_all_items(board, monday)
|
||||
|
||||
all_records = []
|
||||
for row in tqdm(items):
|
||||
data = {}
|
||||
data.update({"address": row['name']})
|
||||
data.update({"client": row['group']['title']})
|
||||
for col in row.get("column_values", []):
|
||||
if col.get("id") in reversed_col_id_map:
|
||||
if col.get("type") == "file":
|
||||
value = col.get("value")
|
||||
no_of_files = 0
|
||||
|
||||
if value:
|
||||
value = json.loads(col["value"])
|
||||
no_of_files = len(value.get('files', []))
|
||||
data.update({reversed_col_id_map[col.get("id")]: no_of_files})
|
||||
elif "no show" in reversed_col_id_map[col.get("id")]:
|
||||
def extract_number_from_text(text):
|
||||
number_str = ''
|
||||
|
||||
for char in text:
|
||||
if char.isnumeric():
|
||||
number_str += char
|
||||
elif number_str:
|
||||
break # stop once a number sequence ends
|
||||
|
||||
return int(number_str) if number_str else None
|
||||
text = col.get("text")
|
||||
if text is None:
|
||||
data.update({
|
||||
reversed_col_id_map[col.get("id")]: col.get("text")
|
||||
})
|
||||
else:
|
||||
data.update({
|
||||
reversed_col_id_map[col.get("id")]: extract_number_from_text(text)
|
||||
})
|
||||
else:
|
||||
data.update({
|
||||
reversed_col_id_map[col.get("id")]: col.get("text")
|
||||
})
|
||||
all_records.append(data)
|
||||
|
||||
# Convert to DataFrame
|
||||
df = pd.DataFrame(all_records)
|
||||
|
||||
filtered_dfs = []
|
||||
|
||||
def get_df(df, column_name, success_critera, job_name):
|
||||
_ = df[
|
||||
df[column_name].str.lower().isin(success_critera)
|
||||
].copy()
|
||||
_["job_type"] = job_name
|
||||
return _
|
||||
|
||||
|
||||
# RA
|
||||
ra = get_df(df, "ra", ["completed rdsap 9.9", "completed rd sap 10"], "RA")
|
||||
filtered_dfs.append(ra)
|
||||
|
||||
|
||||
# ATT
|
||||
att = get_df(df, "pre- att", ["completed"], "ATT")
|
||||
filtered_dfs.append(att)
|
||||
|
||||
# V1 Coordination
|
||||
v1 = get_df(df, "ima lite invoiced".lower(), [
|
||||
"to invoice",
|
||||
], "Coordination Stage 1 v1")
|
||||
filtered_dfs.append(v1)
|
||||
|
||||
# V2 Coordination
|
||||
v2 = get_df(df, "coordination v2 invoiced", ["to invoice"], "Coordination Stage 1 v2 remodel")
|
||||
filtered_dfs.append(v2)
|
||||
|
||||
# # V3 Coordination
|
||||
# v3 = get_df(df, "")
|
||||
# filtered_dfs.append(v3)
|
||||
|
||||
# Coordination stage 2 Please complete
|
||||
cors2 = df[
|
||||
df["rc stage 2 invoice"].str.lower().isin(["to invoice"])
|
||||
].copy()
|
||||
cors2["joby_type"] = "Coordination Stage 2"
|
||||
filtered_dfs.append(cors2)
|
||||
|
||||
# Design stage 1
|
||||
# design1 = get_df(df, "", ["done"], "Design")
|
||||
# filtered_dfs.append(design1)
|
||||
|
||||
# Design revision
|
||||
# design2 = get_df(df, "design revision invoice", [
|
||||
# "Rev. A to invoice".lower(),
|
||||
# "Rev. B to invoice".lower(),
|
||||
# "Rev. C to invoice".lower(),
|
||||
# "Rev. D to invoice".lower(),
|
||||
# ], "Design Revision")
|
||||
# filtered_dfs.append(design2)
|
||||
|
||||
# Design sign off
|
||||
design_sign_pff = get_df(df, "design payment step 3", ["ready to invoice"], "design sign off")
|
||||
|
||||
# Lodgement Phase 1
|
||||
lodg1 = get_df(df, "tm ph1 invoice status".lower(), ["to invoice"], "Lodgement Phase 1")
|
||||
filtered_dfs.append(lodg1)
|
||||
|
||||
|
||||
# Full Lodgement Phase
|
||||
full_lodgement = get_df(df, "tm ph2 invoice status".lower(), ["to invoice"], "Full lodgement phase 2")
|
||||
filtered_dfs.append(full_lodgement)
|
||||
|
||||
# POST EPC
|
||||
post_epc = get_df(df, "post-epc status", ["uploaded", "completed"], "POST EPC")
|
||||
filtered_dfs.append(post_epc)
|
||||
|
||||
|
||||
# POST EPR
|
||||
post_epr = df[
|
||||
df["post-epc status"].str.lower().isin(["post epr completed"])
|
||||
].copy()
|
||||
post_epr["job_type"] = "POST epr"
|
||||
filtered_dfs.append(post_epr)
|
||||
|
||||
# Post ATT
|
||||
post_att = get_df(df, "post-att", ["uploaded", "completed", "to invoice"], "POST ATT")
|
||||
filtered_dfs.append(post_att)
|
||||
|
||||
# Retrofit Evaluation
|
||||
retro = get_df(df, "retrofit evaluation", ["completed", "to invoice"], "Retrofit Evaluation")
|
||||
filtered_dfs.append(retro)
|
||||
|
||||
# RA NO Show
|
||||
ra_ns = df[
|
||||
(df["ra no show evidence"].fillna(-9999) != df["ra no show invoice"].fillna(-9999)) &
|
||||
(df["ra no show evidence"] != 0)
|
||||
].copy()
|
||||
ra_ns["job_type"] = "RA NO SHOW"
|
||||
filtered_dfs.append(ra_ns)
|
||||
|
||||
|
||||
# ATT NO Show
|
||||
# att_ns = df[
|
||||
# df["att no show evidence"].fillna(-9999) != df["att no show invoice"].fillna(-9999)
|
||||
# ].copy()
|
||||
# att_ns["job_type"] = "ATT NO SHOW"
|
||||
# filtered_dfs.append(att_ns)
|
||||
|
||||
|
||||
# Post visit no show
|
||||
epc_ns = df[
|
||||
(df["epc no show evidence"].fillna(-9999) != df["epc no show invoice"].fillna(-9999)) &
|
||||
(df["epc no show evidence"] != 0)
|
||||
].copy()
|
||||
epc_ns["job_type"] = "post EPC NO SHOW"
|
||||
filtered_dfs.append(epc_ns)
|
||||
|
||||
final_df = pd.concat(filtered_dfs).reset_index(drop=True)
|
||||
|
||||
final_df["job_type"] = final_df["job_type"].str.lower()
|
||||
rate_card_df["job_type"] = rate_card_df["job_type"].str.lower()
|
||||
|
||||
# Now perform the merge
|
||||
combined_with_rates = final_df.merge(rate_card_df, on="job_type", how="left")
|
||||
import datetime
|
||||
timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M')
|
||||
|
||||
attribute = ['address', 'client', 'job_type', 'rate']
|
||||
combined_with_rates[attribute].to_excel(f'Shropshire Council HUG2_{timestamp}.xlsx', index=False)
|
||||
200
etl/month_end_automation_wave_2_no_13.py
Normal file
200
etl/month_end_automation_wave_2_no_13.py
Normal file
|
|
@ -0,0 +1,200 @@
|
|||
# Wave 2's month end automation
|
||||
from tqdm import tqdm
|
||||
from monday import MondayClient
|
||||
from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset_ids
|
||||
from pprint import pprint
|
||||
import pandas as pd
|
||||
import json
|
||||
|
||||
monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY"
|
||||
monday = MondayClient(monday_key)
|
||||
# Stonewater SHDF 3.0 - Operations
|
||||
board_ids = ["6222522864"]
|
||||
|
||||
rate_card_data = {
|
||||
"job_type": [
|
||||
"RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel",
|
||||
"Design Archetype", "Design Repetitive", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2",
|
||||
"Post EPR", "Post EPC", "Post ATT", "retrofit evaluation",
|
||||
"RA no show", "ATT no show", "post EPC no show"
|
||||
],
|
||||
"rate": [
|
||||
259, 125, 280, 125, 125,
|
||||
650, 195, 175, 135,
|
||||
120, "Post EPR Please Marianne", 85, 125, 60,
|
||||
25, 25, 25
|
||||
]
|
||||
}
|
||||
|
||||
rate_card_df = pd.DataFrame(rate_card_data)
|
||||
|
||||
for board in tqdm(board_ids):
|
||||
board_data = monday.boards.fetch_boards_by_id(board)
|
||||
columns = board_data["data"]["boards"][0]["columns"]
|
||||
col_id_map = {col["title"].lower(): col["id"] for col in columns}
|
||||
reversed_col_id_map = {v: k for k, v in col_id_map.items()}
|
||||
|
||||
|
||||
items = get_all_items(board, monday)
|
||||
|
||||
all_records = []
|
||||
for row in tqdm(items):
|
||||
data = {}
|
||||
data.update({"address": row['name']})
|
||||
data.update({"client": row['group']['title']})
|
||||
for col in row.get("column_values", []):
|
||||
if col.get("id") in reversed_col_id_map:
|
||||
if col.get("type") == "file":
|
||||
value = col.get("value")
|
||||
no_of_files = 0
|
||||
|
||||
if value:
|
||||
value = json.loads(col["value"])
|
||||
no_of_files = len(value.get('files', []))
|
||||
data.update({reversed_col_id_map[col.get("id")]: no_of_files})
|
||||
elif "no show" in reversed_col_id_map[col.get("id")]:
|
||||
def extract_number_from_text(text):
|
||||
number_str = ''
|
||||
|
||||
for char in text:
|
||||
if char.isnumeric():
|
||||
number_str += char
|
||||
elif number_str:
|
||||
break # stop once a number sequence ends
|
||||
|
||||
return int(number_str) if number_str else None
|
||||
text = col.get("text")
|
||||
if text is None:
|
||||
data.update({
|
||||
reversed_col_id_map[col.get("id")]: col.get("text")
|
||||
})
|
||||
else:
|
||||
data.update({
|
||||
reversed_col_id_map[col.get("id")]: extract_number_from_text(text)
|
||||
})
|
||||
else:
|
||||
data.update({
|
||||
reversed_col_id_map[col.get("id")]: col.get("text")
|
||||
})
|
||||
all_records.append(data)
|
||||
|
||||
# Convert to DataFrame
|
||||
df = pd.DataFrame(all_records)
|
||||
|
||||
filtered_dfs = []
|
||||
|
||||
def get_df(df, column_name, success_critera, job_name):
|
||||
_ = df[
|
||||
df[column_name].str.lower().isin(success_critera)
|
||||
].copy()
|
||||
_["job_type"] = job_name
|
||||
return _
|
||||
|
||||
# RA
|
||||
ra = get_df(df, "ra", ["completed rdsap 9.9", "completed rdsap 10"], "RA")
|
||||
filtered_dfs.append(ra)
|
||||
|
||||
|
||||
# ATT
|
||||
att = get_df(df, "att", ["completed"], "ATT")
|
||||
filtered_dfs.append(att)
|
||||
|
||||
# V1 Coordination
|
||||
v1 = get_df(df, "coordination status (mtp)".lower(), [
|
||||
"ima/mtp complete",
|
||||
], "Coordination Stage 1 v1")
|
||||
filtered_dfs.append(v1)
|
||||
|
||||
# V2 Coordination
|
||||
v2 = get_df(df, "v2 mtp status", ["ima/mtp complete"], "Coordination Stage 1 v2 remodel")
|
||||
filtered_dfs.append(v2)
|
||||
|
||||
# # V3 Coordination
|
||||
# v3 = get_df(df, "")
|
||||
# filtered_dfs.append(v3)
|
||||
|
||||
# Coordination stage 2 Please complete
|
||||
cors2 = df[
|
||||
df["rc stage 2"].str.lower().isin(["to invoice"])
|
||||
]
|
||||
cors2["joby_type"] = "Coordination Stage 2"
|
||||
filtered_dfs.append(cors2)
|
||||
|
||||
# Design stage 1
|
||||
# design1 = get_df(df, "", ["done"], "Design")
|
||||
# filtered_dfs.append(design1)
|
||||
|
||||
# Design revision
|
||||
# design2 = get_df(df, "design revision invoice", [
|
||||
# "Rev. A to invoice".lower(),
|
||||
# "Rev. B to invoice".lower(),
|
||||
# "Rev. C to invoice".lower(),
|
||||
# "Rev. D to invoice".lower(),
|
||||
# ], "Design Revision")
|
||||
# filtered_dfs.append(design2)
|
||||
|
||||
# Lodgement Phase 1
|
||||
# lodg1 = get_df(df, "tm ph1 invoice status".lower(), ["to invoice"], "Lodgement Phase 1")
|
||||
# filtered_dfs.append(lodg1)
|
||||
|
||||
|
||||
# Full Lodgement Phase
|
||||
# full_lodgement = get_df(df, "tm ph2 invoice status".lower(), ["to invoice"], "Full lodgement phase 2")
|
||||
# filtered_dfs.append(full_lodgement)
|
||||
|
||||
# POST EPC
|
||||
post_epc = get_df(df, "post epc", ["done"], "POST EPC")
|
||||
filtered_dfs.append(post_epc)
|
||||
|
||||
|
||||
# POST EPR
|
||||
post_epr = df[
|
||||
df["post epc"].str.lower().isin(["post epr completed"])
|
||||
].copy()
|
||||
post_epr["job_type"] = "POST EPR"
|
||||
filtered_dfs.append(post_epr)
|
||||
|
||||
# Post ATT
|
||||
# post_att = get_df(df, "post-att", ["uploaded", "completed", "to invoice"], "POST ATT")
|
||||
# filtered_dfs.append(post_att)
|
||||
|
||||
# Retrofit Evaluation
|
||||
retro = get_df(df, "retrofit evaluation", ["done"], "Retrofit Evaluation")
|
||||
filtered_dfs.append(retro)
|
||||
|
||||
# RA NO Show
|
||||
ra_ns = df[
|
||||
(df["ra no show evidence"].fillna(-9999) != df["ra no show invoice"].fillna(-9999)) &
|
||||
(df["ra no show evidence"] != 0 )
|
||||
].copy()
|
||||
ra_ns["job_type"] = "RA NO SHOW"
|
||||
filtered_dfs.append(ra_ns)
|
||||
|
||||
|
||||
# ATT NO Show
|
||||
# att_ns = df[
|
||||
# df["att no show evidence"].fillna(-9999) != df["att no show invoice"].fillna(-9999)
|
||||
# ].copy()
|
||||
# att_ns["job_type"] = "ATT NO SHOW"
|
||||
# filtered_dfs.append(att_ns)
|
||||
|
||||
|
||||
# Post visit no show
|
||||
# epc_ns = df[
|
||||
# df["epc no show evidence"].fillna(-9999) != df["epc no show invoice"].fillna(-9999)
|
||||
# ].copy()
|
||||
# epc_ns["job_type"] = "post EPC NO SHOW"
|
||||
# filtered_dfs.append(epc_ns)
|
||||
|
||||
final_df = pd.concat(filtered_dfs).reset_index(drop=True)
|
||||
|
||||
final_df["job_type"] = final_df["job_type"].str.lower()
|
||||
rate_card_df["job_type"] = rate_card_df["job_type"].str.lower()
|
||||
|
||||
# Now perform the merge
|
||||
combined_with_rates = final_df.merge(rate_card_df, on="job_type", how="left")
|
||||
import datetime
|
||||
timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M')
|
||||
|
||||
attribute = ['address', 'client', 'job_type', 'rate']
|
||||
combined_with_rates[attribute].to_excel(f'Stonewater SHDF 3.0 - Operations_{timestamp}.xlsx', index=False)
|
||||
196
etl/month_end_automation_wave_2_no_14.py
Normal file
196
etl/month_end_automation_wave_2_no_14.py
Normal file
|
|
@ -0,0 +1,196 @@
|
|||
# Wave 2's month end automation
|
||||
from tqdm import tqdm
|
||||
from monday import MondayClient
|
||||
from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset_ids
|
||||
from pprint import pprint
|
||||
import pandas as pd
|
||||
import json
|
||||
|
||||
monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY"
|
||||
monday = MondayClient(monday_key)
|
||||
# Decent Homes Stonewater - Operations
|
||||
board_ids = ["9319118237"]
|
||||
|
||||
empty = "Rate card info missing"
|
||||
rate_card_data_example = {
|
||||
"job_type": [
|
||||
"RA", "ATT", "Coordination Stage 1 v1",
|
||||
"Lodgement phase 1", "Full lodgement phase 2",
|
||||
"Post EPR", "Post EPC", "Post ATT", "retrofit evaluation",
|
||||
"RA no show", "ATT no show", "post EPC no show"
|
||||
],
|
||||
"rate": [
|
||||
259, 125, 280, 125,
|
||||
260, 126, 281, 126,
|
||||
262, 127, 282, 127,
|
||||
]
|
||||
}
|
||||
|
||||
|
||||
rate_card_df = pd.DataFrame(rate_card_data_example)
|
||||
|
||||
for board in tqdm(board_ids):
|
||||
board_data = monday.boards.fetch_boards_by_id(board)
|
||||
columns = board_data["data"]["boards"][0]["columns"]
|
||||
col_id_map = {col["title"].lower(): col["id"] for col in columns}
|
||||
reversed_col_id_map = {v: k for k, v in col_id_map.items()}
|
||||
|
||||
|
||||
items = get_all_items(board, monday)
|
||||
|
||||
all_records = []
|
||||
for row in tqdm(items):
|
||||
data = {}
|
||||
data.update({"address": row['name']})
|
||||
data.update({"client": row['group']['title']})
|
||||
for col in row.get("column_values", []):
|
||||
if col.get("id") in reversed_col_id_map:
|
||||
if col.get("type") == "file":
|
||||
value = col.get("value")
|
||||
no_of_files = 0
|
||||
|
||||
if value:
|
||||
value = json.loads(col["value"])
|
||||
no_of_files = len(value.get('files', []))
|
||||
data.update({reversed_col_id_map[col.get("id")]: no_of_files})
|
||||
elif "no show" in reversed_col_id_map[col.get("id")]:
|
||||
def extract_number_from_text(text):
|
||||
number_str = ''
|
||||
|
||||
for char in text:
|
||||
if char.isnumeric():
|
||||
number_str += char
|
||||
elif number_str:
|
||||
break # stop once a number sequence ends
|
||||
|
||||
return int(number_str) if number_str else None
|
||||
text = col.get("text")
|
||||
if text is None:
|
||||
data.update({
|
||||
reversed_col_id_map[col.get("id")]: col.get("text")
|
||||
})
|
||||
else:
|
||||
data.update({
|
||||
reversed_col_id_map[col.get("id")]: extract_number_from_text(text)
|
||||
})
|
||||
else:
|
||||
data.update({
|
||||
reversed_col_id_map[col.get("id")]: col.get("text")
|
||||
})
|
||||
all_records.append(data)
|
||||
|
||||
# Convert to DataFrame
|
||||
df = pd.DataFrame(all_records)
|
||||
|
||||
filtered_dfs = []
|
||||
|
||||
def get_df(df, column_name, success_critera, job_name):
|
||||
_ = df[
|
||||
df[column_name].str.lower().isin(success_critera)
|
||||
].copy()
|
||||
_["job_type"] = job_name
|
||||
return _
|
||||
|
||||
# RA
|
||||
ra = get_df(df, "ra", ["completed rdsap 9.9", "completed rdsap 10"], "RA")
|
||||
filtered_dfs.append(ra)
|
||||
|
||||
|
||||
# ATT
|
||||
att = get_df(df, "att", ["completed"], "ATT")
|
||||
filtered_dfs.append(att)
|
||||
|
||||
# V1 Coordination
|
||||
v1 = get_df(df, "v1 coordination status (ioe,mtp)".lower(), [
|
||||
"rc complete",
|
||||
], "Coordination Stage 1 v1")
|
||||
filtered_dfs.append(v1)
|
||||
|
||||
# # V2 Coordination
|
||||
# v2 = get_df(df, "mtp v2 status", ["rc v2 complete"], "V2 Coordination")
|
||||
# filtered_dfs.append(v2)
|
||||
|
||||
# # # V3 Coordination
|
||||
# v3 = get_df(df, "v3 rc status", ["uploaded"], "V3 Coordination")
|
||||
# filtered_dfs.append(v3)
|
||||
|
||||
# v3 = get_df(df, "v3 invoice status", ["to be invoice"], "V3 Coordination")
|
||||
# filtered_dfs.append(v3)
|
||||
|
||||
# Coordination stage 2 Please complete
|
||||
# cors2 = df[
|
||||
# df["rc stg. 2"].str.lower().isin(["to invoice", "completed"])
|
||||
# ]
|
||||
# cors2["joby_type"] = "Coordination Stage 2"
|
||||
# filtered_dfs.append(cors2)
|
||||
|
||||
# Design stage 1
|
||||
# design1 = get_df(df, "design invoice status", ["to invoice"], "Design")
|
||||
# filtered_dfs.append(design1)
|
||||
|
||||
# Design revision
|
||||
# design2 = get_df(df, "design revision invoice", [
|
||||
# "Rev. A to invoice".lower(),
|
||||
# "Rev. B to invoice".lower(),
|
||||
# "Rev. C to invoice".lower(),
|
||||
# "Rev. D to invoice".lower(),
|
||||
# ], "Design Revision")
|
||||
# filtered_dfs.append(design2)
|
||||
|
||||
# Lodgement Phase 1
|
||||
lodg1 = get_df(df, "phase 1 invoice status (lodgement)".lower(), ["to be invoice"], "Lodgement Phase 1")
|
||||
filtered_dfs.append(lodg1)
|
||||
|
||||
|
||||
# Full Lodgement Phase
|
||||
full_lodgement = get_df(df, "lodgement invoice status".lower(), ["to invoice"], "Full lodgement phase 2")
|
||||
filtered_dfs.append(full_lodgement)
|
||||
|
||||
# POST EPC
|
||||
post_epc = get_df(df, "post epc", ["completed & uploaded"], "POST EPC")
|
||||
filtered_dfs.append(post_epc)
|
||||
|
||||
|
||||
# POST EPR
|
||||
post_epr = df[
|
||||
df["post epc"].str.lower().isin(["post epr completed"])
|
||||
].copy()
|
||||
post_epr["job_type"] = "POST epr"
|
||||
filtered_dfs.append(post_epr)
|
||||
|
||||
# Post ATT
|
||||
post_att = get_df(df, "post att", ["completed & uploaded"], "POST ATT")
|
||||
filtered_dfs.append(post_att)
|
||||
|
||||
# Retrofit Evaluation
|
||||
retro = get_df(df, "retrofit evaluation", ["completed & uploaded"], "Retrofit Evaluation")
|
||||
filtered_dfs.append(retro)
|
||||
|
||||
# RA NO Show
|
||||
ra_ns = df[
|
||||
(df["ra no show evidence"].fillna(-9999) != df["ra no show invoice"].fillna(-9999)) &
|
||||
(df["ra no show evidence"] != 0)
|
||||
].copy()
|
||||
ra_ns["job_type"] = "RA NO SHOW"
|
||||
filtered_dfs.append(ra_ns)
|
||||
|
||||
|
||||
# ATT NO Show
|
||||
# att_ns = df[
|
||||
# df["att no show evidence"].fillna(-9999) != df["att no show invoice"].fillna(-9999)
|
||||
# ].copy()
|
||||
# att_ns["job_type"] = "ATT NO SHOW"
|
||||
# filtered_dfs.append(att_ns)
|
||||
|
||||
|
||||
# Post visit no show
|
||||
epc_ns = df[
|
||||
(df["post epc no show evidence"].fillna(-9999) != df["post epc no show invoice"].fillna(-9999)) &
|
||||
(df["post epc no show evidence"] != 0)
|
||||
].copy()
|
||||
epc_ns["job_type"] = "post EPC NO SHOW"
|
||||
filtered_dfs.append(epc_ns)
|
||||
|
||||
final_df = pd.concat(filtered_dfs).reset_index(drop=True)
|
||||
|
||||
final_df[['address', 'client', 'job_type']]
|
||||
214
etl/month_end_automation_wave_2_no_15.py
Normal file
214
etl/month_end_automation_wave_2_no_15.py
Normal file
|
|
@ -0,0 +1,214 @@
|
|||
# Wave 2's month end automation
|
||||
from tqdm import tqdm
|
||||
from monday import MondayClient
|
||||
from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset_ids
|
||||
from pprint import pprint
|
||||
import pandas as pd
|
||||
import json
|
||||
|
||||
monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY"
|
||||
monday = MondayClient(monday_key)
|
||||
# WCHG SHDF 2.1 Mansard
|
||||
board_ids = ["5636990610"]
|
||||
|
||||
|
||||
empty = "Rate card was empty"
|
||||
|
||||
rate_card_data = {
|
||||
"job_type": [
|
||||
"RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel",
|
||||
"Design Archetype", "Design Repetitive", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2",
|
||||
"Post EPR", "Post EPC", "Post ATT", "retrofit evaluation",
|
||||
"RA no show", "ATT no show", "post EPC no show", "Design Revision"
|
||||
],
|
||||
"rate": [
|
||||
"259 (new RA rate for PAS2035:2023 - old rates for other works - to discuss with KN)", 40, 178.5, empty, empty,
|
||||
empty, 180, 275, 135,
|
||||
120, "60 - please verify with Marianne", 45, 45, 40,
|
||||
25, 25, 25, "Please check price for design revision with Andreas"
|
||||
]
|
||||
}
|
||||
|
||||
rate_card_df = pd.DataFrame(rate_card_data)
|
||||
|
||||
|
||||
for board in tqdm(board_ids):
|
||||
board_data = monday.boards.fetch_boards_by_id(board)
|
||||
columns = board_data["data"]["boards"][0]["columns"]
|
||||
col_id_map = {col["title"].lower(): col["id"] for col in columns}
|
||||
reversed_col_id_map = {v: k for k, v in col_id_map.items()}
|
||||
|
||||
|
||||
items = get_all_items(board, monday)
|
||||
|
||||
all_records = []
|
||||
for row in tqdm(items):
|
||||
data = {}
|
||||
data.update({"address": row['name']})
|
||||
data.update({"client": row['group']['title']})
|
||||
for col in row.get("column_values", []):
|
||||
if col.get("id") in reversed_col_id_map:
|
||||
if col.get("type") == "file":
|
||||
value = col.get("value")
|
||||
no_of_files = 0
|
||||
|
||||
if value:
|
||||
value = json.loads(col["value"])
|
||||
no_of_files = len(value.get('files', []))
|
||||
data.update({reversed_col_id_map[col.get("id")]: no_of_files})
|
||||
elif "no show" in reversed_col_id_map[col.get("id")]:
|
||||
def extract_number_from_text(text):
|
||||
number_str = ''
|
||||
|
||||
for char in text:
|
||||
if char.isnumeric():
|
||||
number_str += char
|
||||
elif number_str:
|
||||
break # stop once a number sequence ends
|
||||
|
||||
return int(number_str) if number_str else None
|
||||
text = col.get("text")
|
||||
if text is None:
|
||||
data.update({
|
||||
reversed_col_id_map[col.get("id")]: col.get("text")
|
||||
})
|
||||
else:
|
||||
data.update({
|
||||
reversed_col_id_map[col.get("id")]: extract_number_from_text(text)
|
||||
})
|
||||
else:
|
||||
data.update({
|
||||
reversed_col_id_map[col.get("id")]: col.get("text")
|
||||
})
|
||||
all_records.append(data)
|
||||
|
||||
# Convert to DataFrame
|
||||
df = pd.DataFrame(all_records)
|
||||
|
||||
filtered_dfs = []
|
||||
|
||||
def get_df(df, column_name, success_critera, job_name=None):
|
||||
_ = df[
|
||||
df[column_name].str.lower().isin(success_critera)
|
||||
].copy()
|
||||
if job_name:
|
||||
_["job_type"] = job_name
|
||||
return _
|
||||
|
||||
# RA
|
||||
ra = get_df(df, "ra status", ["completed rdsap 9.9", "completed rdsap 10"], "RA")
|
||||
filtered_dfs.append(ra)
|
||||
|
||||
|
||||
# PRE- ATT
|
||||
att = get_df(df, "pre att", ["completed"], "ATT")
|
||||
filtered_dfs.append(att)
|
||||
|
||||
# V1 Coordination
|
||||
v1 = get_df(df, "coordination status (ioe mtp)".lower(), [
|
||||
"rc complete",
|
||||
], "Coordination Stage 1 v1")
|
||||
filtered_dfs.append(v1)
|
||||
|
||||
# V2 Coordination
|
||||
v2 = get_df(df, "v2 coordination status (ioe mtp)", ["rc complete"], "Coordination Stage 1 v2 remodel")
|
||||
filtered_dfs.append(v2)
|
||||
|
||||
# # V3 Coordination
|
||||
# v3 = get_df(df, "v3 rc status", ["uploaded"], "V3 Coordination")
|
||||
# filtered_dfs.append(v3)
|
||||
|
||||
|
||||
# v3 = get_df(df, "v3 invoice status", ["to be invoice"], "V3 Coordination")
|
||||
# filtered_dfs.append(v3)
|
||||
|
||||
# Coordination stage 2 Please complete
|
||||
cors2 = df[
|
||||
df["rc stg 2"].str.lower().isin(["to invoice"])
|
||||
]
|
||||
cors2["joby_type"] = "Coordination Stage 2"
|
||||
filtered_dfs.append(cors2)
|
||||
|
||||
# Design stage
|
||||
design = get_df(df, "design invoice status", ["to invoice"])
|
||||
|
||||
# Design archeytpe
|
||||
de = get_df(design, "prop type for invoicing", ["archetype"], "Design Archetype")
|
||||
filtered_dfs.append(de)
|
||||
# Design repetitive
|
||||
de = get_df(design, "prop type for invoicing", ["repetitive"], "Design Repetitive")
|
||||
filtered_dfs.append(de)
|
||||
|
||||
# Design revision
|
||||
design2 = get_df(df, "design revision invoice status", [
|
||||
"to invoice".lower(),
|
||||
], "Design Revision")
|
||||
filtered_dfs.append(design2)
|
||||
|
||||
# Lodgement Phase 1
|
||||
lodg1 = get_df(df, "tm phase 1 invoice satus (lodgment)".lower(), ["to invoice"], "Lodgement Phase 1")
|
||||
filtered_dfs.append(lodg1)
|
||||
|
||||
|
||||
# Full Lodgement Phase
|
||||
full_lodgement = get_df(df, "lodgement invoice status".lower(), ["to invoice"], "Full lodgement phase 2")
|
||||
filtered_dfs.append(full_lodgement)
|
||||
|
||||
# POST EPC
|
||||
post_epc = get_df(df, "post-epc status", ["uploaded", "completed"], "POST EPC")
|
||||
filtered_dfs.append(post_epc)
|
||||
|
||||
|
||||
# # POST EPR
|
||||
post_epr = df[
|
||||
df["post-epc status"].str.lower().isin(["post epr completed"])
|
||||
].copy()
|
||||
post_epr["job_type"] = "POST epr"
|
||||
filtered_dfs.append(post_epr)
|
||||
|
||||
# Post ATT
|
||||
post_att = get_df(df, "post-att status", ["uploaded", "completed"], "POST ATT")
|
||||
filtered_dfs.append(post_att)
|
||||
|
||||
# Retrofit Evaluation
|
||||
retro = get_df(df, "retrofit evaluation", ["completed", "uploaded"], "Retrofit Evaluation")
|
||||
filtered_dfs.append(retro)
|
||||
|
||||
# RA NO Show
|
||||
ra_ns = df[
|
||||
(df["ra no show evidence"].fillna(-9999) != df["ra no show invoice"].fillna(-9999)) &
|
||||
(df["ra no show evidence"] != 0)
|
||||
].copy()
|
||||
ra_ns["job_type"] = "RA NO SHOW"
|
||||
filtered_dfs.append(ra_ns)
|
||||
|
||||
|
||||
# ATT NO Show
|
||||
att_ns = df[
|
||||
(df["att no show evidence"].fillna(-9999) != df["att no show invoice"].fillna(-9999)) &
|
||||
(df["att no show evidence"] != 0)
|
||||
].copy()
|
||||
att_ns["job_type"] = "ATT NO SHOW"
|
||||
filtered_dfs.append(att_ns)
|
||||
|
||||
|
||||
# Post visit no show
|
||||
epc_ns = df[
|
||||
(df["epc no show evidence"].fillna(-9999) != df["epc no show invoice"].fillna(-9999)) &
|
||||
(df["epc no show evidence"] != 0)
|
||||
].copy()
|
||||
epc_ns["job_type"] = "post EPC NO SHOW"
|
||||
filtered_dfs.append(epc_ns)
|
||||
|
||||
final_df = pd.concat(filtered_dfs).reset_index(drop=True)
|
||||
|
||||
final_df["job_type"] = final_df["job_type"].str.lower()
|
||||
rate_card_df["job_type"] = rate_card_df["job_type"].str.lower()
|
||||
|
||||
# Now perform the merge
|
||||
combined_with_rates = final_df.merge(rate_card_df, on="job_type", how="left")
|
||||
import datetime
|
||||
timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M')
|
||||
|
||||
attribute = ['address', 'client', 'job_type', 'rate']
|
||||
combined_with_rates[attribute].to_excel(f'WCHG SHDF 2.1 Mansard {timestamp}.xlsx', index=False)
|
||||
208
etl/month_end_automation_wave_2_no_16.py
Normal file
208
etl/month_end_automation_wave_2_no_16.py
Normal file
|
|
@ -0,0 +1,208 @@
|
|||
# Wave 2's month end automation
|
||||
from tqdm import tqdm
|
||||
from monday import MondayClient
|
||||
from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset_ids
|
||||
from pprint import pprint
|
||||
import pandas as pd
|
||||
import json
|
||||
|
||||
monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY"
|
||||
monday = MondayClient(monday_key)
|
||||
# NCHA SHDF Wave 3 On Hold
|
||||
board_ids = ["6946967610"]
|
||||
|
||||
rate_card_data = {
|
||||
"job_type": [
|
||||
"RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel",
|
||||
"Design Archetype", "Design Repetitive", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2",
|
||||
"Post EPR", "Post EPC", "Post ATT", "retrofit evaluation",
|
||||
"RA no show", "ATT no show", "post EPC no show"
|
||||
],
|
||||
"rate": [
|
||||
207.65, 101, 186.4, 98, 98,
|
||||
450, 150, 163, 135, 120,
|
||||
"Marianne EPR Please", 45, 90.5, 40,
|
||||
25, 25, 25
|
||||
]
|
||||
}
|
||||
|
||||
rate_card_df = pd.DataFrame(rate_card_data)
|
||||
|
||||
for board in tqdm(board_ids):
|
||||
board_data = monday.boards.fetch_boards_by_id(board)
|
||||
columns = board_data["data"]["boards"][0]["columns"]
|
||||
col_id_map = {col["title"].lower(): col["id"] for col in columns}
|
||||
reversed_col_id_map = {v: k for k, v in col_id_map.items()}
|
||||
|
||||
|
||||
items = get_all_items(board, monday)
|
||||
|
||||
all_records = []
|
||||
for row in tqdm(items):
|
||||
data = {}
|
||||
data.update({"address": row['name']})
|
||||
data.update({"client": row['group']['title']})
|
||||
for col in row.get("column_values", []):
|
||||
if col.get("id") in reversed_col_id_map:
|
||||
if col.get("type") == "file":
|
||||
value = col.get("value")
|
||||
no_of_files = 0
|
||||
|
||||
if value:
|
||||
value = json.loads(col["value"])
|
||||
no_of_files = len(value.get('files', []))
|
||||
data.update({reversed_col_id_map[col.get("id")]: no_of_files})
|
||||
elif "no show" in reversed_col_id_map[col.get("id")]:
|
||||
def extract_number_from_text(text):
|
||||
number_str = ''
|
||||
|
||||
for char in text:
|
||||
if char.isnumeric():
|
||||
number_str += char
|
||||
elif number_str:
|
||||
break # stop once a number sequence ends
|
||||
|
||||
return int(number_str) if number_str else None
|
||||
text = col.get("text")
|
||||
if text is None:
|
||||
data.update({
|
||||
reversed_col_id_map[col.get("id")]: col.get("text")
|
||||
})
|
||||
else:
|
||||
data.update({
|
||||
reversed_col_id_map[col.get("id")]: extract_number_from_text(text)
|
||||
})
|
||||
else:
|
||||
data.update({
|
||||
reversed_col_id_map[col.get("id")]: col.get("text")
|
||||
})
|
||||
all_records.append(data)
|
||||
|
||||
# Convert to DataFrame
|
||||
df = pd.DataFrame(all_records)
|
||||
|
||||
filtered_dfs = []
|
||||
|
||||
def get_df(df, column_name, success_critera, job_name):
|
||||
_ = df[
|
||||
df[column_name].str.lower().isin(success_critera)
|
||||
].copy()
|
||||
_["job_type"] = job_name
|
||||
return _
|
||||
|
||||
# RA
|
||||
ra = get_df(df, "ra", ["completed"], "RA")
|
||||
filtered_dfs.append(ra)
|
||||
|
||||
|
||||
# PRE- ATT
|
||||
att = get_df(df, "att", ["completed"], "ATT")
|
||||
filtered_dfs.append(att)
|
||||
|
||||
# V1 Coordination
|
||||
v1 = get_df(df, "coordination status".lower(), [
|
||||
"rc complete",
|
||||
], "Coordination Stage 1 v1")
|
||||
filtered_dfs.append(v1)
|
||||
|
||||
# V2 Coordination
|
||||
# v2 = get_df(df, "mtp v2 status", ["rc v2 complete"], "Coordination Stage 1 v2 remodel")
|
||||
# filtered_dfs.append(v2)
|
||||
|
||||
# # V3 Coordination
|
||||
# v3 = get_df(df, "v3 rc status", ["uploaded"], "Coordination Stage 1 v3 remode")
|
||||
# filtered_dfs.append(v3)
|
||||
|
||||
# v3 = get_df(df, "v3 invoice status", ["to be invoice"], "V3 Coordination")
|
||||
# filtered_dfs.append(v3)
|
||||
|
||||
# Coordination stage 2 Please complete
|
||||
cors2 = df[
|
||||
df["rc stg. 2"].str.lower().isin(["to invoice"])
|
||||
]
|
||||
cors2["joby_type"] = "Coordination Stage 2"
|
||||
filtered_dfs.append(cors2)
|
||||
|
||||
# Design Archetype
|
||||
# design1 = get_df(df, "design invoice status", ["to invoice"], "Design")
|
||||
# filtered_dfs.append(design1)
|
||||
|
||||
|
||||
# Design Repetitive
|
||||
|
||||
# Design Revision
|
||||
# design2 = get_df(df, "design revision invoice", [
|
||||
# "Rev. A to invoice".lower(),
|
||||
# "Rev. B to invoice".lower(),
|
||||
# "Rev. C to invoice".lower(),
|
||||
# "Rev. D to invoice".lower(),
|
||||
# ], "Design Revision")
|
||||
# filtered_dfs.append(design2)
|
||||
|
||||
# Lodgement Phase 1
|
||||
lodg1 = get_df(df, "lodg. phase 1 invoice status".lower(), ["to invoice"], "Lodgement Phase 1")
|
||||
filtered_dfs.append(lodg1)
|
||||
|
||||
|
||||
# Full Lodgement Phase
|
||||
full_lodgement = get_df(df, "full lodgement invoice status".lower(), ["to invoice"], "Full lodgement phase 2")
|
||||
filtered_dfs.append(full_lodgement)
|
||||
|
||||
# POST EPC
|
||||
post_epc = get_df(df, "lodged epc", ["complete", "complete & lodged",], "Post EPC")
|
||||
filtered_dfs.append(post_epc)
|
||||
|
||||
|
||||
# POST EPR
|
||||
post_epr = df[
|
||||
df["lodged epc"].str.lower().isin(["post epr completed"])
|
||||
].copy()
|
||||
post_epr["job_type"] = "POST EPR"
|
||||
filtered_dfs.append(post_epr)
|
||||
|
||||
# Post ATT
|
||||
post_att = get_df(df, "post att", ["done", "post att complete"], "POST ATT")
|
||||
filtered_dfs.append(post_att)
|
||||
|
||||
# Retrofit Evaluation
|
||||
retro = get_df(df, "retrofit evaluation", ["done"], "Retrofit Evaluation")
|
||||
filtered_dfs.append(retro)
|
||||
|
||||
# RA NO Show
|
||||
ra_ns = df[
|
||||
(df["ra no show evidence"].fillna(-9999) != df["ra no show invoice"].fillna(-9999)) &
|
||||
(df["ra no show evidence"] != 0)
|
||||
].copy()
|
||||
ra_ns["job_type"] = "RA NO SHOW"
|
||||
filtered_dfs.append(ra_ns)
|
||||
|
||||
|
||||
# ATT NO Show
|
||||
att_ns = df[
|
||||
(df["att no show evidence"].fillna(-9999) != df["att no show invoice"].fillna(-9999)) &
|
||||
(df["att no show evidence"] != 0)
|
||||
].copy()
|
||||
att_ns["job_type"] = "ATT NO SHOW"
|
||||
filtered_dfs.append(att_ns)
|
||||
|
||||
|
||||
# Post visit no show
|
||||
# epc_ns = df[
|
||||
# df["epc no show evidence"].fillna(-9999) != df["epc no show invoice"].fillna(-9999)
|
||||
# ].copy()
|
||||
# epc_ns["job_type"] = "post EPC NO SHOW"
|
||||
# filtered_dfs.append(epc_ns)
|
||||
|
||||
final_df = pd.concat(filtered_dfs).reset_index(drop=True)
|
||||
|
||||
|
||||
final_df["job_type"] = final_df["job_type"].str.lower()
|
||||
rate_card_df["job_type"] = rate_card_df["job_type"].str.lower()
|
||||
|
||||
# Now perform the merge
|
||||
combined_with_rates = final_df.merge(rate_card_df, on="job_type", how="left")
|
||||
import datetime
|
||||
timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M')
|
||||
|
||||
attribute = ['address', 'client', 'job_type', 'rate']
|
||||
combined_with_rates[attribute].to_excel(f'NCHA SHDF Wave 3 On Hold_{timestamp}.xlsx', index=False)
|
||||
234
etl/month_end_automation_wave_2_no_3.py
Normal file
234
etl/month_end_automation_wave_2_no_3.py
Normal file
|
|
@ -0,0 +1,234 @@
|
|||
# Wave 2's month end automation
|
||||
|
||||
from tqdm import tqdm
|
||||
from monday import MondayClient
|
||||
from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset_ids
|
||||
from pprint import pprint
|
||||
import pandas as pd
|
||||
import json
|
||||
|
||||
monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY"
|
||||
monday = MondayClient(monday_key)
|
||||
# Platform Housing W2 (in use)
|
||||
board_ids = ["4796290860"]
|
||||
|
||||
rate_card_data = {
|
||||
"job_type": [
|
||||
"RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel",
|
||||
"Design Archetype", "Design Repetitive", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2",
|
||||
"Post EPR", "Post EPC", "Post ATT", "retrofit evaluation",
|
||||
"RA no show", "ATT no show", "post EPC no show", "Design Revision"
|
||||
],
|
||||
"rate": [
|
||||
259, 101, 210, 95, 95,
|
||||
450, 150, 195, 135,
|
||||
120, "(60)) - please confirm with Marianne, EPR", 45, 90.5, 42.4,
|
||||
25, 25, 25, "Please ask for Design Revision"
|
||||
]
|
||||
}
|
||||
|
||||
rate_card_df = pd.DataFrame(rate_card_data)
|
||||
|
||||
|
||||
for board in tqdm(board_ids):
|
||||
board_data = monday.boards.fetch_boards_by_id(board)
|
||||
columns = board_data["data"]["boards"][0]["columns"]
|
||||
col_id_map = {col["title"].lower(): col["id"] for col in columns}
|
||||
reversed_col_id_map = {v: k for k, v in col_id_map.items()}
|
||||
|
||||
|
||||
items = get_all_items(board, monday)
|
||||
|
||||
all_records = []
|
||||
for row in tqdm(items):
|
||||
data = {}
|
||||
data.update({"address": row['name']})
|
||||
data.update({"client": row['group']['title']})
|
||||
for col in row.get("column_values", []):
|
||||
if col.get("id") in reversed_col_id_map:
|
||||
if col.get("type") == "file":
|
||||
value = col.get("value")
|
||||
no_of_files = 0
|
||||
|
||||
if value:
|
||||
value = json.loads(col["value"])
|
||||
no_of_files = len(value.get('files', []))
|
||||
data.update({reversed_col_id_map[col.get("id")]: no_of_files})
|
||||
elif "no show" in reversed_col_id_map[col.get("id")]:
|
||||
def extract_number_from_text(text):
|
||||
number_str = ''
|
||||
|
||||
for char in text:
|
||||
if char.isnumeric():
|
||||
number_str += char
|
||||
elif number_str:
|
||||
break # stop once a number sequence ends
|
||||
|
||||
return int(number_str) if number_str else None
|
||||
text = col.get("text")
|
||||
if text is None:
|
||||
data.update({
|
||||
reversed_col_id_map[col.get("id")]: col.get("text")
|
||||
})
|
||||
else:
|
||||
data.update({
|
||||
reversed_col_id_map[col.get("id")]: extract_number_from_text(text)
|
||||
})
|
||||
else:
|
||||
data.update({
|
||||
reversed_col_id_map[col.get("id")]: col.get("text")
|
||||
})
|
||||
all_records.append(data)
|
||||
|
||||
# Convert to DataFrame
|
||||
df = pd.DataFrame(all_records)
|
||||
|
||||
filtered_dfs = []
|
||||
|
||||
# RA
|
||||
ra = df[
|
||||
df["ra"].str.lower().isin(["completed rdsap 10", "completed rdsap 9.9", "completed", "complete"])
|
||||
].copy()
|
||||
ra["job_type"] = "RA"
|
||||
filtered_dfs.append(ra)
|
||||
|
||||
|
||||
# ATT
|
||||
att = df[
|
||||
df["att"].str.lower().isin(["completed"])
|
||||
].copy()
|
||||
att["job_type"] = "ATT"
|
||||
filtered_dfs.append(att)
|
||||
|
||||
# V1 Coordination
|
||||
v1 = df[
|
||||
df["coordination status"].str.lower().isin(["ima/mtp completed"])
|
||||
].copy()
|
||||
v1["job_type"] = "Coordination Stage 1 v1"
|
||||
filtered_dfs.append(v1)
|
||||
|
||||
# V2 Coordination
|
||||
_ = df[df["v2 mtp status"].fillna('').str.lower().isin(['v2 ima-mtp completed', 'v2 completed'])].copy()
|
||||
_["job_type"] = "Coordination Stage 1 v2 remodel"
|
||||
filtered_dfs.append(_)
|
||||
|
||||
# V3 Coordination
|
||||
# v3 = df[
|
||||
# df["v3 invoiced"].str.lower().isin(["to be invoiced"])
|
||||
# ].copy()
|
||||
# v3["job_type"] = "Coordination Stage 1 v3 remodel"
|
||||
# filtered_dfs.append(v3)
|
||||
|
||||
# Coordination stage 2 Please complete
|
||||
cors2 = df[
|
||||
df["rc stg. 2"].str.lower().isin(["to invoice"])
|
||||
]
|
||||
cors2["joby_type"] = "Coordination Stage 2"
|
||||
filtered_dfs.append(cors2)
|
||||
|
||||
# Design Archetype
|
||||
design1 = df[
|
||||
df["design invoice"].str.lower().isin(["complete pending rc"])
|
||||
].copy()
|
||||
design1 = design1[design1["design type for invoicing"].str.lower().isin(['archetype'])].copy()
|
||||
design1["job_type"] = "Design Archetype"
|
||||
filtered_dfs.append(design1)
|
||||
|
||||
# Design Repetitive
|
||||
design1 = df[
|
||||
df["design invoice"].str.lower().isin(["complete pending rc"])
|
||||
].copy()
|
||||
design1 = design1[design1["design type for invoicing"].str.lower().isin(['repetitive'])].copy()
|
||||
design1["job_type"] = "Design repetitive"
|
||||
filtered_dfs.append(design1)
|
||||
|
||||
# Design Revision
|
||||
design_revision = df[
|
||||
df["design revision invoice status"].str.lower().isin(["to invoice"])
|
||||
].copy()
|
||||
design_revision["job_type"] = "Design repetitive"
|
||||
filtered_dfs.append(design_revision)
|
||||
|
||||
# Lodgement Phase 1
|
||||
lodg1 = df[
|
||||
df["phase 1 invoice status (lodgement)"].str.lower().isin(["done"])
|
||||
].copy()
|
||||
lodg1["job_type"] = "Lodgement Phase 1"
|
||||
filtered_dfs.append(lodg1)
|
||||
|
||||
# Full Lodgement Phase
|
||||
lodg2 = df[
|
||||
df["lodgement invoice status (lodgement)"].str.lower().isin(["to invoice"])
|
||||
].copy()
|
||||
lodg2["job_type"] = "Full lodgement phase 2"
|
||||
filtered_dfs.append(lodg2)
|
||||
|
||||
# POST EPC
|
||||
post_epc = df[
|
||||
df["post epc"].str.lower().isin(["success", "pics uploaded"])
|
||||
].copy()
|
||||
post_epc["job_type"] = "POST EPC"
|
||||
filtered_dfs.append(post_epc)
|
||||
|
||||
|
||||
# POST EPR
|
||||
post_epr = df[
|
||||
df["post epc"].str.lower().isin(["post epr completed"])
|
||||
].copy()
|
||||
post_epr["job_type"] = "POST EPR"
|
||||
filtered_dfs.append(post_epr)
|
||||
|
||||
|
||||
# Post ATT
|
||||
post_att = df[
|
||||
df["post att"].str.lower().isin(["uploaded"])
|
||||
].copy()
|
||||
post_att["job_type"] = "POST ATT"
|
||||
filtered_dfs.append(post_att)
|
||||
|
||||
|
||||
# Retrofit Evaluation
|
||||
retro = df[
|
||||
df["retrofit evaluation"].str.lower().isin(["uploaded", "completed", "to invoice"])
|
||||
].copy()
|
||||
retro["job_type"] = "Retrofit Evaluation"
|
||||
filtered_dfs.append(retro)
|
||||
|
||||
# RA NO Show
|
||||
ra_ns = df[
|
||||
(df["ra no show evidence"].fillna(-9999) != df["ra no show invoice"].fillna(-9999)) &
|
||||
(df["ra no show evidence"] != 0)
|
||||
].copy()
|
||||
ra_ns["job_type"] = "RA NO SHOW"
|
||||
filtered_dfs.append(ra_ns)
|
||||
|
||||
|
||||
# ATT NO Show
|
||||
att_ns = df[
|
||||
(df["att no show evidence"].fillna(-9999) != df["att no show invoice"].fillna(-9999)) &
|
||||
(df["att no show evidence"] != 0)
|
||||
].copy()
|
||||
att_ns["job_type"] = "ATT NO SHOW"
|
||||
filtered_dfs.append(att_ns)
|
||||
|
||||
|
||||
# Post visit no show
|
||||
epc_ns = df[
|
||||
(df["post epc no show evidence"].fillna(-9999) != df["post epc no show invoice"].fillna(-9999)) &
|
||||
(df["post epc no show evidence"] != 0)
|
||||
].copy()
|
||||
epc_ns["job_type"] = "Post EPC NO SHOW"
|
||||
filtered_dfs.append(epc_ns)
|
||||
|
||||
final_df = pd.concat(filtered_dfs).reset_index(drop=True)
|
||||
|
||||
final_df["job_type"] = final_df["job_type"].str.lower()
|
||||
rate_card_df["job_type"] = rate_card_df["job_type"].str.lower()
|
||||
|
||||
# Now perform the merge
|
||||
combined_with_rates = final_df.merge(rate_card_df, on="job_type", how="left")
|
||||
import datetime
|
||||
timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M')
|
||||
|
||||
attribute = ['address', 'client', 'job_type', 'rate']
|
||||
combined_with_rates[attribute].to_excel(f'Platform Housing W2 (in use)_{timestamp}.xlsx', index=False)
|
||||
241
etl/month_end_automation_wave_2_no_4.py
Normal file
241
etl/month_end_automation_wave_2_no_4.py
Normal file
|
|
@ -0,0 +1,241 @@
|
|||
# Wave 2's month end automation
|
||||
|
||||
from tqdm import tqdm
|
||||
from monday import MondayClient
|
||||
from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset_ids
|
||||
from pprint import pprint
|
||||
import pandas as pd
|
||||
import json
|
||||
|
||||
monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY"
|
||||
monday = MondayClient(monday_key)
|
||||
# Stonewater (in use)
|
||||
board_ids = ["3584401309"]
|
||||
|
||||
rate_card_data = {
|
||||
"job_type": [
|
||||
"RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel",
|
||||
"Design Archetype", "Design Repetitive", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2",
|
||||
"Post EPR", "Post EPC", "Post ATT", "retrofit evaluation",
|
||||
"RA no show", "ATT no show", "post EPC no show", "Design Revision"
|
||||
],
|
||||
"rate": [
|
||||
165.75, 72.25, 174.25, 174.25, 174.25,
|
||||
175, 175, 124.25, 135,
|
||||
120, "(60) - please check with Marianne", 45, 63.75, 34,
|
||||
25, 25, 25, "Please ask marianne or kev for design revision"
|
||||
]
|
||||
}
|
||||
|
||||
rate_card_df = pd.DataFrame(rate_card_data)
|
||||
|
||||
for board in tqdm(board_ids):
|
||||
board_data = monday.boards.fetch_boards_by_id(board)
|
||||
columns = board_data["data"]["boards"][0]["columns"]
|
||||
col_id_map = {col["title"].lower(): col["id"] for col in columns}
|
||||
reversed_col_id_map = {v: k for k, v in col_id_map.items()}
|
||||
|
||||
|
||||
items = get_all_items(board, monday)
|
||||
|
||||
all_records = []
|
||||
for row in tqdm(items):
|
||||
data = {}
|
||||
data.update({"address": row['name']})
|
||||
data.update({"client": row['group']['title']})
|
||||
for col in row.get("column_values", []):
|
||||
if col.get("id") in reversed_col_id_map:
|
||||
if col.get("type") == "file":
|
||||
value = col.get("value")
|
||||
no_of_files = 0
|
||||
|
||||
if value:
|
||||
value = json.loads(col["value"])
|
||||
no_of_files = len(value.get('files', []))
|
||||
data.update({reversed_col_id_map[col.get("id")]: no_of_files})
|
||||
elif "no show" in reversed_col_id_map[col.get("id")]:
|
||||
def extract_number_from_text(text):
|
||||
number_str = ''
|
||||
|
||||
for char in text:
|
||||
if char.isnumeric():
|
||||
number_str += char
|
||||
elif number_str:
|
||||
break # stop once a number sequence ends
|
||||
|
||||
return int(number_str) if number_str else None
|
||||
text = col.get("text")
|
||||
if text is None:
|
||||
data.update({
|
||||
reversed_col_id_map[col.get("id")]: col.get("text")
|
||||
})
|
||||
else:
|
||||
data.update({
|
||||
reversed_col_id_map[col.get("id")]: extract_number_from_text(text)
|
||||
})
|
||||
else:
|
||||
data.update({
|
||||
reversed_col_id_map[col.get("id")]: col.get("text")
|
||||
})
|
||||
all_records.append(data)
|
||||
|
||||
# Convert to DataFrame
|
||||
df = pd.DataFrame(all_records)
|
||||
|
||||
filtered_dfs = []
|
||||
|
||||
# RA
|
||||
ra = df[
|
||||
df["ra"].str.lower().isin(["completed rdsap 10", "completed rdsap 9.9"])
|
||||
].copy()
|
||||
ra["job_type"] = "RA"
|
||||
filtered_dfs.append(ra)
|
||||
|
||||
|
||||
# ATT
|
||||
att = df[
|
||||
df["att"].str.lower().isin(["completed"])
|
||||
].copy()
|
||||
att["job_type"] = "ATT"
|
||||
filtered_dfs.append(att)
|
||||
|
||||
# V1 Coordination
|
||||
v1 = df[
|
||||
df["v1 coordination status (ioe,mtp)"].str.lower().isin(["rc complete"])
|
||||
].copy()
|
||||
v1["job_type"] = "Coordination Stage 1 v1"
|
||||
filtered_dfs.append(v1)
|
||||
|
||||
# V2 Coordination
|
||||
_ = df[df["mtp v2 status"].str.lower().isin(['rc v2 complete'])].copy()
|
||||
_["job_type"] = "Coordination Stage 1 v2 remodel"
|
||||
filtered_dfs.append(_)
|
||||
|
||||
# V2 Coordination
|
||||
_ = df[df["mtp v2 invoiced"].str.lower().isin(['needs to be invoiced'])].copy()
|
||||
_["job_type"] = "Coordination Stage 1 v2 remodel"
|
||||
filtered_dfs.append(_)
|
||||
|
||||
# V3 Coordination
|
||||
v3 = df[df["v3 rc status"].str.lower().isin(['uploaded'])].copy()
|
||||
v3["job_type"] = "Coordination Stage 1 v3 remodel"
|
||||
filtered_dfs.append(_)
|
||||
|
||||
# V3 Coordination
|
||||
v3 = df[df["v3 invoice status"].str.lower().isin(['to be invoiced'])].copy()
|
||||
v3["job_type"] = "Coordination Stage 1 v3 remodel"
|
||||
filtered_dfs.append(_)
|
||||
|
||||
# Coordination stage 2 Please complete
|
||||
cors2 = df[
|
||||
df["rc stg. 2 status"].str.lower().isin(["to invoice", "completed"])
|
||||
]
|
||||
cors2["joby_type"] = "Coordination Stage 2"
|
||||
filtered_dfs.append(cors2)
|
||||
|
||||
# Design Archetype
|
||||
design1 = df[
|
||||
df["design invoice status"].str.lower().isin(["complete", "to invoice"])
|
||||
].copy()
|
||||
design1 = design1[design1["design type"].str.lower().isin(["archetype"])].copy()
|
||||
design1["job_type"] = "Design Archetype"
|
||||
filtered_dfs.append(design1)
|
||||
|
||||
# Design Repetitive
|
||||
design1 = df[
|
||||
df["design invoice status"].str.lower().isin(["complete", "to invoice"])
|
||||
].copy()
|
||||
design1 = design1[design1["design type"].str.lower().isin(["repetitive"])].copy()
|
||||
design1["job_type"] = "Design Repetitive"
|
||||
filtered_dfs.append(design1)
|
||||
|
||||
# Design Revision
|
||||
design_revision = df[
|
||||
df["design revision invoice status"].str.lower().isin(["to invoice"])
|
||||
].copy()
|
||||
design_revision["job_type"] = "Design repetitive"
|
||||
filtered_dfs.append(design_revision)
|
||||
|
||||
# Lodgement Phase 1
|
||||
lodg1 = df[
|
||||
df["phase 1 invoice status (lodgement)"].str.lower().isin(["done", "to be invoiced"])
|
||||
].copy()
|
||||
lodg1["job_type"] = "Lodgement Phase 1"
|
||||
filtered_dfs.append(lodg1)
|
||||
|
||||
# Full Lodgement Phase
|
||||
_ = df[
|
||||
df["lodgement invoice status"].str.lower().isin(["to invoice"])
|
||||
].copy()
|
||||
_["job_type"] = "Full lodgement phase 2"
|
||||
filtered_dfs.append(_)
|
||||
|
||||
# POST EPC
|
||||
post_epc = df[
|
||||
df["post epc"].str.lower().isin(["completed & uploaded"])
|
||||
].copy()
|
||||
post_epc["job_type"] = "POST EPC"
|
||||
filtered_dfs.append(post_epc)
|
||||
|
||||
|
||||
# POST EPR
|
||||
post_epr = df[
|
||||
df["post epc"].str.lower().isin(["post epr completed"])
|
||||
].copy()
|
||||
post_epr["job_type"] = "POST EPR"
|
||||
filtered_dfs.append(post_epr)
|
||||
|
||||
# Post ATT
|
||||
post_att = df[
|
||||
df["post att"].str.lower().isin(["completed & uploaded"])
|
||||
].copy()
|
||||
post_att["job_type"] = "POST ATT"
|
||||
filtered_dfs.append(post_att)
|
||||
|
||||
|
||||
# Retrofit Evaluation
|
||||
retro = df[
|
||||
df["retrofit evaluation"].str.lower().isin(["completed & uploaded"])
|
||||
].copy()
|
||||
retro["job_type"] = "Retrofit Evaluation"
|
||||
filtered_dfs.append(retro)
|
||||
|
||||
# RA NO Show
|
||||
ra_ns = df[
|
||||
(df["ra no show evidence"].fillna(-9999) != df["ra no show invoice"].fillna(-9999)) &
|
||||
(df["ra no show evidence"] != 0)
|
||||
].copy()
|
||||
ra_ns["job_type"] = "RA NO SHOW"
|
||||
filtered_dfs.append(ra_ns)
|
||||
|
||||
|
||||
# ATT NO Show
|
||||
att_ns = df[
|
||||
(df["att no show evidence"].fillna(-9999) != df["att no show invoice"].fillna(-9999)) &
|
||||
(df["att no show evidence"] != 0)
|
||||
].copy()
|
||||
att_ns["job_type"] = "ATT NO SHOW"
|
||||
filtered_dfs.append(att_ns)
|
||||
|
||||
|
||||
# Post visit no show
|
||||
epc_ns = df[
|
||||
(df["post epc no show evidence"].fillna(-9999) != df["post epc no show invoice"].fillna(-9999)) &
|
||||
(df["post epc no show evidence"] != 0)
|
||||
].copy()
|
||||
epc_ns["job_type"] = "post EPC NO SHOW"
|
||||
filtered_dfs.append(epc_ns)
|
||||
|
||||
final_df = pd.concat(filtered_dfs).reset_index(drop=True)
|
||||
|
||||
|
||||
final_df["job_type"] = final_df["job_type"].str.lower()
|
||||
rate_card_df["job_type"] = rate_card_df["job_type"].str.lower()
|
||||
|
||||
# Now perform the merge
|
||||
combined_with_rates = final_df.merge(rate_card_df, on="job_type", how="left")
|
||||
import datetime
|
||||
timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M')
|
||||
|
||||
attribute = ['address', 'client', 'job_type', 'rate']
|
||||
combined_with_rates[attribute].to_excel(f'Stonewater - (in use)_{timestamp}.xlsx', index=False)
|
||||
233
etl/month_end_automation_wave_2_no_5.py
Normal file
233
etl/month_end_automation_wave_2_no_5.py
Normal file
|
|
@ -0,0 +1,233 @@
|
|||
# Wave 2's month end automation
|
||||
|
||||
from tqdm import tqdm
|
||||
from monday import MondayClient
|
||||
from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset_ids
|
||||
from pprint import pprint
|
||||
import pandas as pd
|
||||
import json
|
||||
|
||||
monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY"
|
||||
monday = MondayClient(monday_key)
|
||||
# ECO 4 NCHA Almshouses Operations
|
||||
board_ids = ["9136254638"]
|
||||
|
||||
rate_card_data = {
|
||||
"job_type": [
|
||||
"RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel",
|
||||
"Design Archetype", "Design Repetitive", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2",
|
||||
"Post EPR", "Post EPC", "Post ATT", "retrofit evaluation",
|
||||
"RA no show", "ATT no show", "post EPC no show"
|
||||
],
|
||||
"rate": [
|
||||
259, 125, 280, 125, 125,
|
||||
650, 195, 175, 135,
|
||||
120, "Post EPR Please Marianne", 85, 125, 60,
|
||||
25, 25, 25
|
||||
]
|
||||
}
|
||||
|
||||
rate_card_df = pd.DataFrame(rate_card_data)
|
||||
|
||||
|
||||
for board in tqdm(board_ids):
|
||||
board_data = monday.boards.fetch_boards_by_id(board)
|
||||
columns = board_data["data"]["boards"][0]["columns"]
|
||||
col_id_map = {col["title"].lower(): col["id"] for col in columns}
|
||||
reversed_col_id_map = {v: k for k, v in col_id_map.items()}
|
||||
|
||||
|
||||
items = get_all_items(board, monday)
|
||||
|
||||
all_records = []
|
||||
for row in tqdm(items):
|
||||
data = {}
|
||||
data.update({"address": row['name']})
|
||||
data.update({"client": row['group']['title']})
|
||||
for col in row.get("column_values", []):
|
||||
if col.get("id") in reversed_col_id_map:
|
||||
if col.get("type") == "file":
|
||||
value = col.get("value")
|
||||
no_of_files = 0
|
||||
|
||||
if value:
|
||||
value = json.loads(col["value"])
|
||||
no_of_files = len(value.get('files', []))
|
||||
data.update({reversed_col_id_map[col.get("id")]: no_of_files})
|
||||
elif "no show" in reversed_col_id_map[col.get("id")]:
|
||||
def extract_number_from_text(text):
|
||||
number_str = ''
|
||||
|
||||
for char in text:
|
||||
if char.isnumeric():
|
||||
number_str += char
|
||||
elif number_str:
|
||||
break # stop once a number sequence ends
|
||||
|
||||
return int(number_str) if number_str else None
|
||||
text = col.get("text")
|
||||
if text is None:
|
||||
data.update({
|
||||
reversed_col_id_map[col.get("id")]: col.get("text")
|
||||
})
|
||||
else:
|
||||
data.update({
|
||||
reversed_col_id_map[col.get("id")]: extract_number_from_text(text)
|
||||
})
|
||||
else:
|
||||
data.update({
|
||||
reversed_col_id_map[col.get("id")]: col.get("text")
|
||||
})
|
||||
all_records.append(data)
|
||||
|
||||
# Convert to DataFrame
|
||||
df = pd.DataFrame(all_records)
|
||||
|
||||
filtered_dfs = []
|
||||
|
||||
# RA
|
||||
ra = df[
|
||||
df["retrofit assessment"].str.lower().isin(["completed rdsap 10", "completed rdsap 9.9"])
|
||||
].copy()
|
||||
ra["job_type"] = "RA"
|
||||
filtered_dfs.append(ra)
|
||||
|
||||
|
||||
# ATT
|
||||
att = df[
|
||||
df["pre att"].str.lower().isin(["completed"])
|
||||
].copy()
|
||||
att["job_type"] = "ATT"
|
||||
filtered_dfs.append(att)
|
||||
|
||||
# V1 Coordination
|
||||
v1 = df[
|
||||
df["coordination status"].str.lower().isin(["ioe/mtp complete"])
|
||||
].copy()
|
||||
v1["job_type"] = "Coordination Stage 1 v1"
|
||||
filtered_dfs.append(v1)
|
||||
|
||||
# V2 Coordination
|
||||
# _ = df[df["mtp v2 invoiced"].str.lower().isin(['done', 'needs to be invoiced'])].copy()
|
||||
# _["job_type"] = "Coordination Stage 1 v2 remodel"
|
||||
# filtered_dfs.append(_)
|
||||
|
||||
# V3 Coordination
|
||||
# v3 = df[df["v3 rc status"].str.lower().isin(['uploaded'])].copy()
|
||||
# v3["job_type"] = "Coordination Stage 1 v3 remodel"
|
||||
# filtered_dfs.append(_)
|
||||
|
||||
# Coordination stage 2 Please complete
|
||||
cors2 = df[
|
||||
df["rc stage 2"].str.lower().isin(["to invoice",])
|
||||
]
|
||||
cors2["joby_type"] = "Coordination Stage 2"
|
||||
filtered_dfs.append(cors2)
|
||||
|
||||
# Design stage 1
|
||||
design1 = df[
|
||||
df["retrofit design status"].str.lower().isin(["to invoice"])
|
||||
].copy()
|
||||
design1 = design1[design1["design type"].str.lower().isin(["archetype"])].copy()
|
||||
design1["job_type"] = "Design Archetype"
|
||||
filtered_dfs.append(design1)
|
||||
|
||||
# Design stage 2
|
||||
design2 = df[
|
||||
df["retrofit design status"].str.lower().isin(["to invoice"])
|
||||
].copy()
|
||||
design2 = design2[design2["design type"].str.lower().isin(["repetitive"])].copy()
|
||||
design2["job_type"] = "Design Repetitive"
|
||||
filtered_dfs.append(design2)
|
||||
|
||||
# Design revision
|
||||
design2 = df[
|
||||
df["retrofit design status"].str.lower().isin(["to invoice"])
|
||||
].copy()
|
||||
design2 = design2[design2["design revision"].str.lower().isin(["A", "B", "C"])].copy()
|
||||
design2["job_type"] = "Design Repetitive"
|
||||
filtered_dfs.append(design2)
|
||||
|
||||
|
||||
# Lodgement Phase 1
|
||||
# lodg1 = df[
|
||||
# df["phase 1 invoice status (lodgement)"].str.lower().isin(["done", "to be invoiced"])
|
||||
# ].copy()
|
||||
# lodg1["job_type"] = "Lodgement Phase 1"
|
||||
# filtered_dfs.append(lodg1)
|
||||
|
||||
# Full Lodgement Phase
|
||||
_ = df[
|
||||
df["trustmark lodgement"].str.lower().isin(["done"])
|
||||
].copy()
|
||||
_["job_type"] = "Full lodgement phase 2"
|
||||
filtered_dfs.append(_)
|
||||
|
||||
# POST EPC
|
||||
post_epc = df[
|
||||
df["post epc status"].str.lower().isin(["done"])
|
||||
].copy()
|
||||
post_epc["job_type"] = "POST EPC"
|
||||
filtered_dfs.append(post_epc)
|
||||
|
||||
|
||||
# POST EPR
|
||||
post_epr = df[
|
||||
df["post epc status"].str.lower().isin(["post epr completed"])
|
||||
].copy()
|
||||
post_epr["job_type"] = "POST epr"
|
||||
filtered_dfs.append(post_epr)
|
||||
|
||||
# Post ATT
|
||||
post_att = df[
|
||||
df["post att status"].str.lower().isin(["done"])
|
||||
].copy()
|
||||
post_att["job_type"] = "POST ATT"
|
||||
filtered_dfs.append(post_att)
|
||||
|
||||
|
||||
# Retrofit Evaluation
|
||||
retro = df[
|
||||
df["retrofit evaluation"].str.lower().isin(["done"])
|
||||
].copy()
|
||||
retro["job_type"] = "Retrofit Evaluation"
|
||||
filtered_dfs.append(retro)
|
||||
|
||||
# RA NO Show
|
||||
ra_ns = df[
|
||||
(df["ra no show evidence"].fillna(-9999) != df["ra no show invoice"].fillna(-9999)) &
|
||||
(df["ra no show evidence"] != 0)
|
||||
].copy()
|
||||
ra_ns["job_type"] = "RA NO SHOW"
|
||||
filtered_dfs.append(ra_ns)
|
||||
|
||||
|
||||
# ATT NO Show
|
||||
att_ns = df[
|
||||
(df["pre att no show evidence"].fillna(-9999) != df["pre att no show invoice"].fillna(-9999)) &
|
||||
(df["pre att no show evidence"] != 0)
|
||||
].copy()
|
||||
att_ns["job_type"] = "ATT NO SHOW"
|
||||
filtered_dfs.append(att_ns)
|
||||
|
||||
|
||||
# Post visit no show
|
||||
epc_ns = df[
|
||||
(df["epc no show evidence"].fillna(-9999) != df["epc no show invoice"].fillna(-9999)) &
|
||||
(df["epc no show evidence"] != 0 )
|
||||
].copy()
|
||||
epc_ns["job_type"] = "post EPC NO SHOW"
|
||||
filtered_dfs.append(epc_ns)
|
||||
|
||||
final_df = pd.concat(filtered_dfs).reset_index(drop=True)
|
||||
|
||||
final_df["job_type"] = final_df["job_type"].str.lower()
|
||||
rate_card_df["job_type"] = rate_card_df["job_type"].str.lower()
|
||||
|
||||
# Now perform the merge
|
||||
combined_with_rates = final_df.merge(rate_card_df, on="job_type", how="left")
|
||||
import datetime
|
||||
timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M')
|
||||
|
||||
attribute = ['address', 'client', 'job_type', 'rate']
|
||||
combined_with_rates[attribute].to_excel(f'ECO 4 NCHA Almshouses Operations_{timestamp}.xlsx', index=False)
|
||||
207
etl/month_end_automation_wave_2_no_6.py
Normal file
207
etl/month_end_automation_wave_2_no_6.py
Normal file
|
|
@ -0,0 +1,207 @@
|
|||
# Wave 2's month end automation
|
||||
|
||||
from tqdm import tqdm
|
||||
from monday import MondayClient
|
||||
from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset_ids
|
||||
from pprint import pprint
|
||||
import pandas as pd
|
||||
import json
|
||||
|
||||
monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY"
|
||||
monday = MondayClient(monday_key)
|
||||
#ECO 4 Wates Operations
|
||||
board_ids = ["9520779048"]
|
||||
|
||||
rate_card_data = {
|
||||
"job_type": [
|
||||
"RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel",
|
||||
"Design Archetype", "Design Repetitive", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2",
|
||||
"Post EPR", "Post EPC", "Post ATT", "retrofit evaluation",
|
||||
"RA no show", "ATT no show", "post EPC no show"
|
||||
],
|
||||
"rate": [
|
||||
259, 125, 280, 125, 125,
|
||||
650, 195, 175, 135,
|
||||
120, "Post EPR Please Marianne", 85, 125, 60,
|
||||
25, 25, 25
|
||||
]
|
||||
}
|
||||
|
||||
rate_card_df = pd.DataFrame(rate_card_data)
|
||||
|
||||
for board in tqdm(board_ids):
|
||||
board_data = monday.boards.fetch_boards_by_id(board)
|
||||
columns = board_data["data"]["boards"][0]["columns"]
|
||||
col_id_map = {col["title"].lower(): col["id"] for col in columns}
|
||||
reversed_col_id_map = {v: k for k, v in col_id_map.items()}
|
||||
|
||||
|
||||
items = get_all_items(board, monday)
|
||||
|
||||
all_records = []
|
||||
for row in tqdm(items):
|
||||
data = {}
|
||||
data.update({"address": row['name']})
|
||||
data.update({"client": row['group']['title']})
|
||||
for col in row.get("column_values", []):
|
||||
if col.get("id") in reversed_col_id_map:
|
||||
if col.get("type") == "file":
|
||||
value = col.get("value")
|
||||
no_of_files = 0
|
||||
|
||||
if value:
|
||||
value = json.loads(col["value"])
|
||||
no_of_files = len(value.get('files', []))
|
||||
data.update({reversed_col_id_map[col.get("id")]: no_of_files})
|
||||
elif "no show" in reversed_col_id_map[col.get("id")]:
|
||||
def extract_number_from_text(text):
|
||||
number_str = ''
|
||||
|
||||
for char in text:
|
||||
if char.isnumeric():
|
||||
number_str += char
|
||||
elif number_str:
|
||||
break # stop once a number sequence ends
|
||||
|
||||
return int(number_str) if number_str else None
|
||||
text = col.get("text")
|
||||
if text is None:
|
||||
data.update({
|
||||
reversed_col_id_map[col.get("id")]: col.get("text")
|
||||
})
|
||||
else:
|
||||
data.update({
|
||||
reversed_col_id_map[col.get("id")]: extract_number_from_text(text)
|
||||
})
|
||||
else:
|
||||
data.update({
|
||||
reversed_col_id_map[col.get("id")]: col.get("text")
|
||||
})
|
||||
all_records.append(data)
|
||||
|
||||
# Convert to DataFrame
|
||||
df = pd.DataFrame(all_records)
|
||||
|
||||
filtered_dfs = []
|
||||
|
||||
def get_df(df, column_name, success_critera, job_name):
|
||||
_ = df[
|
||||
df[column_name].str.lower().isin(success_critera)
|
||||
].copy()
|
||||
_["job_type"] = job_name
|
||||
return _
|
||||
|
||||
# RA
|
||||
ra = get_df(df, "ra status", ["completed & uploaded"], "RA")
|
||||
filtered_dfs.append(ra)
|
||||
|
||||
|
||||
# ATT
|
||||
att = get_df(df, "pre att status", ["completed & uploaded"], "ATT")
|
||||
filtered_dfs.append(att)
|
||||
|
||||
# V1 Coordination
|
||||
v1 = get_df(df, "Coordination Status IOE/MTP".lower(), [
|
||||
"(V1) IOE/MTP Complete".lower(),
|
||||
"(V2) IOE/MTP Complete".lower(),
|
||||
"(V3) IOE/MTP Complete".lower(),
|
||||
], "Coordination Stage 1 v1")
|
||||
filtered_dfs.append(v1)
|
||||
|
||||
# V2 Coordination
|
||||
# _ = df[df["mtp v2 invoiced"].str.lower().isin(['done', 'needs to be invoiced'])].copy()
|
||||
# _["job_type"] = "Coordination Stage 1 v2 remodel"
|
||||
# filtered_dfs.append(_)
|
||||
|
||||
# V3 Coordination
|
||||
# v3 = df[df["v3 rc status"].str.lower().isin(['uploaded'])].copy()
|
||||
# v3["job_type"] = "Coordination Stage 1 v3 remodel"
|
||||
# filtered_dfs.append(_)
|
||||
|
||||
# Coordination stage 2 Please complete
|
||||
cors2 = df[
|
||||
df["rc stage 2 status"].str.lower().isin(["to invoice"])
|
||||
]
|
||||
cors2["joby_type"] = "Coordination Stage 2"
|
||||
filtered_dfs.append(cors2)
|
||||
|
||||
# Design stage 1
|
||||
design1 = get_df(df, "retrofit design status", ["completed"], "Design")
|
||||
filtered_dfs.append(design1)
|
||||
|
||||
# Design stage 2
|
||||
design2 = get_df(df, "design revision invoice", [
|
||||
"Rev. A to invoice".lower(),
|
||||
"Rev. B to invoice".lower(),
|
||||
"Rev. C to invoice".lower(),
|
||||
"Rev. D to invoice".lower(),
|
||||
], "Design Repetitive")
|
||||
filtered_dfs.append(design2)
|
||||
|
||||
# Lodgement Phase 1
|
||||
# lodg1 = df[
|
||||
# df["phase 1 invoice status (lodgement)"].str.lower().isin(["done", "to be invoiced"])
|
||||
# ].copy()
|
||||
# lodg1["job_type"] = "Lodgement Phase 1"
|
||||
# filtered_dfs.append(lodg1)
|
||||
|
||||
# Full Lodgement Phase
|
||||
full_lodgement = get_df(df, "full lodgement", ["completed"], "Full lodgement phase 2")
|
||||
filtered_dfs.append(full_lodgement)
|
||||
|
||||
# POST EPC
|
||||
post_epc = get_df(df, "post epc & evaluation status", ["uploaded"], "POST EPC")
|
||||
filtered_dfs.append(post_epc)
|
||||
|
||||
|
||||
# POST EPR
|
||||
post_epr = df[
|
||||
df["post epc & evaluation status"].str.lower().isin(["post epr completed"])
|
||||
].copy()
|
||||
post_epr["job_type"] = "POST epr"
|
||||
filtered_dfs.append(post_epr)
|
||||
|
||||
# Post ATT
|
||||
post_att = get_df(df, "post att status", ["uploaded"], "POST ATT")
|
||||
filtered_dfs.append(post_att)
|
||||
|
||||
|
||||
# Retrofit Evaluation
|
||||
retro = get_df(df, "post epc & evaluation status", ["uploaded"], "Retrofit Evaluation")
|
||||
filtered_dfs.append(retro)
|
||||
|
||||
# RA NO Show
|
||||
# ra_ns = df[
|
||||
# df["ra no show evidence"].fillna(-9999) != df["ra no show invoice"].fillna(-9999)
|
||||
# ].copy()
|
||||
# ra_ns["job_type"] = "RA NO SHOW"
|
||||
# filtered_dfs.append(ra_ns)
|
||||
|
||||
|
||||
# # ATT NO Show
|
||||
# att_ns = df[
|
||||
# df["pre att no show evidence"].fillna(-9999) != df["pre att no show invoice"].fillna(-9999)
|
||||
# ].copy()
|
||||
# att_ns["job_type"] = "ATT NO SHOW"
|
||||
# filtered_dfs.append(att_ns)
|
||||
|
||||
|
||||
# # Post visit no show
|
||||
# epc_ns = df[
|
||||
# df["epc no show evidence"].fillna(-9999) != df["epc no show invoice"].fillna(-9999)
|
||||
# ].copy()
|
||||
# epc_ns["job_type"] = "post EPC NO SHOW"
|
||||
# filtered_dfs.append(epc_ns)
|
||||
|
||||
final_df = pd.concat(filtered_dfs).reset_index(drop=True)
|
||||
|
||||
final_df["job_type"] = final_df["job_type"].str.lower()
|
||||
rate_card_df["job_type"] = rate_card_df["job_type"].str.lower()
|
||||
|
||||
# Now perform the merge
|
||||
combined_with_rates = final_df.merge(rate_card_df, on="job_type", how="left")
|
||||
import datetime
|
||||
timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M')
|
||||
|
||||
attribute = ['address', 'client', 'job_type', 'rate']
|
||||
combined_with_rates[attribute].to_excel(f'ECO 4 Wates Operations_{timestamp}.xlsx', index=False)
|
||||
257
etl/month_end_automation_wave_2_no_7.py
Normal file
257
etl/month_end_automation_wave_2_no_7.py
Normal file
|
|
@ -0,0 +1,257 @@
|
|||
# Wave 2's month end automation
|
||||
|
||||
from tqdm import tqdm
|
||||
from monday import MondayClient
|
||||
from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset_ids
|
||||
from pprint import pprint
|
||||
import pandas as pd
|
||||
import json
|
||||
|
||||
monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY"
|
||||
monday = MondayClient(monday_key)
|
||||
#Home Group Wave 2SP+
|
||||
board_ids = ["4254419092"]
|
||||
|
||||
rate_card_data_sp_plus = {
|
||||
"job_type": [
|
||||
"RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel",
|
||||
"Design Archetype", "Design Repetitive", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2",
|
||||
"Post EPR", "Post EPC", "Post ATT", "retrofit evaluation",
|
||||
"RA no show", "ATT no show", "post EPC no show"
|
||||
],
|
||||
"rate": [
|
||||
170, 70, 200, "check with Kevin", "check with Kevin",
|
||||
470, 155, 165, 135,
|
||||
120, "60 but check with Kevin as EPR", 45, 70, 40,
|
||||
30, 30, 30
|
||||
]
|
||||
}
|
||||
|
||||
emp_msg = "was empty in rate card - ask Marianne/Kevin"
|
||||
rate_card_data_net_zero = {
|
||||
"job_type": [
|
||||
"RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel",
|
||||
"Design Archetype", "Design Repetitive", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2",
|
||||
"Post EPR", "Post EPC", "Post ATT", "retrofit evaluation",
|
||||
"RA no show", "ATT no show", "post EPC no show"
|
||||
],
|
||||
"rate": [
|
||||
170, 70, 200, emp_msg, emp_msg,
|
||||
325, 140, 165, 135,
|
||||
120, "60 but check with Kevin as EPR", 45, 70, 40,
|
||||
30, 30, 30
|
||||
]
|
||||
}
|
||||
|
||||
error_message = "Unsure which client this one is - sorry!"
|
||||
rate_card_data_error_msg= {
|
||||
"job_type": [
|
||||
"RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel",
|
||||
"Design Archetype", "Design Repetitive", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2",
|
||||
"Post EPR", "Post EPC", "Post ATT", "retrofit evaluation",
|
||||
"RA no show", "ATT no show", "post EPC no show"
|
||||
],
|
||||
"rate": [
|
||||
error_message, error_message, error_message, error_message, error_message,
|
||||
error_message, error_message, error_message, error_message,
|
||||
error_message, error_message, error_message, error_message, error_message,
|
||||
error_message, error_message, error_message
|
||||
]
|
||||
}
|
||||
|
||||
rate_card_df_sp_plus = pd.DataFrame(rate_card_data_sp_plus)
|
||||
rate_card_df_net_zero = pd.DataFrame(rate_card_data_net_zero)
|
||||
rate_card_df_error_message = pd.DataFrame(rate_card_data_error_msg)
|
||||
|
||||
|
||||
for board in tqdm(board_ids):
|
||||
board_data = monday.boards.fetch_boards_by_id(board)
|
||||
columns = board_data["data"]["boards"][0]["columns"]
|
||||
col_id_map = {col["title"].lower(): col["id"] for col in columns}
|
||||
reversed_col_id_map = {v: k for k, v in col_id_map.items()}
|
||||
|
||||
|
||||
items = get_all_items(board, monday)
|
||||
|
||||
all_records = []
|
||||
for row in tqdm(items):
|
||||
data = {}
|
||||
data.update({"address": row['name']})
|
||||
data.update({"client": row['group']['title']})
|
||||
for col in row.get("column_values", []):
|
||||
if col.get("id") in reversed_col_id_map:
|
||||
if col.get("type") == "file":
|
||||
value = col.get("value")
|
||||
no_of_files = 0
|
||||
|
||||
if value:
|
||||
value = json.loads(col["value"])
|
||||
no_of_files = len(value.get('files', []))
|
||||
data.update({reversed_col_id_map[col.get("id")]: no_of_files})
|
||||
elif "no show" in reversed_col_id_map[col.get("id")]:
|
||||
def extract_number_from_text(text):
|
||||
number_str = ''
|
||||
|
||||
for char in text:
|
||||
if char.isnumeric():
|
||||
number_str += char
|
||||
elif number_str:
|
||||
break # stop once a number sequence ends
|
||||
|
||||
return int(number_str) if number_str else None
|
||||
text = col.get("text")
|
||||
if text is None:
|
||||
data.update({
|
||||
reversed_col_id_map[col.get("id")]: col.get("text")
|
||||
})
|
||||
else:
|
||||
data.update({
|
||||
reversed_col_id_map[col.get("id")]: extract_number_from_text(text)
|
||||
})
|
||||
else:
|
||||
data.update({
|
||||
reversed_col_id_map[col.get("id")]: col.get("text")
|
||||
})
|
||||
all_records.append(data)
|
||||
|
||||
# Convert to DataFrame
|
||||
df = pd.DataFrame(all_records)
|
||||
|
||||
filtered_dfs = []
|
||||
|
||||
def get_df(df, column_name, success_critera, job_name=None):
|
||||
_ = df[
|
||||
df[column_name].str.lower().isin(success_critera)
|
||||
].copy()
|
||||
if job_name:
|
||||
_["job_type"] = job_name
|
||||
return _
|
||||
|
||||
|
||||
# RA
|
||||
ra = get_df(df, "ra", ["completed rdsap 9.9", "completed rdsap 10"], "RA")
|
||||
filtered_dfs.append(ra)
|
||||
|
||||
|
||||
# ATT
|
||||
att = get_df(df, "pre-att", ["completed"], "ATT")
|
||||
filtered_dfs.append(att)
|
||||
|
||||
# V1 Coordination
|
||||
v1 = get_df(df, "osmosis rc status".lower(), [
|
||||
"rc completed",
|
||||
], "Coordination Stage 1 v1")
|
||||
filtered_dfs.append(v1)
|
||||
|
||||
# V2 Coordination
|
||||
v2 = get_df(df, "v2 ioe mtp", ["completed"], "Coordination Stage 1 v2 remodel")
|
||||
filtered_dfs.append(v2)
|
||||
|
||||
# V3 Coordination
|
||||
v3 = get_df(df, "v3 rc status", ["rc completed"], "Coordination Stage 1 v3 remodel")
|
||||
filtered_dfs.append(v3)
|
||||
|
||||
# Coordination stage 2 Please complete
|
||||
cors2 = df[
|
||||
df["rc stage 2"].str.lower().isin(["to invoice"])
|
||||
]
|
||||
cors2["joby_type"] = "Coordination Stage 2"
|
||||
filtered_dfs.append(cors2)
|
||||
|
||||
# Design stage Archetype
|
||||
design1 = get_df(df, "design invoice status", ["to invoice"])
|
||||
design1 = get_df(design1, "design type for invoicing", ["archetype"], "Design Archetype")
|
||||
filtered_dfs.append(design1)
|
||||
|
||||
# Design stage Repetitive
|
||||
design1 = get_df(df, "design invoice status", ["to invoice"])
|
||||
design1 = get_df(design1, "design type for invoicing", ["repetitive"], "Design Repetitive")
|
||||
filtered_dfs.append(design1)
|
||||
|
||||
# Design revision
|
||||
design2 = get_df(df, "design revision invoice status", [
|
||||
"to invoice"
|
||||
], "Design Revision")
|
||||
filtered_dfs.append(design2)
|
||||
|
||||
# Lodgement Phase 1
|
||||
lodg1 = get_df(df, "TM Phase 1 Invoicing Status".lower(), ["done", "to invoice"], "Lodgement phase 1")
|
||||
filtered_dfs.append(lodg1)
|
||||
|
||||
# Full Lodgement Phase
|
||||
full_lodgement = get_df(df, "Jun-te TM Phase 2 Invoicing Status".lower(), ["to invoice"], "Full lodgement phase 2")
|
||||
filtered_dfs.append(full_lodgement)
|
||||
|
||||
# POST EPC
|
||||
post_epc = get_df(df, "post-epc status", ["complete & uploaded"], "POST EPC")
|
||||
filtered_dfs.append(post_epc)
|
||||
|
||||
|
||||
# POST EPR
|
||||
post_epr = df[
|
||||
df["post-epc status"].str.lower().isin(["post epr completed"])
|
||||
].copy()
|
||||
post_epr["job_type"] = "POST epr"
|
||||
filtered_dfs.append(post_epr)
|
||||
|
||||
# Post ATT
|
||||
post_att = get_df(df, "post att invoicing status ", ["to invoice"], "POST ATT")
|
||||
filtered_dfs.append(post_att)
|
||||
|
||||
|
||||
# Retrofit Evaluation
|
||||
retro = get_df(df, "retrofit evaluation", ["completed & uploaded"], "Retrofit Evaluation")
|
||||
filtered_dfs.append(retro)
|
||||
|
||||
# RA NO Show
|
||||
ra_ns = df[
|
||||
(df["ra no show evidence"].fillna(-9999) != df["ra no show invoice"].fillna(-9999)) &
|
||||
(df["ra no show evidence"] != 0)
|
||||
].copy()
|
||||
ra_ns["job_type"] = "RA NO SHOW"
|
||||
filtered_dfs.append(ra_ns)
|
||||
|
||||
|
||||
# ATT NO Show
|
||||
att_ns = df[
|
||||
(df["att no show evidence"].fillna(-9999) != df["att no show invoice"].fillna(-9999)) &
|
||||
(df["att no show evidence"] != 0 )
|
||||
].copy()
|
||||
att_ns["job_type"] = "ATT NO SHOW"
|
||||
filtered_dfs.append(att_ns)
|
||||
|
||||
|
||||
# Post visit no show
|
||||
epc_ns = df[
|
||||
(df["epc no show evidence"].fillna(-9999) != df["epc no show invoice"].fillna(-9999)) &
|
||||
(df["epc no show evidence"] != 0 )
|
||||
].copy()
|
||||
epc_ns["job_type"] = "post EPC NO SHOW"
|
||||
filtered_dfs.append(epc_ns)
|
||||
|
||||
final_df = pd.concat(filtered_dfs).reset_index(drop=True)
|
||||
|
||||
final_df["job_type"] = final_df["job_type"].str.lower()
|
||||
rate_card_df_net_zero["job_type"] = rate_card_df_net_zero["job_type"].str.lower()
|
||||
rate_card_df_sp_plus["job_type"] = rate_card_df_sp_plus["job_type"].str.lower()
|
||||
rate_card_df_error_message["job_type"] = rate_card_df_error_message["job_type"].str.lower()
|
||||
|
||||
# Now perform the merge
|
||||
net_zero_df = final_df[final_df['client'].str.contains('shdf net zero'.lower(), case=False, na=False)]
|
||||
sp_plus_df = final_df[final_df['client'].str.contains('SHDF 2.0 SP+'.lower(), case=False, na=False)]
|
||||
other_df = final_df[~final_df.index.isin(net_zero_df.index) & ~final_df.index.isin(sp_plus_df.index)]
|
||||
|
||||
combined_with_rates_net_zero_df = net_zero_df.merge(rate_card_df_net_zero, on="job_type", how="left")
|
||||
combined_with_rates_sp_plus = sp_plus_df.merge(rate_card_df_sp_plus, on="job_type", how="left")
|
||||
combined_with_rates_other_from_home_group = other_df.merge(rate_card_df_error_message, on="job_type", how="left")
|
||||
import datetime
|
||||
timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M')
|
||||
|
||||
attribute = ['address', 'client', 'job_type', 'rate']
|
||||
combined_with_rates_sp_plus[attribute].to_excel(f'HomeGroup Wave 2SP+_{timestamp}.xlsx', index=False)
|
||||
combined_with_rates_net_zero_df[attribute].to_excel(f'HomeGroup Wave NetZero_{timestamp}.xlsx', index=False)
|
||||
combined_with_rates_other_from_home_group[attribute].to_excel(f'HomeGroup Wave Unsure_who_to_bill_{timestamp}.xlsx', index=False)
|
||||
|
||||
|
||||
# TO DO check everything in excel
|
||||
# make logic for seperation
|
||||
218
etl/month_end_automation_wave_2_no_8.py
Normal file
218
etl/month_end_automation_wave_2_no_8.py
Normal file
|
|
@ -0,0 +1,218 @@
|
|||
# Wave 2's month end automation
|
||||
from tqdm import tqdm
|
||||
from monday import MondayClient
|
||||
from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset_ids
|
||||
from pprint import pprint
|
||||
import pandas as pd
|
||||
import json
|
||||
|
||||
monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY"
|
||||
monday = MondayClient(monday_key)
|
||||
# NCHA SHDF 2.1 SBS
|
||||
board_ids = ["8668578700"]
|
||||
|
||||
rate_card_data = {
|
||||
"job_type": [
|
||||
"RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel",
|
||||
"Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2",
|
||||
"Post EPR", "Post EPC", "Post ATT", "retrofit evaluation",
|
||||
"RA no show", "ATT no show", "post EPC no show", "Design Revision"
|
||||
],
|
||||
"rate": [
|
||||
259, 125, 280, 125, 125,
|
||||
650, 415, 195, 175, 135,
|
||||
120, "60 - Double check with Kevin/Marianne EPR", 85, 125, 60,
|
||||
45, 45, 45, "Design Revision check with Kevin/Marianne"
|
||||
]
|
||||
}
|
||||
|
||||
rate_card_df = pd.DataFrame(rate_card_data)
|
||||
|
||||
|
||||
for board in tqdm(board_ids):
|
||||
board_data = monday.boards.fetch_boards_by_id(board)
|
||||
columns = board_data["data"]["boards"][0]["columns"]
|
||||
col_id_map = {col["title"].lower(): col["id"] for col in columns}
|
||||
reversed_col_id_map = {v: k for k, v in col_id_map.items()}
|
||||
|
||||
|
||||
items = get_all_items(board, monday)
|
||||
|
||||
all_records = []
|
||||
for row in tqdm(items):
|
||||
data = {}
|
||||
data.update({"address": row['name']})
|
||||
data.update({"client": row['group']['title']})
|
||||
for col in row.get("column_values", []):
|
||||
if col.get("id") in reversed_col_id_map:
|
||||
if col.get("type") == "file":
|
||||
value = col.get("value")
|
||||
no_of_files = 0
|
||||
|
||||
if value:
|
||||
value = json.loads(col["value"])
|
||||
no_of_files = len(value.get('files', []))
|
||||
data.update({reversed_col_id_map[col.get("id")]: no_of_files})
|
||||
elif "no show" in reversed_col_id_map[col.get("id")]:
|
||||
def extract_number_from_text(text):
|
||||
number_str = ''
|
||||
|
||||
for char in text:
|
||||
if char.isnumeric():
|
||||
number_str += char
|
||||
elif number_str:
|
||||
break # stop once a number sequence ends
|
||||
|
||||
return int(number_str) if number_str else None
|
||||
text = col.get("text")
|
||||
if text is None:
|
||||
data.update({
|
||||
reversed_col_id_map[col.get("id")]: col.get("text")
|
||||
})
|
||||
else:
|
||||
data.update({
|
||||
reversed_col_id_map[col.get("id")]: extract_number_from_text(text)
|
||||
})
|
||||
else:
|
||||
data.update({
|
||||
reversed_col_id_map[col.get("id")]: col.get("text")
|
||||
})
|
||||
all_records.append(data)
|
||||
|
||||
# Convert to DataFrame
|
||||
df = pd.DataFrame(all_records)
|
||||
|
||||
filtered_dfs = []
|
||||
|
||||
def get_df(df, column_name, success_critera, job_name=None):
|
||||
_ = df[
|
||||
df[column_name].str.lower().isin(success_critera)
|
||||
].copy()
|
||||
if job_name:
|
||||
_["job_type"] = job_name
|
||||
return _
|
||||
|
||||
|
||||
# RA
|
||||
ra = get_df(df, "ra", ["completed rdsap 9.9", "completed rdsap 10"], "RA")
|
||||
filtered_dfs.append(ra)
|
||||
|
||||
|
||||
# ATT
|
||||
att = get_df(df, "att", ["completed"], "ATT")
|
||||
filtered_dfs.append(att)
|
||||
|
||||
# V1 Coordination
|
||||
v1 = get_df(df, "v1 coordination status".lower(), [
|
||||
"rc complete",
|
||||
], "Coordination Stage 1 v1")
|
||||
filtered_dfs.append(v1)
|
||||
|
||||
# V2 Coordination
|
||||
v2 = get_df(df, "v2 coordination status", ["rc v2 complete", "uploaded"], "Coordination Stage 1 v2 remodel")
|
||||
filtered_dfs.append(v2)
|
||||
|
||||
# # V3 Coordination
|
||||
# v3 = get_df(df, "")
|
||||
# # filtered_dfs.append(v3)
|
||||
|
||||
# Corodination stage 2
|
||||
cors2 = get_df(df, "rc stg. 2", ["to invoice"], "Coordination Stage 2")
|
||||
filtered_dfs.append(cors2)
|
||||
|
||||
# Design Archtype Complex
|
||||
design1 = get_df(df, "rd invoiced", ["to invoice"])
|
||||
design1 = get_df(design1, "design type", ["archetype (complex)"])
|
||||
design1 = get_df(design1, "design upload to sharepoint", ["done"], "Design Archetype Complex")
|
||||
filtered_dfs.append(design1)
|
||||
|
||||
# Design Archtype Simple
|
||||
design1 = get_df(df, "rd invoiced", ["to invoice"])
|
||||
design1 = get_df(design1, "design type", ["archetype (simple)"])
|
||||
design1 = get_df(design1, "design upload to sharepoint", ["done"], "Design Archetype Simple")
|
||||
filtered_dfs.append(design1)
|
||||
|
||||
# Design Repitive Simple
|
||||
design1 = get_df(df, "rd invoiced", ["to invoice"])
|
||||
design1 = get_df(design1, "design type", ["Design Repetitive Simple"])
|
||||
design1 = get_df(design1, "design upload to sharepoint", ["done"], "Design Repetitive Simple")
|
||||
filtered_dfs.append(design1)
|
||||
|
||||
# Design revision
|
||||
design2 = get_df(df, "design revision invoice status", [
|
||||
"to invoice"
|
||||
], "Design Revision")
|
||||
filtered_dfs.append(design2)
|
||||
|
||||
# Lodgement Phase 1
|
||||
lodg1 = get_df(df, "Lodg. Phase 1 Invoice Status".lower(), ["to be invoiced"], "Lodgement Phase 1")
|
||||
filtered_dfs.append(lodg1)
|
||||
|
||||
# Lodgement Phase 1
|
||||
lodg1 = get_df(df, "phase 1 to be invoiced".lower(), ["phase 1 to be invoiced"], "Lodgement Phase 1")
|
||||
filtered_dfs.append(lodg1)
|
||||
|
||||
# Full Lodgement Phase
|
||||
full_lodgement = get_df(df, "full lodgement invoice status".lower(), ["to be invoice"], "Full Lodgement phase 2")
|
||||
filtered_dfs.append(full_lodgement)
|
||||
|
||||
# POST EPC
|
||||
post_epc = get_df(df, "post epc status", ["uploaded"], "POST EPC")
|
||||
filtered_dfs.append(post_epc)
|
||||
|
||||
|
||||
# POST EPR
|
||||
post_epr = df[
|
||||
df["post epc status"].str.lower().isin(["post epr completed"])
|
||||
].copy()
|
||||
post_epr["job_type"] = "POST epr"
|
||||
filtered_dfs.append(post_epr)
|
||||
|
||||
# Post ATT
|
||||
post_att = get_df(df, "post att", ["post att uploaded"], "POST ATT")
|
||||
filtered_dfs.append(post_att)
|
||||
|
||||
|
||||
# Retrofit Evaluation
|
||||
retro = get_df(df, "retrofit evaluation", ["done"], "Retrofit Evaluation")
|
||||
filtered_dfs.append(retro)
|
||||
|
||||
# RA NO Show
|
||||
ra_ns = df[
|
||||
(df["ra no show evidence"].fillna(-9999) != df["ra no show invoice"].fillna(-9999)) &
|
||||
(df["ra no show evidence"] != 0 )
|
||||
].copy()
|
||||
ra_ns["job_type"] = "RA NO SHOW"
|
||||
filtered_dfs.append(ra_ns)
|
||||
|
||||
|
||||
# ATT NO Show
|
||||
att_ns = df[
|
||||
(df["att no show evidence"].fillna(-9999) != df["att no show invoice"].fillna(-9999)) &
|
||||
(df["att no show evidence"] !=0 )
|
||||
].copy()
|
||||
att_ns["job_type"] = "ATT NO SHOW"
|
||||
filtered_dfs.append(att_ns)
|
||||
|
||||
|
||||
# Post visit no show
|
||||
epc_ns = df[
|
||||
df["post works no show evidence"].fillna(-9999) != df["post works no show invoice"].fillna(-9999)
|
||||
].copy()
|
||||
epc_ns["job_type"] = "post EPC NO SHOW"
|
||||
filtered_dfs.append(epc_ns)
|
||||
|
||||
final_df = pd.concat(filtered_dfs).reset_index(drop=True)
|
||||
|
||||
final_df[['address', 'client', 'job_type']]
|
||||
|
||||
final_df["job_type"] = final_df["job_type"].str.lower()
|
||||
rate_card_df["job_type"] = rate_card_df["job_type"].str.lower()
|
||||
|
||||
# Now perform the merge
|
||||
combined_with_rates = final_df.merge(rate_card_df, on="job_type", how="left")
|
||||
import datetime
|
||||
timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M')
|
||||
|
||||
attribute = ['address', 'client', 'job_type', 'rate']
|
||||
combined_with_rates[attribute].to_excel(f'NCHA SHDF 2.1 SBS_{timestamp}.xlsx', index=False)
|
||||
207
etl/month_end_automation_wave_2_no_9.py
Normal file
207
etl/month_end_automation_wave_2_no_9.py
Normal file
|
|
@ -0,0 +1,207 @@
|
|||
# Wave 2's month end automation
|
||||
from tqdm import tqdm
|
||||
from monday import MondayClient
|
||||
from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset_ids
|
||||
from pprint import pprint
|
||||
import pandas as pd
|
||||
import json
|
||||
|
||||
monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY"
|
||||
monday = MondayClient(monday_key)
|
||||
# NCHA Almshouses
|
||||
board_ids = ["5423364294"]
|
||||
|
||||
rate_card_data = {
|
||||
"job_type": [
|
||||
"RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel",
|
||||
"Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2",
|
||||
"Post EPR", "Post EPC", "Post ATT", "retrofit evaluation",
|
||||
"RA no show", "ATT no show", "post EPC no show"
|
||||
],
|
||||
"rate": [
|
||||
259, 125, 280, 125, 125,
|
||||
650, 415, 195, 175, 135,
|
||||
120, "Post EPR Please Marianne", 85, 125, 60,
|
||||
25, 25, 25
|
||||
]
|
||||
}
|
||||
|
||||
rate_card_df = pd.DataFrame(rate_card_data)
|
||||
|
||||
|
||||
for board in tqdm(board_ids):
|
||||
board_data = monday.boards.fetch_boards_by_id(board)
|
||||
columns = board_data["data"]["boards"][0]["columns"]
|
||||
col_id_map = {col["title"].lower(): col["id"] for col in columns}
|
||||
reversed_col_id_map = {v: k for k, v in col_id_map.items()}
|
||||
|
||||
|
||||
items = get_all_items(board, monday)
|
||||
|
||||
all_records = []
|
||||
for row in tqdm(items):
|
||||
data = {}
|
||||
data.update({"address": row['name']})
|
||||
data.update({"client": row['group']['title']})
|
||||
for col in row.get("column_values", []):
|
||||
if col.get("id") in reversed_col_id_map:
|
||||
if col.get("type") == "file":
|
||||
value = col.get("value")
|
||||
no_of_files = 0
|
||||
|
||||
if value:
|
||||
value = json.loads(col["value"])
|
||||
no_of_files = len(value.get('files', []))
|
||||
data.update({reversed_col_id_map[col.get("id")]: no_of_files})
|
||||
elif "no show" in reversed_col_id_map[col.get("id")]:
|
||||
def extract_number_from_text(text):
|
||||
number_str = ''
|
||||
|
||||
for char in text:
|
||||
if char.isnumeric():
|
||||
number_str += char
|
||||
elif number_str:
|
||||
break # stop once a number sequence ends
|
||||
|
||||
return int(number_str) if number_str else None
|
||||
text = col.get("text")
|
||||
if text is None:
|
||||
data.update({
|
||||
reversed_col_id_map[col.get("id")]: col.get("text")
|
||||
})
|
||||
else:
|
||||
data.update({
|
||||
reversed_col_id_map[col.get("id")]: extract_number_from_text(text)
|
||||
})
|
||||
else:
|
||||
data.update({
|
||||
reversed_col_id_map[col.get("id")]: col.get("text")
|
||||
})
|
||||
all_records.append(data)
|
||||
|
||||
# Convert to DataFrame
|
||||
df = pd.DataFrame(all_records)
|
||||
|
||||
filtered_dfs = []
|
||||
|
||||
def get_df(df, column_name, success_critera, job_name):
|
||||
_ = df[
|
||||
df[column_name].str.lower().isin(success_critera)
|
||||
].copy()
|
||||
_["job_type"] = job_name
|
||||
return _
|
||||
|
||||
|
||||
# RA
|
||||
ra = get_df(df, "ra", ["completed rdsap 9.9", "completed rdsap 10"], "RA")
|
||||
filtered_dfs.append(ra)
|
||||
|
||||
|
||||
# ATT
|
||||
att = get_df(df, "att", ["completed"], "ATT")
|
||||
filtered_dfs.append(att)
|
||||
|
||||
# V1 Coordination
|
||||
v1 = get_df(df, "coordination status (mtp)".lower(), [
|
||||
"rc complete",
|
||||
], "Coordination Stage 1 v1")
|
||||
filtered_dfs.append(v1)
|
||||
|
||||
# V2 Coordination
|
||||
# v2 = get_df(df, "v2 coordination status", ["rc v2 complete", "uploaded"], "Coordination Stage 1 v2 remodel")
|
||||
# filtered_dfs.append(v2)
|
||||
|
||||
# # V3 Coordination
|
||||
# Coordination Stage 1 v3 remode
|
||||
# v3 = get_df(df, "")
|
||||
# # filtered_dfs.append(v3)
|
||||
|
||||
# Coordination stage 2 Please complete
|
||||
cors2 = df[
|
||||
df["rc stage 2"].str.lower().isin(["to invoice"])
|
||||
].copy()
|
||||
cors2["joby_type"] = "Coordination Stage 2"
|
||||
filtered_dfs.append(cors2)
|
||||
|
||||
# Design stage 1
|
||||
# design1 = get_df(df, "rd invoiced", ["to invoice"], "Design")
|
||||
# filtered_dfs.append(design1)
|
||||
|
||||
# Design revision
|
||||
# design2 = get_df(df, "design revision invoice", [
|
||||
# "Rev. A to invoice".lower(),
|
||||
# "Rev. B to invoice".lower(),
|
||||
# "Rev. C to invoice".lower(),
|
||||
# "Rev. D to invoice".lower(),
|
||||
# ], "Design Revision")
|
||||
# filtered_dfs.append(design2)
|
||||
|
||||
# Lodgement Phase 1
|
||||
# lodg1 = get_df(df, "Lodg. Phase 1 Invoice Status".lower(), ["to be invoiced"], "Lodgement Phase 1")
|
||||
# filtered_dfs.append(lodg1)
|
||||
|
||||
|
||||
# Full Lodgement Phase
|
||||
full_lodgement = get_df(df, "trustmark lodgement".lower(), ["done"], "Full lodgement phase 2")
|
||||
filtered_dfs.append(full_lodgement)
|
||||
|
||||
# POST EPC
|
||||
# post_epc = get_df(df, "post epc status", ["uploaded"], "POST EPC")
|
||||
# filtered_dfs.append(post_epc)
|
||||
|
||||
|
||||
# # POST EPR
|
||||
# post_epr = df[
|
||||
# df["post-epc status"].str.lower().isin(["post epr completed"])
|
||||
# ].copy()
|
||||
# post_epr["job_type"] = "POST epr"
|
||||
# filtered_dfs.append(post_epr)
|
||||
|
||||
# Post ATT
|
||||
# post_att = get_df(df, "post att", ["post att uploaded"], "POST ATT")
|
||||
# filtered_dfs.append(post_att)
|
||||
|
||||
|
||||
# Retrofit Evaluation
|
||||
retro = get_df(df, "retrofit evaluation", ["done"], "Retrofit Evaluation")
|
||||
filtered_dfs.append(retro)
|
||||
|
||||
# RA NO Show
|
||||
ra_ns = df[
|
||||
(df["ra no show evidence"].fillna(-9999) != df["ra no show invoice"].fillna(-9999)) &
|
||||
(df["ra no show evidence"] != 0)
|
||||
].copy()
|
||||
ra_ns["job_type"] = "RA NO SHOW"
|
||||
filtered_dfs.append(ra_ns)
|
||||
|
||||
|
||||
# ATT NO Show
|
||||
att_ns = df[
|
||||
(df["att no show evidence"].fillna(-9999) != df["att no show invoice"].fillna(-9999)) &
|
||||
(df["att no show evidence"] != 0)
|
||||
].copy()
|
||||
att_ns["job_type"] = "ATT NO SHOW"
|
||||
filtered_dfs.append(att_ns)
|
||||
|
||||
|
||||
# Post visit no show
|
||||
# epc_ns = df[
|
||||
# df["post epc no show evidence"].fillna(-9999) != df["post epc no show invoice"].fillna(-9999)
|
||||
# ].copy()
|
||||
# epc_ns["job_type"] = "post EPC no show"
|
||||
# filtered_dfs.append(epc_ns)
|
||||
|
||||
final_df = pd.concat(filtered_dfs).reset_index(drop=True)
|
||||
|
||||
final_df[['address', 'client', 'job_type']]
|
||||
|
||||
final_df["job_type"] = final_df["job_type"].str.lower()
|
||||
rate_card_df["job_type"] = rate_card_df["job_type"].str.lower()
|
||||
|
||||
# Now perform the merge
|
||||
combined_with_rates = final_df.merge(rate_card_df, on="job_type", how="left")
|
||||
import datetime
|
||||
timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M')
|
||||
|
||||
attribute = ['address', 'client', 'job_type', 'rate']
|
||||
combined_with_rates[attribute].to_excel(f'NCHA Almshouses_{timestamp}.xlsx', index=False)
|
||||
267
etl/month_end_automation_wave_3_layout.py
Normal file
267
etl/month_end_automation_wave_3_layout.py
Normal file
|
|
@ -0,0 +1,267 @@
|
|||
# Wave 3's month end automation
|
||||
|
||||
from tqdm import tqdm
|
||||
from monday import MondayClient
|
||||
from etl.osmosis_complaince_address_to_files import get_all_items, extract_asset_ids
|
||||
from pprint import pprint
|
||||
import pandas as pd
|
||||
import json
|
||||
|
||||
monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY"
|
||||
monday = MondayClient(monday_key)
|
||||
board_ids = [
|
||||
# "9349630181", # WCHG Walkups-Operations
|
||||
# "8829428746", # 2502 Accent Housing
|
||||
# "8830772914", # "L&Q London"
|
||||
# "9601691730", # Cardo Wales & West - Wave 3
|
||||
"9660895490", # Northumberland County SHDF Wave 3
|
||||
]
|
||||
|
||||
empty = "Rate card info missing"
|
||||
rate_card_data_example = {
|
||||
"job_type": [
|
||||
"RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel",
|
||||
"Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2",
|
||||
"Post EPR", "Post EPC", "Post ATT", "retrofit evaluation",
|
||||
"RA no show", "ATT no show", "post EPC no show", "Full cost MTP", "measure modelling"
|
||||
],
|
||||
"rate": [
|
||||
259, 125, 280, 125, 125,
|
||||
650, 415, 195, 175, 135,
|
||||
120, "Post EPR Please Marianne", 85, 125, 60,
|
||||
25, 25, 25, "Mariann please input full cost mtp", "Marianne please input measure modelling"
|
||||
]
|
||||
}
|
||||
|
||||
|
||||
rate_card_data_2502_accent_housing = {
|
||||
"job_type": [
|
||||
"RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel",
|
||||
"Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2",
|
||||
"Post EPR", "Post EPC", "Post ATT", "retrofit evaluation",
|
||||
"RA no show", "ATT no show", "post EPC no show", "Full cost MTP", "measure modelling"
|
||||
],
|
||||
"rate": [
|
||||
empty, empty, empty, empty, empty,
|
||||
empty, empty, empty, empty, empty,
|
||||
empty, empty, empty, empty, empty,
|
||||
empty, empty, empty, 280, 150
|
||||
]
|
||||
}
|
||||
|
||||
rate_card_data_l_and_q_london = {
|
||||
"job_type": [
|
||||
"RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel",
|
||||
"Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2",
|
||||
"Post EPR", "Post EPC", "Post ATT", "retrofit evaluation",
|
||||
"RA no show", "ATT no show", "post EPC no show", "Full cost MTP", "measure modelling"
|
||||
],
|
||||
"rate": [
|
||||
empty, empty, empty, empty, empty,
|
||||
empty, empty, empty, empty, empty,
|
||||
empty, empty, empty, empty, empty,
|
||||
empty, empty, empty, 280, 150
|
||||
]
|
||||
}
|
||||
|
||||
|
||||
rate_card_data_northhumberland_country_shdf_wave_3 = {
|
||||
"job_type": [
|
||||
"RA", "ATT", "Coordination Stage 1 v1", "Coordination Stage 1 v2 remodel", "Coordination Stage 1 v3 remodel",
|
||||
"Design Archetype Complex", "Design Archetype Simple", "Design Repetitive Simple", "Coordination Stage 2", "Lodgement phase 1", "Full lodgement phase 2",
|
||||
"Post EPR", "Post EPC", "Post ATT", "retrofit evaluation",
|
||||
"RA no show", "ATT no show", "post EPC no show", "Full cost MTP", "measure modelling"
|
||||
],
|
||||
"rate": [
|
||||
empty, empty, empty, empty, empty,
|
||||
empty, empty, empty, empty, empty,
|
||||
empty, empty, empty, empty, empty,
|
||||
empty, empty, empty, 280, 150
|
||||
]
|
||||
}
|
||||
|
||||
|
||||
|
||||
# rate_card_df = pd.DataFrame(rate_card_data_example)
|
||||
# rate_card_df = pd.DataFrame(rate_card_data_2502_accent_housing)
|
||||
# rate_card_df = pd.DataFrame(rate_card_data_l_and_q_london)
|
||||
rate_card = pd.DataFrame(rate_card_data_northhumberland_country_shdf_wave_3)
|
||||
|
||||
|
||||
for board in tqdm(board_ids):
|
||||
print(f"working on board {board}")
|
||||
board_data = monday.boards.fetch_boards_by_id(board)
|
||||
columns = board_data["data"]["boards"][0]["columns"]
|
||||
col_id_map = {col["title"].lower(): col["id"] for col in columns}
|
||||
reversed_col_id_map = {v: k for k, v in col_id_map.items()}
|
||||
|
||||
|
||||
items = get_all_items(board, monday)
|
||||
|
||||
all_records = []
|
||||
for row in tqdm(items):
|
||||
data = {}
|
||||
data.update({"address": row['name']})
|
||||
data.update({"client": row['group']['title']})
|
||||
for col in row.get("column_values", []):
|
||||
if col.get("id") in reversed_col_id_map:
|
||||
if col.get("type") == "file":
|
||||
value = col.get("value")
|
||||
no_of_files = 0
|
||||
|
||||
if value:
|
||||
value = json.loads(col["value"])
|
||||
no_of_files = len(value.get('files', []))
|
||||
data.update({reversed_col_id_map[col.get("id")]: no_of_files})
|
||||
else:
|
||||
data.update({
|
||||
reversed_col_id_map[col.get("id")]: col.get("text")
|
||||
})
|
||||
all_records.append(data)
|
||||
|
||||
# Convert to DataFrame
|
||||
df = pd.DataFrame(all_records)
|
||||
|
||||
filtered_dfs = []
|
||||
|
||||
|
||||
def get_df(df, column_name, success_critera, job_name=None):
|
||||
_ = pd.DataFrame()
|
||||
if column_name in col_id_map:
|
||||
_ = df[
|
||||
df[column_name].str.lower().isin(success_critera)
|
||||
].copy()
|
||||
if job_name:
|
||||
_["job_type"] = job_name
|
||||
|
||||
|
||||
return _
|
||||
|
||||
|
||||
# RA
|
||||
ra = get_df(df, "ra invoicing status", ["to invoice"], "RA")
|
||||
if not ra.empty:
|
||||
filtered_dfs.append(ra)
|
||||
|
||||
|
||||
att = get_df(df, "post att invoicing status", ["to invoice"], "ATT")
|
||||
if not att.empty:
|
||||
filtered_dfs.append(att)
|
||||
|
||||
modeling = get_df(df, "mtp invoicing status", ["modelling to invoice"], "Measure Modelling")
|
||||
if not modeling.empty:
|
||||
filtered_dfs.append(modeling)
|
||||
|
||||
try:
|
||||
# Only needed for one board in wave 3
|
||||
full_cost = get_df(df, "mtp invoicing status", ["(V1) Full cost MTP to invoice (no previous modelling)".lower()], "full cost mtp")
|
||||
if not full_cost.empty:
|
||||
filtered_dfs(full_cost)
|
||||
except Exception as e:
|
||||
print(e)
|
||||
|
||||
v1 = get_df(df, "mtp invoicing status", ["(v1) ioe/mtp to invoice"], "Coordination Stage 1 v1")
|
||||
if not v1.empty:
|
||||
filtered_dfs.append(v1)
|
||||
|
||||
v2 = get_df(df, "mtp invoicing status", ["(v2) ioe/mtp to invoice"], "Coordination Stage 1 v2 remodel")
|
||||
if not v2.empty:
|
||||
filtered_dfs.append(v2)
|
||||
|
||||
v3 = get_df(df, "mtp invoicing status", ["(v3) ioe/mtp to invoice"], "Coordination Stage 1 v3 remodel")
|
||||
if not v3.empty:
|
||||
filtered_dfs.append(v3)
|
||||
|
||||
# Coordination stage 2 Please complete
|
||||
cors2 = get_df(df, "rc stage 2", ["to invoice"], "Coordination Stage 2")
|
||||
if not cors2.empty:
|
||||
filtered_dfs.append(cors2)
|
||||
|
||||
# Design archetype complex
|
||||
design = get_df(df, "design invoicing status", ["to invoice"])
|
||||
design1 = get_df(design, "design invoice type", ["archetype (complex)"], "Design Archetype Complex")
|
||||
if not design1.empty :
|
||||
filtered_dfs.append(design1)
|
||||
|
||||
# Design archetype simple
|
||||
design1 = get_df(design, "design invoice type", ["archetype (simple)"], "Design Archetype Simple")
|
||||
if not design1.empty:
|
||||
filtered_dfs.append(design1)
|
||||
|
||||
# Design repetitive simple
|
||||
design1 = get_df(design, "design invoice type", ["archetype (simple)"], "Design Archetype repetitive")
|
||||
if not design1.empty:
|
||||
filtered_dfs.append(design1)
|
||||
|
||||
# Design repetitive complex
|
||||
design1 = get_df(design, "design invoice type", ["archetype (complex)"], "Design Archetype complex")
|
||||
if not design1.empty:
|
||||
filtered_dfs.append(design1)
|
||||
|
||||
# Design Revision
|
||||
revision_letter = ['a', 'b', 'c', 'd']
|
||||
for letter in revision_letter:
|
||||
design = get_df(df, "design revision invoice", [f"rev. {letter} to invoice"], "Design Revision")
|
||||
if not design.empty:
|
||||
filtered_dfs.append(design)
|
||||
|
||||
# Lodgement Phase 1
|
||||
lodg1 = get_df(df, "lodgement phase 1 invoicing status", ["to invoice"], "Lodgement Phase 1")
|
||||
if not lodg1.empty:
|
||||
filtered_dfs(lodg1)
|
||||
|
||||
# Full Lodgement Phase
|
||||
lodg2 = get_df(df, "full lodgement invoicing status", ["to invoice"], "Full lodgement phase 2")
|
||||
if not lodg2.empty:
|
||||
filtered_dfs.append(lodg2)
|
||||
|
||||
# POST EPC
|
||||
post_epc = get_df(df, "post epc & eval. invoicing status", ["epc to invoice"], "POST EPC")
|
||||
if not post_epc.empty:
|
||||
filtered_dfs.append(post_epc)
|
||||
|
||||
|
||||
# POST EPR
|
||||
post_epr = get_df(df, "post epc & eval. invoicing status", ["epr to invoice"], "POST EPR")
|
||||
if not post_epr.empty:
|
||||
filtered_dfs.append(post_epr)
|
||||
|
||||
# post att
|
||||
post_att = get_df(df, "post att invoicing status", ["to invoice"], "POST ATT")
|
||||
if not post_att.empty:
|
||||
filtered_dfs.append(post_epc)
|
||||
|
||||
# Retrofit Evaluation
|
||||
rc = get_df(df, "rc stage 2 invoicing status", ["to invoice"], "retrofit evaluation")
|
||||
if not rc.empty:
|
||||
filtered_dfs.append(rc)
|
||||
|
||||
# RA NO Show
|
||||
ra_ns = get_df(df,"ra no show invoice", ["to invoice","to invoice (+1 previous no show)", "to invoice (+2 previous no shows)"], "RA NO SHOW")
|
||||
if not ra_ns.empty:
|
||||
filtered_dfs.append(ra_ns)
|
||||
|
||||
|
||||
# ATT NO Show
|
||||
att_ns = get_df(df, "pre att no show invoice", ["to invoice","to invoice (+1 previous no show)", "to invoice (+2 previous no shows)"], "ATT NO SHOW")
|
||||
if not att_ns.empty:
|
||||
filtered_dfs.append(att_ns)
|
||||
|
||||
|
||||
# Post visit no show
|
||||
epc_ns = get_df(df, "post works no show invoice", ["to invoice","to invoice (+1 previous no show)", "to invoice (+2 previous no shows)"], "post EPC NO SHOW")
|
||||
if not epc_ns.empty:
|
||||
filtered_dfs.append(epc_ns)
|
||||
|
||||
final_df = pd.concat(filtered_dfs).reset_index(drop=True)
|
||||
|
||||
final_df["job_type"] = final_df["job_type"].str.lower()
|
||||
rate_card_df["job_type"] = rate_card_df["job_type"].str.lower()
|
||||
|
||||
# Now perform the merge
|
||||
combined_with_rates = final_df.merge(rate_card_df, on="job_type", how="left")
|
||||
import datetime
|
||||
timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M')
|
||||
|
||||
attribute = ['address', 'client', 'job_type', 'rate']
|
||||
combined_with_rates[attribute].to_excel(f'L&Q London {timestamp}.xlsx', index=False)
|
||||
|
|
@ -91,7 +91,7 @@ def get_all_items(board_id, monday):
|
|||
limit = 25 # Adjust the limit based on how many items you want per request
|
||||
all_items = [] # List to store all fetched items
|
||||
cursor = None # Start without a cursor for the first page
|
||||
|
||||
print(f"Connecting to Monday API and retrieving data for board {board_id}")
|
||||
# Loop through pages
|
||||
while True:
|
||||
# Fetch items for the current page
|
||||
|
|
@ -116,8 +116,7 @@ def get_all_items(board_id, monday):
|
|||
# If there's no cursor, we've reached the last page
|
||||
if not cursor:
|
||||
break
|
||||
print(f"cursor {cursor}")
|
||||
print(f"len all_itemms {len(all_items)}")
|
||||
print("Loading...")
|
||||
return all_items
|
||||
|
||||
def upload_to_sharepoint(to_upload, master_folder_name):
|
||||
|
|
@ -128,36 +127,37 @@ def upload_to_sharepoint(to_upload, master_folder_name):
|
|||
print(f"Uploading {file_name} to sharepoint")
|
||||
osmosis.upload_file(file_path, parent_folder + f"/{master_folder_name}", file_name)
|
||||
|
||||
# Step 1: Fetch column IDs
|
||||
board_data = monday.boards.fetch_boards_by_id(board_id)
|
||||
columns = board_data["data"]["boards"][0]["columns"]
|
||||
col_id_map = {col["title"].lower(): col["id"] for col in columns}
|
||||
if __name__ == "__main__":
|
||||
# Step 1: Fetch column IDs
|
||||
board_data = monday.boards.fetch_boards_by_id(board_id)
|
||||
columns = board_data["data"]["boards"][0]["columns"]
|
||||
col_id_map = {col["title"].lower(): col["id"] for col in columns}
|
||||
|
||||
name_id = col_id_map.get("name") # Replace with actual title if different
|
||||
files_id = col_id_map.get("file(s)") # Replace with actual title if different
|
||||
name_id = col_id_map.get("name") # Replace with actual title if different
|
||||
files_id = col_id_map.get("file(s)") # Replace with actual title if different
|
||||
|
||||
if not name_id or not files_id:
|
||||
raise Exception("Could not find 'name' or 'file(s)' columns")
|
||||
if not name_id or not files_id:
|
||||
raise Exception("Could not find 'name' or 'file(s)' columns")
|
||||
|
||||
items = get_all_items(board_id, monday)
|
||||
for i,item in enumerate(tqdm(items)):
|
||||
if i>329:
|
||||
item_name = item["name"]
|
||||
item_name = sanitize_name(item_name, ignore_dot=True)
|
||||
print(f"Item name is {item_name}")
|
||||
asset_ids = extract_asset_ids(item, files_id)
|
||||
items = get_all_items(board_id, monday)
|
||||
for i,item in enumerate(tqdm(items)):
|
||||
if i>329:
|
||||
item_name = item["name"]
|
||||
item_name = sanitize_name(item_name, ignore_dot=True)
|
||||
print(f"Item name is {item_name}")
|
||||
asset_ids = extract_asset_ids(item, files_id)
|
||||
|
||||
to_upload = []
|
||||
for asset_id in asset_ids:
|
||||
try:
|
||||
public_url, file_name = get_public_url(asset_id)
|
||||
print(f"Downloading {file_name}")
|
||||
file_path = download_file_from_public_url(public_url, file_name)
|
||||
to_upload.append(file_path)
|
||||
except Exception as e:
|
||||
print(f"Failed to download/upload asset {asset_id}: {e}")
|
||||
to_upload = []
|
||||
for asset_id in asset_ids:
|
||||
try:
|
||||
public_url, file_name = get_public_url(asset_id)
|
||||
print(f"Downloading {file_name}")
|
||||
file_path = download_file_from_public_url(public_url, file_name)
|
||||
to_upload.append(file_path)
|
||||
except Exception as e:
|
||||
print(f"Failed to download/upload asset {asset_id}: {e}")
|
||||
|
||||
if to_upload:
|
||||
upload_to_sharepoint(to_upload, item_name)
|
||||
if to_upload:
|
||||
upload_to_sharepoint(to_upload, item_name)
|
||||
|
||||
# Liv green # Cocuun # Wates
|
||||
# Liv green # Cocuun # Wates
|
||||
Binary file not shown.
|
|
@ -8,57 +8,81 @@ from etl.scraper.scraper import SharePointInstaller
|
|||
from etl.scraper.scraper import SharePointScraper
|
||||
import pandas as pd
|
||||
from tqdm import tqdm
|
||||
import time
|
||||
|
||||
|
||||
osmosis = SharePointScraper(SharePointInstaller.OSMOSIS_WAVE_2)
|
||||
osmosis = SharePointScraper(SharePointInstaller.OSMOSIS_WAVE_3)
|
||||
|
||||
|
||||
parent_folder = "/Osmosis ACD/Osmosis ACD Projects/WCHG/WCHG Walkups/Property Folders"
|
||||
parent_folder = "/Osmosis-ACD Projects/Cardo/Cardo (Wales & West)/2506 Cardo Property Folders"
|
||||
|
||||
asset_list = pd.read_excel("osmosis_data/asset_list.xlsx", sheet_name="Sheet1")
|
||||
asset_list = pd.read_excel("osmosis_data/asset_list.xlsx", sheet_name="Sheet 1")
|
||||
|
||||
|
||||
new_asset_list = []
|
||||
# Create asset list and location
|
||||
for index, address in tqdm(asset_list.iterrows()):
|
||||
folder_name = address['Name'] + " " + address['Postcode']
|
||||
webUrl = osmosis.create_dir(folder_name, parent_folder)
|
||||
|
||||
first_folder = "1. Retrofit Assessment"
|
||||
osmosis.create_dir(first_folder, parent_folder + f"/{folder_name}")
|
||||
osmosis.create_dir("A. Assessment", parent_folder + f"/{folder_name}/{first_folder}")
|
||||
osmosis.create_dir("B. Air Tightness Tests", parent_folder + f"/{folder_name}/{first_folder}")
|
||||
if index > 39:
|
||||
folder_name = address['Name'] + " " + address['Postcode']
|
||||
webUrl = osmosis.create_dir(folder_name, parent_folder)
|
||||
time.sleep(1)
|
||||
print(f"building folders insidea {folder_name}")
|
||||
|
||||
second_folder = "2. RC Mid-Term Plan"
|
||||
osmosis.create_dir(second_folder, parent_folder + f"/{folder_name}")
|
||||
osmosis.create_dir("SAP", parent_folder + f"/{folder_name}/{second_folder}")
|
||||
print("building retrofit assessment")
|
||||
first_folder = "1. Retrofit Assessment"
|
||||
osmosis.create_dir(first_folder, parent_folder + f"/{folder_name}")
|
||||
osmosis.create_dir("A. Assessment", parent_folder + f"/{folder_name}/{first_folder}")
|
||||
osmosis.create_dir("B. Air Tightness Tests", parent_folder + f"/{folder_name}/{first_folder}")
|
||||
|
||||
third_folder = "3. Retrofit Design"
|
||||
osmosis.create_dir(third_folder, parent_folder + f"/{folder_name}")
|
||||
print("building RC MID Term plan")
|
||||
second_folder = "2. RC Mid-Term Plan"
|
||||
osmosis.create_dir(second_folder, parent_folder + f"/{folder_name}")
|
||||
osmosis.create_dir("SAP", parent_folder + f"/{folder_name}/{second_folder}")
|
||||
|
||||
fourth_folder = "4. Post EPC"
|
||||
osmosis.create_dir(fourth_folder, parent_folder + f"/{folder_name}")
|
||||
osmosis.create_dir(f"{address['Name']} - POST EPC Photos", parent_folder + f"/{folder_name}/{fourth_folder}")
|
||||
print("building Retrofit Design")
|
||||
third_folder = "3. Retrofit Design"
|
||||
osmosis.create_dir(third_folder, parent_folder + f"/{folder_name}")
|
||||
|
||||
fifth_folder = "5. Trustmark Lodgement"
|
||||
osmosis.create_dir(fifth_folder, parent_folder + f"/{folder_name}")
|
||||
osmosis.create_dir("1. Works", parent_folder + f"/{folder_name}/{fifth_folder}")
|
||||
print("building post epc")
|
||||
fourth_folder = "4. Post EPC"
|
||||
osmosis.create_dir(fourth_folder, parent_folder + f"/{folder_name}")
|
||||
osmosis.create_dir(f"{address['Name']} - POST EPC Photos", parent_folder + f"/{folder_name}/{fourth_folder}")
|
||||
|
||||
osmosis.create_dir("2. Required Documents", parent_folder + f"/{folder_name}/{fifth_folder}")
|
||||
osmosis.create_dir("3. Additional Documents", parent_folder + f"/{folder_name}/{fifth_folder}")
|
||||
|
||||
asset_data = {
|
||||
"Name": address['Name'],
|
||||
"Postcode": address['Postcode'],
|
||||
"Sharepoint": webUrl,
|
||||
}
|
||||
print("Building Trust mark Lodgement")
|
||||
fifth_folder = "5. Trustmark Lodgement"
|
||||
osmosis.create_dir(fifth_folder, parent_folder + f"/{folder_name}")
|
||||
osmosis.create_dir("1. Works", parent_folder + f"/{folder_name}/{fifth_folder}")
|
||||
|
||||
new_asset_list.append(asset_data)
|
||||
osmosis.create_dir("2. Required Documents", parent_folder + f"/{folder_name}/{fifth_folder}")
|
||||
osmosis.create_dir("3. Additional Documents", parent_folder + f"/{folder_name}/{fifth_folder}")
|
||||
|
||||
asset_data = {
|
||||
"Name": address['Name'],
|
||||
"Postcode": address['Postcode'],
|
||||
"Sharepoint": webUrl,
|
||||
}
|
||||
print(asset_data)
|
||||
|
||||
new_asset_list.append(asset_data)
|
||||
|
||||
|
||||
# Osmosist File strucutre
|
||||
# Run this is you just want to get url
|
||||
def just_url(asset_list):
|
||||
new_asset_list = []
|
||||
for index, address in tqdm(asset_list.iterrows()):
|
||||
folder_name = address['Name'] + " " + address['Postcode']
|
||||
webUrl = osmosis.create_dir(folder_name, parent_folder)
|
||||
asset_data = {
|
||||
"Name": address['Name'],
|
||||
"Postcode": address['Postcode'],
|
||||
"Sharepoint": webUrl,
|
||||
}
|
||||
print(asset_data)
|
||||
|
||||
new_asset_list.append(asset_data)
|
||||
return new_asset_list
|
||||
|
||||
new_asset_list = just_url(asset_list=asset_list)
|
||||
df = pd.DataFrame(new_asset_list)
|
||||
df.to_csv("output.csv", index=False)
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,53 @@
|
|||
import os
|
||||
from pprint import pprint
|
||||
|
||||
os.environ["SHAREPOINT_CLIENT_ID"] = "895e3b77-b1d7-43ec-b18f-dcfe07cdfeaf"
|
||||
os.environ["SHAREPOINT_CLIENT_SECRET"] = "SOf8Q~-is4wdQiqvEEm9FlJQRAY9ELGaj5Qz-a6E"
|
||||
os.environ["SHAREPOINT_TENANT_ID"] = "c3f7519c-2719-4547-af04-6da6cbfd8f8f"
|
||||
os.environ["SOUTH_COAST_INSULATION_SERVICE_SHAREPOINT_ID"] = "b5a51507-9427-4ee0-b03e-90ec7681e2d3"
|
||||
os.environ["JJC_SERVICE_SHAREPOINT_ID"] = "7fdd0485-bbf3-4b29-b30f-98c81c2a6284"
|
||||
|
||||
from etl.hubSpotClient.hubspot import DealStage, HubSpotClient
|
||||
from etl.surveyedData.surveryedData import surveyedDataProcessor
|
||||
from etl.scraper.scraper import SharePointScraper, SharePointInstaller
|
||||
from etl.utils.utils import get_sharepoint_path
|
||||
|
||||
def string_to_installer(installer):
|
||||
if installer.upper() == "J & J CRUMP":
|
||||
return SharePointInstaller.JJC
|
||||
elif installer.upper() == "SCIS":
|
||||
return SharePointInstaller.SOUTH_COAST_INSULATION
|
||||
elif installer.upper() == "SGEC":
|
||||
return SharePointInstaller.JJC
|
||||
else:
|
||||
return None
|
||||
|
||||
# Local development
|
||||
os.environ["DATABASE_URL"] = "postgresql://postgres:makingwarmhomes@db:5432/postgres"
|
||||
|
||||
hubspotClient = HubSpotClient()
|
||||
|
||||
# Gets all deals and puts it into a SubmissionInfoFromDeal class
|
||||
# KHALIM - I ADDED A SCRIPT TO ONLY DOWNLOAD 1 deal for speed sake
|
||||
deals = hubspotClient.get_deals_from_deal_stage(DealStage.SURVEYED_COMPLETE_NEEDS_SIGN_OFF)
|
||||
|
||||
|
||||
for deal in deals:
|
||||
sharepoint_url = deal.submission_folder_path
|
||||
installer = string_to_installer(deal.installer)
|
||||
sp = SharePointScraper(installer)
|
||||
path = get_sharepoint_path(sharepoint_url)
|
||||
|
||||
files = sp.downloadt_files_from_path(path)
|
||||
sdp = surveyedDataProcessor("fake address", files)
|
||||
|
||||
# Class Object for EPR Summary Informaiton ( Transform )
|
||||
sdp.epr_summary_information
|
||||
|
||||
# File path to epr
|
||||
sdp.epr_summary_information_file_path
|
||||
|
||||
break
|
||||
|
||||
|
||||
|
||||
44
etl/read_stuff_from_s3_example.py
Normal file
44
etl/read_stuff_from_s3_example.py
Normal file
|
|
@ -0,0 +1,44 @@
|
|||
import boto3
|
||||
import os
|
||||
|
||||
|
||||
def print_hello_from_etl_module():
|
||||
print("You are printing from a etl module we made in poetry")
|
||||
|
||||
def split_s3_url(s3_url):
|
||||
if not s3_url.startswith("s3://"):
|
||||
raise ValueError("Invalid S3 URL. Must start with 's3://'")
|
||||
|
||||
path = s3_url[5:]
|
||||
parts = path.split('/', 1)
|
||||
|
||||
if len(parts) != 2:
|
||||
raise ValueError("S3 URL must include a key after the bucket name")
|
||||
return parts[0], parts[1]
|
||||
|
||||
def create_temp_file(content_bytes, relative_path):
|
||||
# Save under /tmp/s3/
|
||||
full_path = os.path.join("/tmp/s3", relative_path)
|
||||
|
||||
# Make sure the directory exists
|
||||
os.makedirs(os.path.dirname(full_path), exist_ok=True)
|
||||
|
||||
# Write content to file
|
||||
with open(full_path, 'wb') as temp_file:
|
||||
temp_file.write(content_bytes)
|
||||
|
||||
print(f"Temporary file created at: {full_path}")
|
||||
return full_path
|
||||
|
||||
def download_data_from_s3(s3_uri):
|
||||
s3 = boto3.resource('s3')
|
||||
bucket_name, file_name = split_s3_url(s3_uri)
|
||||
|
||||
obj = s3.Object(bucket_name, file_name)
|
||||
data = obj.get()['Body'].read()
|
||||
|
||||
# Save using full S3 key as relative path
|
||||
return create_temp_file(data, file_name)
|
||||
|
||||
# Example usage
|
||||
# download_data_from_s3("s3://retrofit-energy-assessments-dev/JAFFERSONS ENERGY CONSULTANTS/VDE001/12103116/docs & plans/77 Perryn Road, W3 7LT EPR.pdf")
|
||||
|
|
@ -43,6 +43,7 @@ class surveyedDataProcessor():
|
|||
self.hubspot_deal_id = None
|
||||
self.epr_with_data = None
|
||||
self.epr_summary_information = None
|
||||
self.epr_summary_information_file_path = None
|
||||
self.full_sap_xml = None
|
||||
self.lig_sap_xml = None
|
||||
self.rd_sap_xml = None
|
||||
|
|
@ -69,6 +70,7 @@ class surveyedDataProcessor():
|
|||
self.epr_with_data = pdf.get_reader()
|
||||
elif pdf.type == ReportType.ENERGY_PERFORMANCE_REPORT_SUMMARY_INFORMATION:
|
||||
self.epr_summary_information = pdf.get_reader()
|
||||
self.epr_summary_information_file_path = file
|
||||
|
||||
elif file.lower().endswith('.xml'):
|
||||
xml = xmlReader(file)
|
||||
|
|
|
|||
152
poetry.lock
generated
152
poetry.lock
generated
|
|
@ -84,6 +84,46 @@ charset-normalizer = ["charset-normalizer"]
|
|||
html5lib = ["html5lib"]
|
||||
lxml = ["lxml"]
|
||||
|
||||
[[package]]
|
||||
name = "boto3"
|
||||
version = "1.39.6"
|
||||
description = "The AWS SDK for Python"
|
||||
optional = false
|
||||
python-versions = ">=3.9"
|
||||
groups = ["main"]
|
||||
files = [
|
||||
{file = "boto3-1.39.6-py3-none-any.whl", hash = "sha256:db965dc9019df7b1d20e8d8ab7a653956f275865175a8652419ebfd03de03d83"},
|
||||
{file = "boto3-1.39.6.tar.gz", hash = "sha256:e75bfcd444e199767642f28ef8dc4f972846dc3118e48a7e09f9c458dae2021e"},
|
||||
]
|
||||
|
||||
[package.dependencies]
|
||||
botocore = ">=1.39.6,<1.40.0"
|
||||
jmespath = ">=0.7.1,<2.0.0"
|
||||
s3transfer = ">=0.13.0,<0.14.0"
|
||||
|
||||
[package.extras]
|
||||
crt = ["botocore[crt] (>=1.21.0,<2.0a0)"]
|
||||
|
||||
[[package]]
|
||||
name = "botocore"
|
||||
version = "1.39.6"
|
||||
description = "Low-level, data-driven core of boto 3."
|
||||
optional = false
|
||||
python-versions = ">=3.9"
|
||||
groups = ["main"]
|
||||
files = [
|
||||
{file = "botocore-1.39.6-py3-none-any.whl", hash = "sha256:9c002724e9b97cec610dbbb3bb019b3248ff6bf58407835621f0461e740af90b"},
|
||||
{file = "botocore-1.39.6.tar.gz", hash = "sha256:d3a6c207d233ddee3289c1d56646047bef18b21a1faebb3d83a6fca149fd0f59"},
|
||||
]
|
||||
|
||||
[package.dependencies]
|
||||
jmespath = ">=0.7.1,<2.0.0"
|
||||
python-dateutil = ">=2.1,<3.0.0"
|
||||
urllib3 = {version = ">=1.25.4,<2.2.0 || >2.2.0,<3", markers = "python_version >= \"3.10\""}
|
||||
|
||||
[package.extras]
|
||||
crt = ["awscrt (==0.23.8)"]
|
||||
|
||||
[[package]]
|
||||
name = "certifi"
|
||||
version = "2025.4.26"
|
||||
|
|
@ -717,6 +757,18 @@ docs = ["Jinja2 (==2.11.3)", "MarkupSafe (==1.1.1)", "Pygments (==2.8.1)", "alab
|
|||
qa = ["flake8 (==5.0.4)", "mypy (==0.971)", "types-setuptools (==67.2.0.1)"]
|
||||
testing = ["Django", "attrs", "colorama", "docopt", "pytest (<9.0.0)"]
|
||||
|
||||
[[package]]
|
||||
name = "jmespath"
|
||||
version = "1.0.1"
|
||||
description = "JSON Matching Expressions"
|
||||
optional = false
|
||||
python-versions = ">=3.7"
|
||||
groups = ["main"]
|
||||
files = [
|
||||
{file = "jmespath-1.0.1-py3-none-any.whl", hash = "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980"},
|
||||
{file = "jmespath-1.0.1.tar.gz", hash = "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe"},
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "jupyter-client"
|
||||
version = "8.6.3"
|
||||
|
|
@ -1191,23 +1243,81 @@ dev = ["abi3audit", "black (==24.10.0)", "check-manifest", "coverage", "packagin
|
|||
test = ["pytest", "pytest-xdist", "setuptools"]
|
||||
|
||||
[[package]]
|
||||
name = "psycopg2"
|
||||
name = "psycopg2-binary"
|
||||
version = "2.9.10"
|
||||
description = "psycopg2 - Python-PostgreSQL Database Adapter"
|
||||
optional = false
|
||||
python-versions = ">=3.8"
|
||||
groups = ["main"]
|
||||
files = [
|
||||
{file = "psycopg2-2.9.10-cp310-cp310-win32.whl", hash = "sha256:5df2b672140f95adb453af93a7d669d7a7bf0a56bcd26f1502329166f4a61716"},
|
||||
{file = "psycopg2-2.9.10-cp310-cp310-win_amd64.whl", hash = "sha256:c6f7b8561225f9e711a9c47087388a97fdc948211c10a4bccbf0ba68ab7b3b5a"},
|
||||
{file = "psycopg2-2.9.10-cp311-cp311-win32.whl", hash = "sha256:47c4f9875125344f4c2b870e41b6aad585901318068acd01de93f3677a6522c2"},
|
||||
{file = "psycopg2-2.9.10-cp311-cp311-win_amd64.whl", hash = "sha256:0435034157049f6846e95103bd8f5a668788dd913a7c30162ca9503fdf542cb4"},
|
||||
{file = "psycopg2-2.9.10-cp312-cp312-win32.whl", hash = "sha256:65a63d7ab0e067e2cdb3cf266de39663203d38d6a8ed97f5ca0cb315c73fe067"},
|
||||
{file = "psycopg2-2.9.10-cp312-cp312-win_amd64.whl", hash = "sha256:4a579d6243da40a7b3182e0430493dbd55950c493d8c68f4eec0b302f6bbf20e"},
|
||||
{file = "psycopg2-2.9.10-cp313-cp313-win_amd64.whl", hash = "sha256:91fd603a2155da8d0cfcdbf8ab24a2d54bca72795b90d2a3ed2b6da8d979dee2"},
|
||||
{file = "psycopg2-2.9.10-cp39-cp39-win32.whl", hash = "sha256:9d5b3b94b79a844a986d029eee38998232451119ad653aea42bb9220a8c5066b"},
|
||||
{file = "psycopg2-2.9.10-cp39-cp39-win_amd64.whl", hash = "sha256:88138c8dedcbfa96408023ea2b0c369eda40fe5d75002c0964c78f46f11fa442"},
|
||||
{file = "psycopg2-2.9.10.tar.gz", hash = "sha256:12ec0b40b0273f95296233e8750441339298e6a572f7039da5b260e3c8b60e11"},
|
||||
{file = "psycopg2-binary-2.9.10.tar.gz", hash = "sha256:4b3df0e6990aa98acda57d983942eff13d824135fe2250e6522edaa782a06de2"},
|
||||
{file = "psycopg2_binary-2.9.10-cp310-cp310-macosx_12_0_x86_64.whl", hash = "sha256:0ea8e3d0ae83564f2fc554955d327fa081d065c8ca5cc6d2abb643e2c9c1200f"},
|
||||
{file = "psycopg2_binary-2.9.10-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:3e9c76f0ac6f92ecfc79516a8034a544926430f7b080ec5a0537bca389ee0906"},
|
||||
{file = "psycopg2_binary-2.9.10-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2ad26b467a405c798aaa1458ba09d7e2b6e5f96b1ce0ac15d82fd9f95dc38a92"},
|
||||
{file = "psycopg2_binary-2.9.10-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:270934a475a0e4b6925b5f804e3809dd5f90f8613621d062848dd82f9cd62007"},
|
||||
{file = "psycopg2_binary-2.9.10-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:48b338f08d93e7be4ab2b5f1dbe69dc5e9ef07170fe1f86514422076d9c010d0"},
|
||||
{file = "psycopg2_binary-2.9.10-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7f4152f8f76d2023aac16285576a9ecd2b11a9895373a1f10fd9db54b3ff06b4"},
|
||||
{file = "psycopg2_binary-2.9.10-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:32581b3020c72d7a421009ee1c6bf4a131ef5f0a968fab2e2de0c9d2bb4577f1"},
|
||||
{file = "psycopg2_binary-2.9.10-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:2ce3e21dc3437b1d960521eca599d57408a695a0d3c26797ea0f72e834c7ffe5"},
|
||||
{file = "psycopg2_binary-2.9.10-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:e984839e75e0b60cfe75e351db53d6db750b00de45644c5d1f7ee5d1f34a1ce5"},
|
||||
{file = "psycopg2_binary-2.9.10-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:3c4745a90b78e51d9ba06e2088a2fe0c693ae19cc8cb051ccda44e8df8a6eb53"},
|
||||
{file = "psycopg2_binary-2.9.10-cp310-cp310-win32.whl", hash = "sha256:e5720a5d25e3b99cd0dc5c8a440570469ff82659bb09431c1439b92caf184d3b"},
|
||||
{file = "psycopg2_binary-2.9.10-cp310-cp310-win_amd64.whl", hash = "sha256:3c18f74eb4386bf35e92ab2354a12c17e5eb4d9798e4c0ad3a00783eae7cd9f1"},
|
||||
{file = "psycopg2_binary-2.9.10-cp311-cp311-macosx_12_0_x86_64.whl", hash = "sha256:04392983d0bb89a8717772a193cfaac58871321e3ec69514e1c4e0d4957b5aff"},
|
||||
{file = "psycopg2_binary-2.9.10-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:1a6784f0ce3fec4edc64e985865c17778514325074adf5ad8f80636cd029ef7c"},
|
||||
{file = "psycopg2_binary-2.9.10-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b5f86c56eeb91dc3135b3fd8a95dc7ae14c538a2f3ad77a19645cf55bab1799c"},
|
||||
{file = "psycopg2_binary-2.9.10-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2b3d2491d4d78b6b14f76881905c7a8a8abcf974aad4a8a0b065273a0ed7a2cb"},
|
||||
{file = "psycopg2_binary-2.9.10-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2286791ececda3a723d1910441c793be44625d86d1a4e79942751197f4d30341"},
|
||||
{file = "psycopg2_binary-2.9.10-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:512d29bb12608891e349af6a0cccedce51677725a921c07dba6342beaf576f9a"},
|
||||
{file = "psycopg2_binary-2.9.10-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:5a507320c58903967ef7384355a4da7ff3f28132d679aeb23572753cbf2ec10b"},
|
||||
{file = "psycopg2_binary-2.9.10-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:6d4fa1079cab9018f4d0bd2db307beaa612b0d13ba73b5c6304b9fe2fb441ff7"},
|
||||
{file = "psycopg2_binary-2.9.10-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:851485a42dbb0bdc1edcdabdb8557c09c9655dfa2ca0460ff210522e073e319e"},
|
||||
{file = "psycopg2_binary-2.9.10-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:35958ec9e46432d9076286dda67942ed6d968b9c3a6a2fd62b48939d1d78bf68"},
|
||||
{file = "psycopg2_binary-2.9.10-cp311-cp311-win32.whl", hash = "sha256:ecced182e935529727401b24d76634a357c71c9275b356efafd8a2a91ec07392"},
|
||||
{file = "psycopg2_binary-2.9.10-cp311-cp311-win_amd64.whl", hash = "sha256:ee0e8c683a7ff25d23b55b11161c2663d4b099770f6085ff0a20d4505778d6b4"},
|
||||
{file = "psycopg2_binary-2.9.10-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:880845dfe1f85d9d5f7c412efea7a08946a46894537e4e5d091732eb1d34d9a0"},
|
||||
{file = "psycopg2_binary-2.9.10-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:9440fa522a79356aaa482aa4ba500b65f28e5d0e63b801abf6aa152a29bd842a"},
|
||||
{file = "psycopg2_binary-2.9.10-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e3923c1d9870c49a2d44f795df0c889a22380d36ef92440ff618ec315757e539"},
|
||||
{file = "psycopg2_binary-2.9.10-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7b2c956c028ea5de47ff3a8d6b3cc3330ab45cf0b7c3da35a2d6ff8420896526"},
|
||||
{file = "psycopg2_binary-2.9.10-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f758ed67cab30b9a8d2833609513ce4d3bd027641673d4ebc9c067e4d208eec1"},
|
||||
{file = "psycopg2_binary-2.9.10-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8cd9b4f2cfab88ed4a9106192de509464b75a906462fb846b936eabe45c2063e"},
|
||||
{file = "psycopg2_binary-2.9.10-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:6dc08420625b5a20b53551c50deae6e231e6371194fa0651dbe0fb206452ae1f"},
|
||||
{file = "psycopg2_binary-2.9.10-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:d7cd730dfa7c36dbe8724426bf5612798734bff2d3c3857f36f2733f5bfc7c00"},
|
||||
{file = "psycopg2_binary-2.9.10-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:155e69561d54d02b3c3209545fb08938e27889ff5a10c19de8d23eb5a41be8a5"},
|
||||
{file = "psycopg2_binary-2.9.10-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c3cc28a6fd5a4a26224007712e79b81dbaee2ffb90ff406256158ec4d7b52b47"},
|
||||
{file = "psycopg2_binary-2.9.10-cp312-cp312-win32.whl", hash = "sha256:ec8a77f521a17506a24a5f626cb2aee7850f9b69a0afe704586f63a464f3cd64"},
|
||||
{file = "psycopg2_binary-2.9.10-cp312-cp312-win_amd64.whl", hash = "sha256:18c5ee682b9c6dd3696dad6e54cc7ff3a1a9020df6a5c0f861ef8bfd338c3ca0"},
|
||||
{file = "psycopg2_binary-2.9.10-cp313-cp313-macosx_12_0_x86_64.whl", hash = "sha256:26540d4a9a4e2b096f1ff9cce51253d0504dca5a85872c7f7be23be5a53eb18d"},
|
||||
{file = "psycopg2_binary-2.9.10-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:e217ce4d37667df0bc1c397fdcd8de5e81018ef305aed9415c3b093faaeb10fb"},
|
||||
{file = "psycopg2_binary-2.9.10-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:245159e7ab20a71d989da00f280ca57da7641fa2cdcf71749c193cea540a74f7"},
|
||||
{file = "psycopg2_binary-2.9.10-cp313-cp313-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3c4ded1a24b20021ebe677b7b08ad10bf09aac197d6943bfe6fec70ac4e4690d"},
|
||||
{file = "psycopg2_binary-2.9.10-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3abb691ff9e57d4a93355f60d4f4c1dd2d68326c968e7db17ea96df3c023ef73"},
|
||||
{file = "psycopg2_binary-2.9.10-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8608c078134f0b3cbd9f89b34bd60a943b23fd33cc5f065e8d5f840061bd0673"},
|
||||
{file = "psycopg2_binary-2.9.10-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:230eeae2d71594103cd5b93fd29d1ace6420d0b86f4778739cb1a5a32f607d1f"},
|
||||
{file = "psycopg2_binary-2.9.10-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:bb89f0a835bcfc1d42ccd5f41f04870c1b936d8507c6df12b7737febc40f0909"},
|
||||
{file = "psycopg2_binary-2.9.10-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:f0c2d907a1e102526dd2986df638343388b94c33860ff3bbe1384130828714b1"},
|
||||
{file = "psycopg2_binary-2.9.10-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:f8157bed2f51db683f31306aa497311b560f2265998122abe1dce6428bd86567"},
|
||||
{file = "psycopg2_binary-2.9.10-cp313-cp313-win_amd64.whl", hash = "sha256:27422aa5f11fbcd9b18da48373eb67081243662f9b46e6fd07c3eb46e4535142"},
|
||||
{file = "psycopg2_binary-2.9.10-cp38-cp38-macosx_12_0_x86_64.whl", hash = "sha256:eb09aa7f9cecb45027683bb55aebaaf45a0df8bf6de68801a6afdc7947bb09d4"},
|
||||
{file = "psycopg2_binary-2.9.10-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b73d6d7f0ccdad7bc43e6d34273f70d587ef62f824d7261c4ae9b8b1b6af90e8"},
|
||||
{file = "psycopg2_binary-2.9.10-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ce5ab4bf46a211a8e924d307c1b1fcda82368586a19d0a24f8ae166f5c784864"},
|
||||
{file = "psycopg2_binary-2.9.10-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:056470c3dc57904bbf63d6f534988bafc4e970ffd50f6271fc4ee7daad9498a5"},
|
||||
{file = "psycopg2_binary-2.9.10-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:73aa0e31fa4bb82578f3a6c74a73c273367727de397a7a0f07bd83cbea696baa"},
|
||||
{file = "psycopg2_binary-2.9.10-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:8de718c0e1c4b982a54b41779667242bc630b2197948405b7bd8ce16bcecac92"},
|
||||
{file = "psycopg2_binary-2.9.10-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:5c370b1e4975df846b0277b4deba86419ca77dbc25047f535b0bb03d1a544d44"},
|
||||
{file = "psycopg2_binary-2.9.10-cp38-cp38-musllinux_1_2_ppc64le.whl", hash = "sha256:ffe8ed017e4ed70f68b7b371d84b7d4a790368db9203dfc2d222febd3a9c8863"},
|
||||
{file = "psycopg2_binary-2.9.10-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:8aecc5e80c63f7459a1a2ab2c64df952051df196294d9f739933a9f6687e86b3"},
|
||||
{file = "psycopg2_binary-2.9.10-cp39-cp39-macosx_12_0_x86_64.whl", hash = "sha256:7a813c8bdbaaaab1f078014b9b0b13f5de757e2b5d9be6403639b298a04d218b"},
|
||||
{file = "psycopg2_binary-2.9.10-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d00924255d7fc916ef66e4bf22f354a940c67179ad3fd7067d7a0a9c84d2fbfc"},
|
||||
{file = "psycopg2_binary-2.9.10-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7559bce4b505762d737172556a4e6ea8a9998ecac1e39b5233465093e8cee697"},
|
||||
{file = "psycopg2_binary-2.9.10-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e8b58f0a96e7a1e341fc894f62c1177a7c83febebb5ff9123b579418fdc8a481"},
|
||||
{file = "psycopg2_binary-2.9.10-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6b269105e59ac96aba877c1707c600ae55711d9dcd3fc4b5012e4af68e30c648"},
|
||||
{file = "psycopg2_binary-2.9.10-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:79625966e176dc97ddabc142351e0409e28acf4660b88d1cf6adb876d20c490d"},
|
||||
{file = "psycopg2_binary-2.9.10-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:8aabf1c1a04584c168984ac678a668094d831f152859d06e055288fa515e4d30"},
|
||||
{file = "psycopg2_binary-2.9.10-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:19721ac03892001ee8fdd11507e6a2e01f4e37014def96379411ca99d78aeb2c"},
|
||||
{file = "psycopg2_binary-2.9.10-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:7f5d859928e635fa3ce3477704acee0f667b3a3d3e4bb109f2b18d4005f38287"},
|
||||
{file = "psycopg2_binary-2.9.10-cp39-cp39-win32.whl", hash = "sha256:3216ccf953b3f267691c90c6fe742e45d890d8272326b4a8b20850a03d05b7b8"},
|
||||
{file = "psycopg2_binary-2.9.10-cp39-cp39-win_amd64.whl", hash = "sha256:30e34c4e97964805f715206c7b789d54a78b70f3ff19fbe590104b71c45600e5"},
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -1682,6 +1792,24 @@ urllib3 = ">=1.21.1,<3"
|
|||
socks = ["PySocks (>=1.5.6,!=1.5.7)"]
|
||||
use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"]
|
||||
|
||||
[[package]]
|
||||
name = "s3transfer"
|
||||
version = "0.13.0"
|
||||
description = "An Amazon S3 Transfer Manager"
|
||||
optional = false
|
||||
python-versions = ">=3.9"
|
||||
groups = ["main"]
|
||||
files = [
|
||||
{file = "s3transfer-0.13.0-py3-none-any.whl", hash = "sha256:0148ef34d6dd964d0d8cf4311b2b21c474693e57c2e069ec708ce043d2b527be"},
|
||||
{file = "s3transfer-0.13.0.tar.gz", hash = "sha256:f5e6db74eb7776a37208001113ea7aa97695368242b364d73e91c981ac522177"},
|
||||
]
|
||||
|
||||
[package.dependencies]
|
||||
botocore = ">=1.37.4,<2.0a.0"
|
||||
|
||||
[package.extras]
|
||||
crt = ["botocore[crt] (>=1.37.4,<2.0a.0)"]
|
||||
|
||||
[[package]]
|
||||
name = "six"
|
||||
version = "1.17.0"
|
||||
|
|
@ -1969,4 +2097,4 @@ files = [
|
|||
[metadata]
|
||||
lock-version = "2.1"
|
||||
python-versions = ">=3.12"
|
||||
content-hash = "1d5c1e0bfc12e88ca9b4c46141c848064a45e9cc4b60990fa3ec7ecb5ef71209"
|
||||
content-hash = "dfda98ea4e00851a83a2f67c231b59476d407a1e38006610722c64842976e736"
|
||||
|
|
|
|||
|
|
@ -15,7 +15,6 @@ dependencies = [
|
|||
"openpyxl (>=3.1.5,<4.0.0)",
|
||||
"fuzzywuzzy (>=0.18.0,<0.19.0)",
|
||||
"sqlmodel (>=0.0.24,<0.0.25)",
|
||||
"psycopg2 (>=2.9.10,<3.0.0)",
|
||||
"pydantic-settings (>=2.8.1,<3.0.0)",
|
||||
"alembic (>=1.15.1,<2.0.0)",
|
||||
"pytest (>=8.3.5,<9.0.0)",
|
||||
|
|
@ -23,6 +22,8 @@ dependencies = [
|
|||
"beautifulsoup4 (>=4.13.4,<5.0.0)",
|
||||
"tqdm (>=4.67.1,<5.0.0)",
|
||||
"hubspot-api-client (>=12.0.0,<13.0.0)",
|
||||
"boto3 (>=1.39.6,<2.0.0)",
|
||||
"psycopg2-binary (>=2.9.10,<3.0.0)",
|
||||
]
|
||||
|
||||
[tool.poetry]
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue