Merge pull request #61 from Hestia-Homes/feature/nextjs

Feature/nextjs
This commit is contained in:
Jun-te Kim 2025-05-29 15:00:13 +01:00 committed by GitHub
commit 9d96d26a33
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
27 changed files with 6359 additions and 80 deletions

View file

@ -17,21 +17,12 @@ RUN useradd -m -s /usr/bin/bash ${USER} \
# Install Poetry
RUN pip install --no-cache-dir poetry
# Download and install nvm:
# RUN curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.3/install.sh | bash
# # in lieu of restarting the shell
# RUN \. "$HOME/.nvm/nvm.sh"
# # Download and install Node.js:
# RUN nvm install 22
# # Verify the Node.js version:
# RUN node -v # Should print "v22.16.0".
# RUN nvm current # Should print "v22.16.0".
# # Verify npm version:
# RUN npm -v # Should print "10.9.2".
# Install Node.js 22 (from NodeSource)
RUN curl -fsSL https://deb.nodesource.com/setup_22.x | bash - \
&& apt install -y nodejs \
&& node -v \
&& npm -v
# Install aws
@ -53,4 +44,4 @@ RUN terraform -install-autocomplete
# Set the working directory
WORKDIR /workspaces/survey-extractor:q
WORKDIR /workspaces/survey-extractor

View file

@ -1,5 +1,10 @@
import os
import pandas as pd
from etl.utils.logger import Logger
import logging
logger = Logger(name="hubspot_to_invoice", level=logging.DEBUG).get_logger()
os.environ["SHAREPOINT_CLIENT_ID"] = "895e3b77-b1d7-43ec-b18f-dcfe07cdfeaf"
os.environ["SHAREPOINT_CLIENT_SECRET"] = "SOf8Q~-is4wdQiqvEEm9FlJQRAY9ELGaj5Qz-a6E"
os.environ["SHAREPOINT_TENANT_ID"] = "c3f7519c-2719-4547-af04-6da6cbfd8f8f"
@ -43,6 +48,7 @@ output_path = os.path.abspath(lewis_view)
sp.upload_to_sharepoint(output_path, lewis_view)
sp.upload_to_sharepoint(dbLoader.sp.get_master_rate_card_path(), "COPY_OF_RATE_CARD_USED.xlsx")
sp.upload_to_sharepoint("app.log", "debug.log")
# Commented out as i don't want to sync up hubspot_to_db just yet

View file

@ -6,8 +6,10 @@ from tqdm import tqdm
import os
from etl.scraper.scraper import SharePointInstaller
from etl.scraper.scraper import SharePointScraper
board_id = "5185076280"
# wates
# board_id = "4965130190"
# liz green
board_id = "6097500103"
monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY"
monday = MondayClient(monday_key)
@ -20,7 +22,23 @@ osmosis = SharePointScraper(SharePointInstaller.OSMOSIS_WAVE_2)
parent_folder = "/Osmosis ACD/Osmosis ACD Projects/"
# Change this per installer
parent_folder += "Installer Documentation/Platform Housing Group/Cocuun"
# parent_folder += "Stonewater/Wates/REDO"
parent_folder += "Installer Documentation/Platform Housing Group/Liv Green"
import re
def sanitize_name(name: str, ignore_dot = False) -> str:
# Remove leading/trailing whitespace and collapse multiple spaces into a single space
if ignore_dot:
name = ''.join(char for char in name if char.isalnum() or char.isspace() or char == ")" or char == "(").strip()
else:
name = ''.join(char for char in name if char.isalnum() or char.isspace() or char == "." or char == ")" or char == "(").strip()
name = re.sub(r'\s+', ' ', name)
# Remove or replace other SharePoint-invalid characters if necessary
invalid_chars = r'[\"*:<>?/\\|#%~{}]'
name = re.sub(invalid_chars, '', name)
return name
def extract_asset_ids(item, file_column_id):
@ -102,8 +120,10 @@ def get_all_items(board_id, monday):
def upload_to_sharepoint(to_upload, master_folder_name):
osmosis.create_dir(master_folder_name, parent_folder)
for file_path in to_upload:
print(f"uploading {file_path}")
osmosis.upload_file(file_path, parent_folder + f"/{master_folder_name}", file_path[5:])
file_name = file_path[5:]
file_name = sanitize_name(file_name)
print(f"Uploading {file_name} to sharepoint")
osmosis.upload_file(file_path, parent_folder + f"/{master_folder_name}", file_name)
# Step 1: Fetch column IDs
board_data = monday.boards.fetch_boards_by_id(board_id)
@ -117,19 +137,26 @@ if not name_id or not files_id:
raise Exception("Could not find 'name' or 'file(s)' columns")
items = get_all_items(board_id, monday)
for item in tqdm(items):
item_name = item["name"]
asset_ids = extract_asset_ids(item, files_id)
for i,item in enumerate(tqdm(items)):
if i>123:
item_name = item["name"]
item_name = sanitize_name(item_name, ignore_dot=True)
print(f"Item name is {item_name}")
asset_ids = extract_asset_ids(item, files_id)
to_upload = []
for asset_id in asset_ids:
try:
public_url, file_name = get_public_url(asset_id)
print(f"Downloading {file_name} from {public_url}")
file_path = download_file_from_public_url(public_url, file_name)
to_upload.append(file_path)
except Exception as e:
print(f"Failed to download/upload asset {asset_id}: {e}")
to_upload = []
for asset_id in asset_ids:
try:
public_url, file_name = get_public_url(asset_id)
print(f"Downloading {file_name}")
file_path = download_file_from_public_url(public_url, file_name)
to_upload.append(file_path)
except Exception as e:
print(f"Failed to download/upload asset {asset_id}: {e}")
if to_upload:
upload_to_sharepoint(to_upload, item_name)
if to_upload:
upload_to_sharepoint(to_upload, item_name)
# Liv green # Cocuun # Wates

View file

@ -128,7 +128,7 @@ class QuidosSiteNotesExtractor(SiteNotesExtractor):
uprn = get_value('UPRN'),
postcode = get_value('Postcode'),
region = get_value('Region'),
address = get_value('Address'),
address = ','.join(get_value('Address').split(',')[:-1]).strip(),
town = get_value('Town'),
county = get_value('County'),
property_tenure = get_value('Property Tenure'),

View file

@ -25,4 +25,8 @@ def test_floor_area_calculator(local_survey):
# Floor area is important to work out invoice, make a test to work out invoice correctly
area = work_out_total_floor_area(local_survey.pre_site_note)
assert area == ('73-97m', 91)
def test_address_and_post_code(local_survey):
assert local_survey.pre_site_note.survey_information.address == "10 Turnberry Close, ST. LEONARDS-ON-SEA"
assert local_survey.pre_site_note.survey_information.postcode == "TN38 0WL"

View file

@ -1,37 +0,0 @@
import os
# WarmFront Sharepoint KEYS
os.environ["SHAREPOINT_CLIENT_ID"] = "895e3b77-b1d7-43ec-b18f-dcfe07cdfeaf"
os.environ["SHAREPOINT_CLIENT_SECRET"] = "SOf8Q~-is4wdQiqvEEm9FlJQRAY9ELGaj5Qz-a6E"
os.environ["SHAREPOINT_TENANT_ID"] = "c3f7519c-2719-4547-af04-6da6cbfd8f8f"
from etl.surveyPrice.surveyPrice import SurveyPrice
import pytest
@pytest.fixture(scope="module")
def sp():
return SurveyPrice()
def cavity_price_dataframe_sanity_check(df):
assert df.shape == (160, 5)
assert df.columns.tolist() == ['WORK TYPE', 'Floor Area Group', 'Trickle Vent', 'Wetrooms', 'PRICE']
def test_get_price_matrix_jjc_empties(sp):
jjc_empties_price_table = sp.get_cavity_pricing_table("JJC - EMPTIES")
cavity_price_dataframe_sanity_check(jjc_empties_price_table)
def test_get_price_matrix_jjc_general_extraction(sp):
sp = SurveyPrice()
jjc_empties_price_table = sp.get_cavity_pricing_table("JJC - GENERAL EXTRACTIONS")
cavity_price_dataframe_sanity_check(jjc_empties_price_table)
def test_get_price_matrix_jjc_foam(sp):
sp = SurveyPrice()
jjc_empties_price_table = sp.get_cavity_pricing_table("JJC - FORMALDEHYDE EXTRACTION")
cavity_price_dataframe_sanity_check(jjc_empties_price_table)

View file

@ -2,19 +2,32 @@ import logging
import os
class Logger:
def __init__(self, name, level=logging.INFO):
def __init__(self, name, level=logging.INFO, log_file='app.log'):
self.logger = logging.getLogger(name)
self.logger.setLevel(level)
self.log_file = log_file
# ✅ Prevent adding multiple handlers
if not self.logger.handlers:
# Console handler
c_handler = logging.StreamHandler()
c_handler.setLevel(level)
# File handler
f_handler = logging.FileHandler(log_file)
f_handler.setLevel(level)
# Formatter
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
c_handler.setFormatter(formatter)
f_handler.setFormatter(formatter)
# Add handlers to the logger
self.logger.addHandler(c_handler)
self.logger.addHandler(f_handler)
def get_logger(self):
return self.logger
return self.logger
def get_log_file_path(self):
return self.log_file

View file

@ -226,7 +226,7 @@ class SharePointClient:
'Authorization': f"Bearer {self.access_token['access_token']}"
}
logger.debug("Access token retrieved successfully.")
# logger.debug("Access token retrieved successfully.")
@api_call_decorator
def get_documents_drive(self):
@ -235,7 +235,7 @@ class SharePointClient:
:return: Tuple containing HTTP method, URL, and None for data.
"""
url = f"https://graph.microsoft.com/v1.0/sites/{self.site_id}/drive"
logger.debug(f"Getting document drive from URL: {url}")
# logger.debug(f"Getting document drive from URL: {url}")
return 'GET', url, None
@api_call_decorator
@ -250,7 +250,7 @@ class SharePointClient:
:return: Tuple containing HTTP method, URL, and None for data.
"""
url = f"https://graph.microsoft.com/v1.0/drives/{self.document_drive_id}/root:/{folder_path}:/children?$top={page_size}"
logger.debug(f"Listing folder contents from URL: {url}")
# logger.debug(f"Listing folder contents from URL: {url}")
return 'GET', url, None
@ -286,12 +286,12 @@ class SharePointClient:
:return: Response JSON from the API
"""
url = f"https://graph.microsoft.com/v1.0/drives/{self.document_drive_id}/root:/{sharepoint_parent_id}/{file_name}:/content"
logger.debug(f"Uploading file to URL: {url}")
# logger.debug(f"Uploading file to URL: {url}")
response = requests.put(url, headers=self.headers, data=file_stream)
if response.status_code in (200, 201):
logger.info(f"File '{file_name}' uploaded successfully.")
# logger.info(f"File '{file_name}' uploaded successfully.")
return response.json()
else:
retry = handle_error(response)

41
frontend/.gitignore vendored Normal file
View file

@ -0,0 +1,41 @@
# See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
# dependencies
/node_modules
/.pnp
.pnp.*
.yarn/*
!.yarn/patches
!.yarn/plugins
!.yarn/releases
!.yarn/versions
# testing
/coverage
# next.js
/.next/
/out/
# production
/build
# misc
.DS_Store
*.pem
# debug
npm-debug.log*
yarn-debug.log*
yarn-error.log*
.pnpm-debug.log*
# env files (can opt-in for committing if needed)
.env*
# vercel
.vercel
# typescript
*.tsbuildinfo
next-env.d.ts

36
frontend/README.md Normal file
View file

@ -0,0 +1,36 @@
This is a [Next.js](https://nextjs.org) project bootstrapped with [`create-next-app`](https://nextjs.org/docs/app/api-reference/cli/create-next-app).
## Getting Started
First, run the development server:
```bash
npm run dev
# or
yarn dev
# or
pnpm dev
# or
bun dev
```
Open [http://localhost:3000](http://localhost:3000) with your browser to see the result.
You can start editing the page by modifying `app/page.tsx`. The page auto-updates as you edit the file.
This project uses [`next/font`](https://nextjs.org/docs/app/building-your-application/optimizing/fonts) to automatically optimize and load [Geist](https://vercel.com/font), a new font family for Vercel.
## Learn More
To learn more about Next.js, take a look at the following resources:
- [Next.js Documentation](https://nextjs.org/docs) - learn about Next.js features and API.
- [Learn Next.js](https://nextjs.org/learn) - an interactive Next.js tutorial.
You can check out [the Next.js GitHub repository](https://github.com/vercel/next.js) - your feedback and contributions are welcome!
## Deploy on Vercel
The easiest way to deploy your Next.js app is to use the [Vercel Platform](https://vercel.com/new?utm_medium=default-template&filter=next.js&utm_source=create-next-app&utm_campaign=create-next-app-readme) from the creators of Next.js.
Check out our [Next.js deployment documentation](https://nextjs.org/docs/app/building-your-application/deploying) for more details.

View file

@ -0,0 +1,16 @@
import { dirname } from "path";
import { fileURLToPath } from "url";
import { FlatCompat } from "@eslint/eslintrc";
const __filename = fileURLToPath(import.meta.url);
const __dirname = dirname(__filename);
const compat = new FlatCompat({
baseDirectory: __dirname,
});
const eslintConfig = [
...compat.extends("next/core-web-vitals", "next/typescript"),
];
export default eslintConfig;

7
frontend/next.config.ts Normal file
View file

@ -0,0 +1,7 @@
import type { NextConfig } from "next";
const nextConfig: NextConfig = {
/* config options here */
};
export default nextConfig;

6063
frontend/package-lock.json generated Normal file

File diff suppressed because it is too large Load diff

27
frontend/package.json Normal file
View file

@ -0,0 +1,27 @@
{
"name": "frontend",
"version": "0.1.0",
"private": true,
"scripts": {
"dev": "next dev",
"build": "next build",
"start": "next start",
"lint": "next lint"
},
"dependencies": {
"react": "^19.0.0",
"react-dom": "^19.0.0",
"next": "15.3.2"
},
"devDependencies": {
"typescript": "^5",
"@types/node": "^20",
"@types/react": "^19",
"@types/react-dom": "^19",
"@tailwindcss/postcss": "^4",
"tailwindcss": "^4",
"eslint": "^9",
"eslint-config-next": "15.3.2",
"@eslint/eslintrc": "^3"
}
}

View file

@ -0,0 +1,5 @@
const config = {
plugins: ["@tailwindcss/postcss"],
};
export default config;

1
frontend/public/file.svg Normal file
View file

@ -0,0 +1 @@
<svg fill="none" viewBox="0 0 16 16" xmlns="http://www.w3.org/2000/svg"><path d="M14.5 13.5V5.41a1 1 0 0 0-.3-.7L9.8.29A1 1 0 0 0 9.08 0H1.5v13.5A2.5 2.5 0 0 0 4 16h8a2.5 2.5 0 0 0 2.5-2.5m-1.5 0v-7H8v-5H3v12a1 1 0 0 0 1 1h8a1 1 0 0 0 1-1M9.5 5V2.12L12.38 5zM5.13 5h-.62v1.25h2.12V5zm-.62 3h7.12v1.25H4.5zm.62 3h-.62v1.25h7.12V11z" clip-rule="evenodd" fill="#666" fill-rule="evenodd"/></svg>

After

Width:  |  Height:  |  Size: 391 B

View file

@ -0,0 +1 @@
<svg fill="none" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 16 16"><g clip-path="url(#a)"><path fill-rule="evenodd" clip-rule="evenodd" d="M10.27 14.1a6.5 6.5 0 0 0 3.67-3.45q-1.24.21-2.7.34-.31 1.83-.97 3.1M8 16A8 8 0 1 0 8 0a8 8 0 0 0 0 16m.48-1.52a7 7 0 0 1-.96 0H7.5a4 4 0 0 1-.84-1.32q-.38-.89-.63-2.08a40 40 0 0 0 3.92 0q-.25 1.2-.63 2.08a4 4 0 0 1-.84 1.31zm2.94-4.76q1.66-.15 2.95-.43a7 7 0 0 0 0-2.58q-1.3-.27-2.95-.43a18 18 0 0 1 0 3.44m-1.27-3.54a17 17 0 0 1 0 3.64 39 39 0 0 1-4.3 0 17 17 0 0 1 0-3.64 39 39 0 0 1 4.3 0m1.1-1.17q1.45.13 2.69.34a6.5 6.5 0 0 0-3.67-3.44q.65 1.26.98 3.1M8.48 1.5l.01.02q.41.37.84 1.31.38.89.63 2.08a40 40 0 0 0-3.92 0q.25-1.2.63-2.08a4 4 0 0 1 .85-1.32 7 7 0 0 1 .96 0m-2.75.4a6.5 6.5 0 0 0-3.67 3.44 29 29 0 0 1 2.7-.34q.31-1.83.97-3.1M4.58 6.28q-1.66.16-2.95.43a7 7 0 0 0 0 2.58q1.3.27 2.95.43a18 18 0 0 1 0-3.44m.17 4.71q-1.45-.12-2.69-.34a6.5 6.5 0 0 0 3.67 3.44q-.65-1.27-.98-3.1" fill="#666"/></g><defs><clipPath id="a"><path fill="#fff" d="M0 0h16v16H0z"/></clipPath></defs></svg>

After

Width:  |  Height:  |  Size: 1 KiB

1
frontend/public/next.svg Normal file
View file

@ -0,0 +1 @@
<svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 394 80"><path fill="#000" d="M262 0h68.5v12.7h-27.2v66.6h-13.6V12.7H262V0ZM149 0v12.7H94v20.4h44.3v12.6H94v21h55v12.6H80.5V0h68.7zm34.3 0h-17.8l63.8 79.4h17.9l-32-39.7 32-39.6h-17.9l-23 28.6-23-28.6zm18.3 56.7-9-11-27.1 33.7h17.8l18.3-22.7z"/><path fill="#000" d="M81 79.3 17 0H0v79.3h13.6V17l50.2 62.3H81Zm252.6-.4c-1 0-1.8-.4-2.5-1s-1.1-1.6-1.1-2.6.3-1.8 1-2.5 1.6-1 2.6-1 1.8.3 2.5 1a3.4 3.4 0 0 1 .6 4.3 3.7 3.7 0 0 1-3 1.8zm23.2-33.5h6v23.3c0 2.1-.4 4-1.3 5.5a9.1 9.1 0 0 1-3.8 3.5c-1.6.8-3.5 1.3-5.7 1.3-2 0-3.7-.4-5.3-1s-2.8-1.8-3.7-3.2c-.9-1.3-1.4-3-1.4-5h6c.1.8.3 1.6.7 2.2s1 1.2 1.6 1.5c.7.4 1.5.5 2.4.5 1 0 1.8-.2 2.4-.6a4 4 0 0 0 1.6-1.8c.3-.8.5-1.8.5-3V45.5zm30.9 9.1a4.4 4.4 0 0 0-2-3.3 7.5 7.5 0 0 0-4.3-1.1c-1.3 0-2.4.2-3.3.5-.9.4-1.6 1-2 1.6a3.5 3.5 0 0 0-.3 4c.3.5.7.9 1.3 1.2l1.8 1 2 .5 3.2.8c1.3.3 2.5.7 3.7 1.2a13 13 0 0 1 3.2 1.8 8.1 8.1 0 0 1 3 6.5c0 2-.5 3.7-1.5 5.1a10 10 0 0 1-4.4 3.5c-1.8.8-4.1 1.2-6.8 1.2-2.6 0-4.9-.4-6.8-1.2-2-.8-3.4-2-4.5-3.5a10 10 0 0 1-1.7-5.6h6a5 5 0 0 0 3.5 4.6c1 .4 2.2.6 3.4.6 1.3 0 2.5-.2 3.5-.6 1-.4 1.8-1 2.4-1.7a4 4 0 0 0 .8-2.4c0-.9-.2-1.6-.7-2.2a11 11 0 0 0-2.1-1.4l-3.2-1-3.8-1c-2.8-.7-5-1.7-6.6-3.2a7.2 7.2 0 0 1-2.4-5.7 8 8 0 0 1 1.7-5 10 10 0 0 1 4.3-3.5c2-.8 4-1.2 6.4-1.2 2.3 0 4.4.4 6.2 1.2 1.8.8 3.2 2 4.3 3.4 1 1.4 1.5 3 1.5 5h-5.8z"/></svg>

After

Width:  |  Height:  |  Size: 1.3 KiB

View file

@ -0,0 +1 @@
<svg fill="none" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 1155 1000"><path d="m577.3 0 577.4 1000H0z" fill="#fff"/></svg>

After

Width:  |  Height:  |  Size: 128 B

View file

@ -0,0 +1 @@
<svg fill="none" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 16 16"><path fill-rule="evenodd" clip-rule="evenodd" d="M1.5 2.5h13v10a1 1 0 0 1-1 1h-11a1 1 0 0 1-1-1zM0 1h16v11.5a2.5 2.5 0 0 1-2.5 2.5h-11A2.5 2.5 0 0 1 0 12.5zm3.75 4.5a.75.75 0 1 0 0-1.5.75.75 0 0 0 0 1.5M7 4.75a.75.75 0 1 1-1.5 0 .75.75 0 0 1 1.5 0m1.75.75a.75.75 0 1 0 0-1.5.75.75 0 0 0 0 1.5" fill="#666"/></svg>

After

Width:  |  Height:  |  Size: 385 B

Binary file not shown.

After

Width:  |  Height:  |  Size: 25 KiB

View file

@ -0,0 +1,26 @@
@import "tailwindcss";
:root {
--background: #ffffff;
--foreground: #171717;
}
@theme inline {
--color-background: var(--background);
--color-foreground: var(--foreground);
--font-sans: var(--font-geist-sans);
--font-mono: var(--font-geist-mono);
}
@media (prefers-color-scheme: dark) {
:root {
--background: #0a0a0a;
--foreground: #ededed;
}
}
body {
background: var(--background);
color: var(--foreground);
font-family: Arial, Helvetica, sans-serif;
}

View file

@ -0,0 +1,15 @@
export default function DashboardLayout({
children,
}: {
children: React.ReactNode
}) {
return (
<html lang="en">
<body>
{/* Layout UI */}
{/* Place children where you want to render a page or nested layout */}
<main>{children}</main>
</body>
</html>
)
}

View file

@ -0,0 +1,6 @@
export default function Home() {
return (
<h1>Hello Next.js!</h1>
);
}

27
frontend/tsconfig.json Normal file
View file

@ -0,0 +1,27 @@
{
"compilerOptions": {
"target": "ES2017",
"lib": ["dom", "dom.iterable", "esnext"],
"allowJs": true,
"skipLibCheck": true,
"strict": true,
"noEmit": true,
"esModuleInterop": true,
"module": "esnext",
"moduleResolution": "bundler",
"resolveJsonModule": true,
"isolatedModules": true,
"jsx": "preserve",
"incremental": true,
"plugins": [
{
"name": "next"
}
],
"paths": {
"@/*": ["./src/*"]
}
},
"include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts"],
"exclude": ["node_modules"]
}

1
run_frontend.sh Normal file
View file

@ -0,0 +1 @@
cd frontend && npm run dev