From cd2da455bc5b7bba7754c7acf61bdd55ad2c50d8 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Sun, 28 Dec 2025 14:20:31 +0000 Subject: [PATCH 1/6] works for main --- db/k8s/secrets/postgres-secret-dev.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db/k8s/secrets/postgres-secret-dev.yaml b/db/k8s/secrets/postgres-secret-dev.yaml index f85d639..791f4ec 100644 --- a/db/k8s/secrets/postgres-secret-dev.yaml +++ b/db/k8s/secrets/postgres-secret-dev.yaml @@ -6,7 +6,7 @@ metadata: namespace: arc-systems type: Opaque stringData: - POSTGRES_USER: postgres + POSTGRES_USER: stripe_invoice POSTGRES_PASSWORD: averysecretpasswordPersonAppleWinter938 POSTGRES_DB: stripe_invoice From 6328dee08d26ef6409be5216fb5f4caa048923ba Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Sun, 28 Dec 2025 15:22:21 +0000 Subject: [PATCH 2/6] push --- db/k8s/postgres/postgres-dev-stripe-to-invoice.yaml | 2 +- db/k8s/secrets/postgres-secret-dev.yaml | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/db/k8s/postgres/postgres-dev-stripe-to-invoice.yaml b/db/k8s/postgres/postgres-dev-stripe-to-invoice.yaml index 7aff6a5..6402bc9 100644 --- a/db/k8s/postgres/postgres-dev-stripe-to-invoice.yaml +++ b/db/k8s/postgres/postgres-dev-stripe-to-invoice.yaml @@ -102,6 +102,6 @@ metadata: namespace: default type: Opaque stringData: - POSTGRES_USER: stripe_invoice + POSTGRES_USER: postgres POSTGRES_PASSWORD: averysecretpasswordPersonAppleWinter938 POSTGRES_DB: stripe_invoice \ No newline at end of file diff --git a/db/k8s/secrets/postgres-secret-dev.yaml b/db/k8s/secrets/postgres-secret-dev.yaml index 791f4ec..2dbdaf3 100644 --- a/db/k8s/secrets/postgres-secret-dev.yaml +++ b/db/k8s/secrets/postgres-secret-dev.yaml @@ -1,4 +1,5 @@ -# This will need to be changes to stripe_invoice_dev soon +# Github runners/workers needs access to secret to set env variable for various things +--- apiVersion: v1 kind: Secret metadata: @@ -6,7 +7,7 @@ metadata: namespace: arc-systems type: Opaque stringData: - POSTGRES_USER: stripe_invoice + POSTGRES_USER: postgres POSTGRES_PASSWORD: averysecretpasswordPersonAppleWinter938 POSTGRES_DB: stripe_invoice From 4f589ea44e18e6fb5bcf253f10c9b051b1cc2a0e Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Sun, 28 Dec 2025 15:37:52 +0000 Subject: [PATCH 3/6] push --- .github/workflows/weekly-k8s-backup.yml | 9 +- .../scripts/backup_k8s_storage_to_s3.sh | 154 +++++++++++------- 2 files changed, 99 insertions(+), 64 deletions(-) diff --git a/.github/workflows/weekly-k8s-backup.yml b/.github/workflows/weekly-k8s-backup.yml index 7a7bf20..dad7556 100644 --- a/.github/workflows/weekly-k8s-backup.yml +++ b/.github/workflows/weekly-k8s-backup.yml @@ -48,11 +48,16 @@ jobs: aws sts get-caller-identity - - name: Run backup + - name: Run DEV backup run: | + ENVIRONMENT=dev \ bash mist_infra/scripts/backup_k8s_storage_to_s3.sh - + - name: Run PROD backup + run: | + ENVIRONMENT=prod \ + I_UNDERSTAND_THIS_IS_PROD=true \ + bash mist_infra/scripts/backup_k8s_storage_to_s3.sh # example of restoring a back up # aws s3 cp s3://mist-backups/2025-03-09/k8s_storage_mist_2025-03-09_02-30-01.tar.gz . diff --git a/mist_infra/scripts/backup_k8s_storage_to_s3.sh b/mist_infra/scripts/backup_k8s_storage_to_s3.sh index 4fb95a7..9909ae9 100644 --- a/mist_infra/scripts/backup_k8s_storage_to_s3.sh +++ b/mist_infra/scripts/backup_k8s_storage_to_s3.sh @@ -1,68 +1,103 @@ #!/usr/bin/env bash set -euo pipefail -# ----------------------------- -# Config -# ----------------------------- +# ================================================== +# REQUIRED ENV VARS +# ================================================== +ENVIRONMENT="${ENVIRONMENT:-}" + +if [[ -z "$ENVIRONMENT" ]]; then + echo "❌ ENVIRONMENT must be set to dev or prod" + exit 1 +fi + +# ================================================== +# GLOBAL CONFIG +# ================================================== NAMESPACE="default" - -PG_ENABLED=true -PG_SECRET_NAME="postgres-secret" -PG_POD_SELECTOR="app=postgres" - K8S_STORAGE_ROOT="/k8s_storage" - -# NEVER touch Postgres raw data -TAR_EXCLUDES=( - "$K8S_STORAGE_ROOT/postgres" -) - BACKUP_ROOT="/tmp/k8s-backups" DATE="$(date -u +%Y-%m-%d_%H-%M-%S)" BACKUP_DIR="$BACKUP_ROOT/$DATE" mkdir -p "$BACKUP_DIR" -echo "=== Backup started at $(date -u) ===" +# NEVER touch raw Postgres data +TAR_EXCLUDES=( + "$K8S_STORAGE_ROOT/postgres" +) -# ----------------------------- -# Postgres pg_dump (SAFE) -# ----------------------------- -if [[ "$PG_ENABLED" == "true" ]]; then - echo "=== Postgres pg_dump enabled ===" +# ================================================== +# ENVIRONMENT SWITCH +# ================================================== +case "$ENVIRONMENT" in + dev) + PG_SECRET_NAME="postgres-secret" + PG_POD_SELECTOR="app=postgres-dev" + S3_PREFIX="dev" + ;; + prod) + if [[ "${I_UNDERSTAND_THIS_IS_PROD:-}" != "true" ]]; then + echo "❌ Refusing to run PROD backup without confirmation" + echo " Re-run with: I_UNDERSTAND_THIS_IS_PROD=true" + exit 1 + fi - POSTGRES_POD=$(kubectl get pods \ - -n "$NAMESPACE" \ - -l "$PG_POD_SELECTOR" \ - -o jsonpath='{.items[*].metadata.name}' | awk '{print $1}') - - if [[ -z "$POSTGRES_POD" ]]; then - echo "❌ No Postgres pod found" - kubectl get pods -n "$NAMESPACE" + PG_SECRET_NAME="postgres-prod-secret" + PG_POD_SELECTOR="app=postgres-prod" + S3_PREFIX="prod" + ;; + *) + echo "❌ Invalid ENVIRONMENT: $ENVIRONMENT (must be dev or prod)" exit 1 - fi + ;; +esac - POSTGRES_USER=$(kubectl get secret "$PG_SECRET_NAME" \ - -n "$NAMESPACE" \ - -o jsonpath='{.data.POSTGRES_USER}' | base64 -d) +echo "=== Backup started ($(date -u)) ===" +echo "Environment: $ENVIRONMENT" - POSTGRES_DB=$(kubectl get secret "$PG_SECRET_NAME" \ - -n "$NAMESPACE" \ - -o jsonpath='{.data.POSTGRES_DB}' | base64 -d) +# ================================================== +# POSTGRES DUMP (SAFE) +# ================================================== +POSTGRES_POD=$(kubectl get pods \ + -n "$NAMESPACE" \ + -l "$PG_POD_SELECTOR" \ + -o jsonpath='{.items[*].metadata.name}' | awk '{print $1}') - echo "Dumping database: $POSTGRES_DB" - - kubectl exec -n "$NAMESPACE" "$POSTGRES_POD" -- \ - pg_dump -U "$POSTGRES_USER" "$POSTGRES_DB" \ - > "$BACKUP_DIR/postgres.sql" - - echo "✔ pg_dump complete ($(du -h "$BACKUP_DIR/postgres.sql" | cut -f1))" +if [[ -z "$POSTGRES_POD" ]]; then + echo "❌ No Postgres pod found for selector: $PG_POD_SELECTOR" + kubectl get pods -n "$NAMESPACE" + exit 1 fi -# ----------------------------- -# Normalise permissions (EXCLUDING POSTGRES) -# ----------------------------- -echo "=== Normalising permissions (excluding Postgres) ===" +POSTGRES_USER=$(kubectl get secret "$PG_SECRET_NAME" \ + -n "$NAMESPACE" \ + -o jsonpath='{.data.POSTGRES_USER}' | base64 -d) + +POSTGRES_DB=$(kubectl get secret "$PG_SECRET_NAME" \ + -n "$NAMESPACE" \ + -o jsonpath='{.data.POSTGRES_DB}' 2>/dev/null | base64 -d || true) + +if [[ -z "$POSTGRES_DB" ]]; then + echo "❌ POSTGRES_DB missing in secret $PG_SECRET_NAME" + exit 1 +fi + +echo "Dumping database: $POSTGRES_DB" + +kubectl exec -n "$NAMESPACE" "$POSTGRES_POD" -- \ + pg_dump \ + -h localhost \ + -U "$POSTGRES_USER" \ + "$POSTGRES_DB" \ + > "$BACKUP_DIR/postgres.sql" + +echo "✔ pg_dump complete ($(du -h "$BACKUP_DIR/postgres.sql" | cut -f1))" + +# ================================================== +# NORMALISE PERMISSIONS (EXCLUDING POSTGRES) +# ================================================== +echo "Normalising permissions (excluding Postgres data)..." sudo find "$K8S_STORAGE_ROOT" \ -mindepth 1 \ @@ -70,19 +105,14 @@ sudo find "$K8S_STORAGE_ROOT" \ ! -name postgres \ -exec chmod -R a+rX {} \; || true -# ----------------------------- -# Build tar exclude args -# ----------------------------- +# ================================================== +# ARCHIVE K8S STORAGE (SAFE) +# ================================================== TAR_EXCLUDE_ARGS=() for path in "${TAR_EXCLUDES[@]}"; do TAR_EXCLUDE_ARGS+=(--exclude="$path") done -# ----------------------------- -# Archive k8s storage (SAFE) -# ----------------------------- -echo "=== Archiving k8s storage ===" - tar \ --ignore-failed-read \ --warning=no-file-changed \ @@ -92,21 +122,21 @@ tar \ echo "✔ k8s_storage archived ($(du -h "$BACKUP_DIR/k8s_storage_$DATE.tar.gz" | cut -f1))" -# ----------------------------- -# Upload to S3 -# ----------------------------- -S3_BUCKET="s3://mist-backups/$DATE" +# ================================================== +# UPLOAD TO S3 +# ================================================== +S3_BUCKET="s3://mist-backups/$S3_PREFIX/$DATE" aws s3 cp "$BACKUP_DIR" "$S3_BUCKET" --recursive echo "✔ Uploaded to $S3_BUCKET" -# ----------------------------- -# Restore instructions -# ----------------------------- +# ================================================== +# RESTORE GUIDE +# ================================================== echo "" echo "========================================" -echo "=== RESTORE GUIDE" +echo "=== RESTORE GUIDE ($ENVIRONMENT)" echo "========================================" echo "" echo "Restore volumes:" From d0b9fee59bf44b493b7c0821376cdb5d464bf21f Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Sun, 28 Dec 2025 15:38:52 +0000 Subject: [PATCH 4/6] push --- .github/workflows/weekly-k8s-backup.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/weekly-k8s-backup.yml b/.github/workflows/weekly-k8s-backup.yml index dad7556..c95bd87 100644 --- a/.github/workflows/weekly-k8s-backup.yml +++ b/.github/workflows/weekly-k8s-backup.yml @@ -48,12 +48,12 @@ jobs: aws sts get-caller-identity - - name: Run DEV backup + - name: Run DEV stripe-to-invoice postgres backup run: | ENVIRONMENT=dev \ bash mist_infra/scripts/backup_k8s_storage_to_s3.sh - - name: Run PROD backup + - name: Run PROD stripe-to-invoice postgres backup run: | ENVIRONMENT=prod \ I_UNDERSTAND_THIS_IS_PROD=true \ From a0362b0142f58f7e77f68f8482be0fad959c8979 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Sun, 28 Dec 2025 15:41:22 +0000 Subject: [PATCH 5/6] push --- mist_infra/scripts/backup_k8s_storage_to_s3.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mist_infra/scripts/backup_k8s_storage_to_s3.sh b/mist_infra/scripts/backup_k8s_storage_to_s3.sh index 9909ae9..e667643 100644 --- a/mist_infra/scripts/backup_k8s_storage_to_s3.sh +++ b/mist_infra/scripts/backup_k8s_storage_to_s3.sh @@ -33,7 +33,7 @@ TAR_EXCLUDES=( case "$ENVIRONMENT" in dev) PG_SECRET_NAME="postgres-secret" - PG_POD_SELECTOR="app=postgres-dev" + PG_POD_SELECTOR="app=postgres" S3_PREFIX="dev" ;; prod) From 9b661fab4cefee145daa8ea77e2ae7d52d7f5822 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Sun, 28 Dec 2025 15:59:54 +0000 Subject: [PATCH 6/6] save --- stripe_to_invoice/app/page.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stripe_to_invoice/app/page.tsx b/stripe_to_invoice/app/page.tsx index 768d031..931e83d 100644 --- a/stripe_to_invoice/app/page.tsx +++ b/stripe_to_invoice/app/page.tsx @@ -20,7 +20,7 @@ export default function Home() {

How it works

    -
  1. Log in
    2. +
  2. Log in ( Set up magic link, db has been set up)
  3. Connect Stripe
  4. Connect Xero
  5. Make a payment