From 6dbf511119f4cba02bd6468400a2a16c021c77f7 Mon Sep 17 00:00:00 2001
From: Jun-te Kim <junte.kim@mealcraft.com>
Date: Sun, 14 Dec 2025 01:27:43 +0000
Subject: [PATCH] save

---
 .../scripts/backup_k8s_storage_to_s3.sh       | 153 +++++++++---------
 1 file changed, 74 insertions(+), 79 deletions(-)

diff --git a/mist_infra/scripts/backup_k8s_storage_to_s3.sh b/mist_infra/scripts/backup_k8s_storage_to_s3.sh
index e122300..0e07fa0 100644
--- a/mist_infra/scripts/backup_k8s_storage_to_s3.sh
+++ b/mist_infra/scripts/backup_k8s_storage_to_s3.sh
@@ -1,101 +1,96 @@
 #!/usr/bin/env bash
 set -euo pipefail
 
-# --------------------------------------------------
+# -----------------------------
 # Config
-# --------------------------------------------------
-BACKUP_DATE="$(date +%Y-%m-%d_%H-%M-%S)"
-TMP_DIR="/tmp/infra-backup-${BACKUP_DATE}"
+# -----------------------------
+NAMESPACE="arc-systems"
 
-K8S_STORAGE="/k8s_storage"
-POSTGRES_STORAGE_DIR="postgres"
+PG_ENABLED=true
+PG_SECRET_NAME="postgres-secret"
+PG_POD_SELECTOR="app=postgres"
 
-# Postgres (from secret)
-PGHOST="postgres.default.svc.cluster.local"
-PGPORT="5432"
+K8S_STORAGE_ROOT="/k8s_storage"
 
-# Namespaces
-SOURCE_NS="default"
-TARGET_NS="${KUBERNETES_NAMESPACE:-arc-systems}"
-POSTGRES_SECRET="postgres-secret"
+TAR_EXCLUDES=(
+  "$K8S_STORAGE_ROOT/postgres"
+)
 
-# S3
-S3_BASE="s3://mist-backups/${BACKUP_DATE}"
+BACKUP_ROOT="/tmp/k8s-backups"
+DATE="$(date -u +%Y-%m-%d_%H-%M-%S)"
+BACKUP_DIR="$BACKUP_ROOT/$DATE"
 
-mkdir -p "$TMP_DIR"
+mkdir -p "$BACKUP_DIR"
 
-echo "=== Infra backup started ==="
-echo "Date: $BACKUP_DATE"
-echo "Namespace: $TARGET_NS"
+echo "=== Backup started at $(date -u) ==="
 
-# --------------------------------------------------
-# 0. Ensure postgres-secret exists in runner namespace
-# --------------------------------------------------
-echo "=== Ensuring postgres-secret exists ==="
+# -----------------------------
+# Postgres pg_dump
+# -----------------------------
+if [[ "$PG_ENABLED" == "true" ]]; then
+  echo "=== Postgres pg_dump enabled ==="
 
-if ! kubectl get secret "$POSTGRES_SECRET" -n "$TARGET_NS" >/dev/null 2>&1; then
-  echo "→ Copying postgres-secret from default → ${TARGET_NS}"
+  POSTGRES_POD=$(kubectl get pod \
+    -n "$NAMESPACE" \
+    -l "$PG_POD_SELECTOR" \
+    -o jsonpath='{.items[0].metadata.name}')
 
-  kubectl get secret "$POSTGRES_SECRET" -n "$SOURCE_NS" -o yaml \
-    | sed "s/namespace: ${SOURCE_NS}/namespace: ${TARGET_NS}/" \
-    | kubectl apply -f -
-else
-  echo "✓ postgres-secret already present"
+  POSTGRES_USER=$(kubectl get secret "$PG_SECRET_NAME" \
+    -n "$NAMESPACE" \
+    -o jsonpath='{.data.POSTGRES_USER}' | base64 -d)
+
+  POSTGRES_DB=$(kubectl get secret "$PG_SECRET_NAME" \
+    -n "$NAMESPACE" \
+    -o jsonpath='{.data.POSTGRES_DB}' | base64 -d)
+
+  echo "Dumping database: $POSTGRES_DB"
+
+  kubectl exec -n "$NAMESPACE" "$POSTGRES_POD" -- \
+    pg_dump -U "$POSTGRES_USER" "$POSTGRES_DB" \
+    > "$BACKUP_DIR/postgres.sql"
+
+  echo "✔ pg_dump complete ($(du -h "$BACKUP_DIR/postgres.sql" | cut -f1))"
 fi
 
-# --------------------------------------------------
-# 1. Postgres logical backup (SAFE)
-# --------------------------------------------------
-echo "=== Backing up Postgres via pg_dump ==="
+# -----------------------------
+# Build tar exclude args
+# -----------------------------
+TAR_EXCLUDE_ARGS=()
+for path in "${TAR_EXCLUDES[@]}"; do
+  TAR_EXCLUDE_ARGS+=(--exclude="$path")
+done
 
-export PGPASSWORD="${POSTGRES_PASSWORD}"
+# -----------------------------
+# Archive k8s storage (safe)
+# -----------------------------
+echo "=== Archiving k8s storage ==="
 
-pg_dump \
-  --format=custom \
-  --no-owner \
-  --no-acl \
-  --host="$PGHOST" \
-  --port="$PGPORT" \
-  --username="$POSTGRES_USER" \
-  "$POSTGRES_DB" \
-  > "$TMP_DIR/postgres.dump"
+tar -czf "$BACKUP_DIR/k8s_storage_$DATE.tar.gz" \
+  "${TAR_EXCLUDE_ARGS[@]}" \
+  "$K8S_STORAGE_ROOT"
 
-gzip "$TMP_DIR/postgres.dump"
+echo "✔ k8s_storage archived"
 
-echo "✓ Postgres dump complete"
+# -----------------------------
+# Upload to S3
+# -----------------------------
+S3_BUCKET="s3://mist-backups/$DATE"
 
-# --------------------------------------------------
-# 2. Tar k8s_storage (EXCLUDING Postgres)
-# --------------------------------------------------
-echo "=== Archiving k8s_storage (excluding Postgres) ==="
+aws s3 cp "$BACKUP_DIR" "$S3_BUCKET" --recursive
 
-tar \
-  --exclude="$POSTGRES_STORAGE_DIR" \
-  --ignore-failed-read \
-  --warning=no-file-changed \
-  -czf "$TMP_DIR/k8s_storage_non_postgres.tar.gz" \
-  -C "$K8S_STORAGE" .
+echo "✔ Uploaded to $S3_BUCKET"
 
-echo "✓ Filesystem archive complete"
-
-# --------------------------------------------------
-# 3. Upload to S3
-# --------------------------------------------------
-echo "=== Uploading to S3 ==="
-
-aws s3 cp \
-  "$TMP_DIR/postgres.dump.gz" \
-  "${S3_BASE}/postgres/stripe_invoice.dump.gz"
-
-aws s3 cp \
-  "$TMP_DIR/k8s_storage_non_postgres.tar.gz" \
-  "${S3_BASE}/k8s_storage_non_postgres.tar.gz"
-
-echo "✓ Upload complete"
-
-# --------------------------------------------------
-# 4. Cleanup
-# --------------------------------------------------
-rm -rf "$TMP_DIR"
-
-echo "=== Infra backup finished successfully ==="
+# -----------------------------
+# Restore instructions
+# -----------------------------
+echo ""
+echo "=== RESTORE GUIDE ==="
+echo ""
+echo "Restore volumes:"
+echo "  sudo tar -xzf k8s_storage_$DATE.tar.gz -C /"
+echo ""
+echo "Restore Postgres:"
+echo "  kubectl exec -n $NAMESPACE -i $POSTGRES_POD -- \\"
+echo "    psql -U $POSTGRES_USER $POSTGRES_DB < postgres.sql"
+echo ""
+echo "=== Backup completed successfully ==="