save

mist_infra/scripts/backup_k8s_storage_to_s3.sh

@@ -1,101 +1,96 @@
 #!/usr/bin/env bash
 set -euo pipefail
 
-# --------------------------------------------------
+# -----------------------------
 # Config
-# --------------------------------------------------
+# -----------------------------
-BACKUP_DATE="$(date +%Y-%m-%d_%H-%M-%S)"
+NAMESPACE="arc-systems"
-TMP_DIR="/tmp/infra-backup-${BACKUP_DATE}"
 
-K8S_STORAGE="/k8s_storage"
+PG_ENABLED=true
-POSTGRES_STORAGE_DIR="postgres"
+PG_SECRET_NAME="postgres-secret"
+PG_POD_SELECTOR="app=postgres"
 
-# Postgres (from secret)
+K8S_STORAGE_ROOT="/k8s_storage"
-PGHOST="postgres.default.svc.cluster.local"
-PGPORT="5432"
 
-# Namespaces
+TAR_EXCLUDES=(
-SOURCE_NS="default"
+  "$K8S_STORAGE_ROOT/postgres"
-TARGET_NS="${KUBERNETES_NAMESPACE:-arc-systems}"
+)
-POSTGRES_SECRET="postgres-secret"
 
-# S3
+BACKUP_ROOT="/tmp/k8s-backups"
-S3_BASE="s3://mist-backups/${BACKUP_DATE}"
+DATE="$(date -u +%Y-%m-%d_%H-%M-%S)"
+BACKUP_DIR="$BACKUP_ROOT/$DATE"
 
-mkdir -p "$TMP_DIR"
+mkdir -p "$BACKUP_DIR"
 
-echo "=== Infra backup started ==="
+echo "=== Backup started at $(date -u) ==="
-echo "Date: $BACKUP_DATE"
-echo "Namespace: $TARGET_NS"
 
-# --------------------------------------------------
+# -----------------------------
-# 0. Ensure postgres-secret exists in runner namespace
+# Postgres pg_dump
-# --------------------------------------------------
+# -----------------------------
-echo "=== Ensuring postgres-secret exists ==="
+if [[ "$PG_ENABLED" == "true" ]]; then
+  echo "=== Postgres pg_dump enabled ==="
 
-if ! kubectl get secret "$POSTGRES_SECRET" -n "$TARGET_NS" >/dev/null 2>&1; then
+  POSTGRES_POD=$(kubectl get pod \
-  echo "→ Copying postgres-secret from default → ${TARGET_NS}"
+    -n "$NAMESPACE" \
+    -l "$PG_POD_SELECTOR" \
+    -o jsonpath='{.items[0].metadata.name}')
 
-  kubectl get secret "$POSTGRES_SECRET" -n "$SOURCE_NS" -o yaml \
+  POSTGRES_USER=$(kubectl get secret "$PG_SECRET_NAME" \
-    | sed "s/namespace: ${SOURCE_NS}/namespace: ${TARGET_NS}/" \
+    -n "$NAMESPACE" \
-    | kubectl apply -f -
+    -o jsonpath='{.data.POSTGRES_USER}' | base64 -d)
-else
+
-  echo "✓ postgres-secret already present"
+  POSTGRES_DB=$(kubectl get secret "$PG_SECRET_NAME" \
+    -n "$NAMESPACE" \
+    -o jsonpath='{.data.POSTGRES_DB}' | base64 -d)
+
+  echo "Dumping database: $POSTGRES_DB"
+
+  kubectl exec -n "$NAMESPACE" "$POSTGRES_POD" -- \
+    pg_dump -U "$POSTGRES_USER" "$POSTGRES_DB" \
+    > "$BACKUP_DIR/postgres.sql"
+
+  echo "✔ pg_dump complete ($(du -h "$BACKUP_DIR/postgres.sql" | cut -f1))"
 fi
 
-# --------------------------------------------------
+# -----------------------------
-# 1. Postgres logical backup (SAFE)
+# Build tar exclude args
-# --------------------------------------------------
+# -----------------------------
-echo "=== Backing up Postgres via pg_dump ==="
+TAR_EXCLUDE_ARGS=()
+for path in "${TAR_EXCLUDES[@]}"; do
+  TAR_EXCLUDE_ARGS+=(--exclude="$path")
+done
 
-export PGPASSWORD="${POSTGRES_PASSWORD}"
+# -----------------------------
+# Archive k8s storage (safe)
+# -----------------------------
+echo "=== Archiving k8s storage ==="
 
-pg_dump \
+tar -czf "$BACKUP_DIR/k8s_storage_$DATE.tar.gz" \
-  --format=custom \
+  "${TAR_EXCLUDE_ARGS[@]}" \
-  --no-owner \
+  "$K8S_STORAGE_ROOT"
-  --no-acl \
-  --host="$PGHOST" \
-  --port="$PGPORT" \
-  --username="$POSTGRES_USER" \
-  "$POSTGRES_DB" \
-  > "$TMP_DIR/postgres.dump"
 
-gzip "$TMP_DIR/postgres.dump"
+echo "✔ k8s_storage archived"
 
-echo "✓ Postgres dump complete"
+# -----------------------------
+# Upload to S3
+# -----------------------------
+S3_BUCKET="s3://mist-backups/$DATE"
 
-# --------------------------------------------------
+aws s3 cp "$BACKUP_DIR" "$S3_BUCKET" --recursive
-# 2. Tar k8s_storage (EXCLUDING Postgres)
-# --------------------------------------------------
-echo "=== Archiving k8s_storage (excluding Postgres) ==="
 
-tar \
+echo "✔ Uploaded to $S3_BUCKET"
-  --exclude="$POSTGRES_STORAGE_DIR" \
-  --ignore-failed-read \
-  --warning=no-file-changed \
-  -czf "$TMP_DIR/k8s_storage_non_postgres.tar.gz" \
-  -C "$K8S_STORAGE" .
 
-echo "✓ Filesystem archive complete"
+# -----------------------------
-
+# Restore instructions
-# --------------------------------------------------
+# -----------------------------
-# 3. Upload to S3
+echo ""
-# --------------------------------------------------
+echo "=== RESTORE GUIDE ==="
-echo "=== Uploading to S3 ==="
+echo ""
-
+echo "Restore volumes:"
-aws s3 cp \
+echo "  sudo tar -xzf k8s_storage_$DATE.tar.gz -C /"
-  "$TMP_DIR/postgres.dump.gz" \
+echo ""
-  "${S3_BASE}/postgres/stripe_invoice.dump.gz"
+echo "Restore Postgres:"
-
+echo "  kubectl exec -n $NAMESPACE -i $POSTGRES_POD -- \\"
-aws s3 cp \
+echo "    psql -U $POSTGRES_USER $POSTGRES_DB < postgres.sql"
-  "$TMP_DIR/k8s_storage_non_postgres.tar.gz" \
+echo ""
-  "${S3_BASE}/k8s_storage_non_postgres.tar.gz"
+echo "=== Backup completed successfully ==="
-
-echo "✓ Upload complete"
-
-# --------------------------------------------------
-# 4. Cleanup
-# --------------------------------------------------
-rm -rf "$TMP_DIR"
-
-echo "=== Infra backup finished successfully ==="