it is what it is

db/k8s/secrets/aws-backup-secret.yaml

@@ -1,10 +0,0 @@
-apiVersion: v1
-kind: Secret
-metadata:
-  name: aws-backup-secret
-type: Opaque
-stringData:
-  AWS_ACCESS_KEY_ID: xxx
-  AWS_SECRET_ACCESS_KEY: yyy
-  AWS_REGION: eu-west-2
-  S3_BUCKET: mist-db-backups

mist_infra/arc/autoscaling-runner-set.yaml

@@ -55,6 +55,8 @@ spec:
           envFrom:
             - secretRef:
                 name: aws-secrets
+            - secretRef:
+                name: postgres-secret
 
           # 👇 MOUNT PV STORAGE READ-ONLY
           volumeMounts:

mist_infra/scripts/backup_k8s_storage_to_s3.sh

@@ -4,93 +4,98 @@ set -euo pipefail
 # --------------------------------------------------
 # Config
 # --------------------------------------------------
-BACKUP_DATE="$(date +%Y-%m-%d)"
-TMP_DIR="/tmp/backup-${BACKUP_DATE}"
+BACKUP_DATE="$(date +%Y-%m-%d_%H-%M-%S)"
+TMP_DIR="/tmp/infra-backup-${BACKUP_DATE}"
 
 K8S_STORAGE="/k8s_storage"
+POSTGRES_STORAGE_DIR="postgres"
+
+# Postgres (from secret)
+PGHOST="postgres.default.svc.cluster.local"
+PGPORT="5432"
+
+# Namespaces
+SOURCE_NS="default"
+TARGET_NS="${KUBERNETES_NAMESPACE:-arc-systems}"
+POSTGRES_SECRET="postgres-secret"
+
+# S3
 S3_BASE="s3://mist-backups/${BACKUP_DATE}"
 
-POSTGRES_NAMESPACE="default"
-POSTGRES_POD_LABEL="app=postgres"
-POSTGRES_USER="postgres"
-POSTGRES_DB="stripe_invoice"
+mkdir -p "$TMP_DIR"
 
-mkdir -p "$TMP_DIR/postgres"
-
-echo "=== Backup date: $BACKUP_DATE ==="
+echo "=== Infra backup started ==="
+echo "Date: $BACKUP_DATE"
+echo "Namespace: $TARGET_NS"
 
 # --------------------------------------------------
-# 1. Discover schemas (for documentation)
+# 0. Ensure postgres-secret exists in runner namespace
 # --------------------------------------------------
-echo "=== Discovering Postgres schemas ==="
+echo "=== Ensuring postgres-secret exists ==="
 
-POSTGRES_POD="$(kubectl get pods -n "$POSTGRES_NAMESPACE" \
-  -l "$POSTGRES_POD_LABEL" \
-  -o jsonpath='{.items[0].metadata.name}')"
+if ! kubectl get secret "$POSTGRES_SECRET" -n "$TARGET_NS" >/dev/null 2>&1; then
+  echo "→ Copying postgres-secret from default → ${TARGET_NS}"
 
-SCHEMAS="$(kubectl exec -n "$POSTGRES_NAMESPACE" "$POSTGRES_POD" -- \
-  psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -Atc \
-  "SELECT schema_name FROM information_schema.schemata
-   WHERE schema_name NOT IN ('pg_catalog', 'information_schema')
-   ORDER BY schema_name;")"
-
-# Write README
-{
-  echo "Database: ${POSTGRES_DB}"
-  echo "Schemas:"
-  for s in $SCHEMAS; do
-    echo "- $s"
-  done
-  echo
-  echo "Backup date: ${BACKUP_DATE}"
-  echo "Host: mist"
-} > "$TMP_DIR/postgres/README.txt"
-
-echo "✓ Schemas documented"
+  kubectl get secret "$POSTGRES_SECRET" -n "$SOURCE_NS" -o yaml \
+    | sed "s/namespace: ${SOURCE_NS}/namespace: ${TARGET_NS}/" \
+    | kubectl apply -f -
+else
+  echo "✓ postgres-secret already present"
+fi
 
 # --------------------------------------------------
-# 2. Postgres logical backup (FULL DB)
+# 1. Postgres logical backup (SAFE)
 # --------------------------------------------------
-echo "=== Dumping Postgres database ==="
+echo "=== Backing up Postgres via pg_dump ==="
 
-kubectl exec -n "$POSTGRES_NAMESPACE" "$POSTGRES_POD" -- \
-  pg_dump -U "$POSTGRES_USER" "$POSTGRES_DB" \
-  | gzip > "$TMP_DIR/postgres/stripe_invoice.sql.gz"
+export PGPASSWORD="${POSTGRES_PASSWORD}"
+
+pg_dump \
+  --format=custom \
+  --no-owner \
+  --no-acl \
+  --host="$PGHOST" \
+  --port="$PGPORT" \
+  --username="$POSTGRES_USER" \
+  "$POSTGRES_DB" \
+  > "$TMP_DIR/postgres.dump"
+
+gzip "$TMP_DIR/postgres.dump"
 
 echo "✓ Postgres dump complete"
 
 # --------------------------------------------------
-# 3. Filesystem backup (best-effort)
+# 2. Tar k8s_storage (EXCLUDING Postgres)
 # --------------------------------------------------
-echo "=== Archiving /k8s_storage ==="
+echo "=== Archiving k8s_storage (excluding Postgres) ==="
 
 tar \
+  --exclude="$POSTGRES_STORAGE_DIR" \
   --ignore-failed-read \
   --warning=no-file-changed \
-  -czf "$TMP_DIR/k8s_storage.tar.gz" \
+  -czf "$TMP_DIR/k8s_storage_non_postgres.tar.gz" \
   -C "$K8S_STORAGE" .
 
 echo "✓ Filesystem archive complete"
 
 # --------------------------------------------------
-# 4. Upload to S3
+# 3. Upload to S3
 # --------------------------------------------------
 echo "=== Uploading to S3 ==="
 
-aws s3 cp "$TMP_DIR/k8s_storage.tar.gz" \
-  "${S3_BASE}/k8s_storage.tar.gz"
+aws s3 cp \
+  "$TMP_DIR/postgres.dump.gz" \
+  "${S3_BASE}/postgres/stripe_invoice.dump.gz"
 
-aws s3 cp "$TMP_DIR/postgres/stripe_invoice.sql.gz" \
-  "${S3_BASE}/postgres/stripe_invoice.sql.gz"
-
-aws s3 cp "$TMP_DIR/postgres/README.txt" \
-  "${S3_BASE}/postgres/README.txt"
+aws s3 cp \
+  "$TMP_DIR/k8s_storage_non_postgres.tar.gz" \
+  "${S3_BASE}/k8s_storage_non_postgres.tar.gz"
 
 echo "✓ Upload complete"
 
 # --------------------------------------------------
-# 5. Cleanup
+# 4. Cleanup
 # --------------------------------------------------
 rm -rf "$TMP_DIR"
 
-echo "=== Backup finished successfully ==="
+echo "=== Infra backup finished successfully ==="