From 61f99f7fc58c2aca6e2b846bb3010978e0bf1912 Mon Sep 17 00:00:00 2001 From: Jun-te Kim Date: Sun, 14 Dec 2025 01:10:47 +0000 Subject: [PATCH] it is what it is --- db/k8s/secrets/aws-backup-secret.yaml | 10 -- mist_infra/arc/autoscaling-runner-set.yaml | 2 + .../scripts/backup_k8s_storage_to_s3.sh | 107 +++++++++--------- 3 files changed, 58 insertions(+), 61 deletions(-) delete mode 100644 db/k8s/secrets/aws-backup-secret.yaml diff --git a/db/k8s/secrets/aws-backup-secret.yaml b/db/k8s/secrets/aws-backup-secret.yaml deleted file mode 100644 index 548907b..0000000 --- a/db/k8s/secrets/aws-backup-secret.yaml +++ /dev/null @@ -1,10 +0,0 @@ -apiVersion: v1 -kind: Secret -metadata: - name: aws-backup-secret -type: Opaque -stringData: - AWS_ACCESS_KEY_ID: xxx - AWS_SECRET_ACCESS_KEY: yyy - AWS_REGION: eu-west-2 - S3_BUCKET: mist-db-backups diff --git a/mist_infra/arc/autoscaling-runner-set.yaml b/mist_infra/arc/autoscaling-runner-set.yaml index 4520809..916a53f 100644 --- a/mist_infra/arc/autoscaling-runner-set.yaml +++ b/mist_infra/arc/autoscaling-runner-set.yaml @@ -55,6 +55,8 @@ spec: envFrom: - secretRef: name: aws-secrets + - secretRef: + name: postgres-secret # 👇 MOUNT PV STORAGE READ-ONLY volumeMounts: diff --git a/mist_infra/scripts/backup_k8s_storage_to_s3.sh b/mist_infra/scripts/backup_k8s_storage_to_s3.sh index e330647..e122300 100644 --- a/mist_infra/scripts/backup_k8s_storage_to_s3.sh +++ b/mist_infra/scripts/backup_k8s_storage_to_s3.sh @@ -4,93 +4,98 @@ set -euo pipefail # -------------------------------------------------- # Config # -------------------------------------------------- -BACKUP_DATE="$(date +%Y-%m-%d)" -TMP_DIR="/tmp/backup-${BACKUP_DATE}" +BACKUP_DATE="$(date +%Y-%m-%d_%H-%M-%S)" +TMP_DIR="/tmp/infra-backup-${BACKUP_DATE}" K8S_STORAGE="/k8s_storage" +POSTGRES_STORAGE_DIR="postgres" + +# Postgres (from secret) +PGHOST="postgres.default.svc.cluster.local" +PGPORT="5432" + +# Namespaces +SOURCE_NS="default" +TARGET_NS="${KUBERNETES_NAMESPACE:-arc-systems}" +POSTGRES_SECRET="postgres-secret" + +# S3 S3_BASE="s3://mist-backups/${BACKUP_DATE}" -POSTGRES_NAMESPACE="default" -POSTGRES_POD_LABEL="app=postgres" -POSTGRES_USER="postgres" -POSTGRES_DB="stripe_invoice" +mkdir -p "$TMP_DIR" -mkdir -p "$TMP_DIR/postgres" - -echo "=== Backup date: $BACKUP_DATE ===" +echo "=== Infra backup started ===" +echo "Date: $BACKUP_DATE" +echo "Namespace: $TARGET_NS" # -------------------------------------------------- -# 1. Discover schemas (for documentation) +# 0. Ensure postgres-secret exists in runner namespace # -------------------------------------------------- -echo "=== Discovering Postgres schemas ===" +echo "=== Ensuring postgres-secret exists ===" -POSTGRES_POD="$(kubectl get pods -n "$POSTGRES_NAMESPACE" \ - -l "$POSTGRES_POD_LABEL" \ - -o jsonpath='{.items[0].metadata.name}')" +if ! kubectl get secret "$POSTGRES_SECRET" -n "$TARGET_NS" >/dev/null 2>&1; then + echo "→ Copying postgres-secret from default → ${TARGET_NS}" -SCHEMAS="$(kubectl exec -n "$POSTGRES_NAMESPACE" "$POSTGRES_POD" -- \ - psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -Atc \ - "SELECT schema_name FROM information_schema.schemata - WHERE schema_name NOT IN ('pg_catalog', 'information_schema') - ORDER BY schema_name;")" - -# Write README -{ - echo "Database: ${POSTGRES_DB}" - echo "Schemas:" - for s in $SCHEMAS; do - echo "- $s" - done - echo - echo "Backup date: ${BACKUP_DATE}" - echo "Host: mist" -} > "$TMP_DIR/postgres/README.txt" - -echo "✓ Schemas documented" + kubectl get secret "$POSTGRES_SECRET" -n "$SOURCE_NS" -o yaml \ + | sed "s/namespace: ${SOURCE_NS}/namespace: ${TARGET_NS}/" \ + | kubectl apply -f - +else + echo "✓ postgres-secret already present" +fi # -------------------------------------------------- -# 2. Postgres logical backup (FULL DB) +# 1. Postgres logical backup (SAFE) # -------------------------------------------------- -echo "=== Dumping Postgres database ===" +echo "=== Backing up Postgres via pg_dump ===" -kubectl exec -n "$POSTGRES_NAMESPACE" "$POSTGRES_POD" -- \ - pg_dump -U "$POSTGRES_USER" "$POSTGRES_DB" \ - | gzip > "$TMP_DIR/postgres/stripe_invoice.sql.gz" +export PGPASSWORD="${POSTGRES_PASSWORD}" + +pg_dump \ + --format=custom \ + --no-owner \ + --no-acl \ + --host="$PGHOST" \ + --port="$PGPORT" \ + --username="$POSTGRES_USER" \ + "$POSTGRES_DB" \ + > "$TMP_DIR/postgres.dump" + +gzip "$TMP_DIR/postgres.dump" echo "✓ Postgres dump complete" # -------------------------------------------------- -# 3. Filesystem backup (best-effort) +# 2. Tar k8s_storage (EXCLUDING Postgres) # -------------------------------------------------- -echo "=== Archiving /k8s_storage ===" +echo "=== Archiving k8s_storage (excluding Postgres) ===" tar \ + --exclude="$POSTGRES_STORAGE_DIR" \ --ignore-failed-read \ --warning=no-file-changed \ - -czf "$TMP_DIR/k8s_storage.tar.gz" \ + -czf "$TMP_DIR/k8s_storage_non_postgres.tar.gz" \ -C "$K8S_STORAGE" . echo "✓ Filesystem archive complete" # -------------------------------------------------- -# 4. Upload to S3 +# 3. Upload to S3 # -------------------------------------------------- echo "=== Uploading to S3 ===" -aws s3 cp "$TMP_DIR/k8s_storage.tar.gz" \ - "${S3_BASE}/k8s_storage.tar.gz" +aws s3 cp \ + "$TMP_DIR/postgres.dump.gz" \ + "${S3_BASE}/postgres/stripe_invoice.dump.gz" -aws s3 cp "$TMP_DIR/postgres/stripe_invoice.sql.gz" \ - "${S3_BASE}/postgres/stripe_invoice.sql.gz" - -aws s3 cp "$TMP_DIR/postgres/README.txt" \ - "${S3_BASE}/postgres/README.txt" +aws s3 cp \ + "$TMP_DIR/k8s_storage_non_postgres.tar.gz" \ + "${S3_BASE}/k8s_storage_non_postgres.tar.gz" echo "✓ Upload complete" # -------------------------------------------------- -# 5. Cleanup +# 4. Cleanup # -------------------------------------------------- rm -rf "$TMP_DIR" -echo "=== Backup finished successfully ===" +echo "=== Infra backup finished successfully ==="