From 5453f92c6972d98f0508a913aaf50fb1953db6b1 Mon Sep 17 00:00:00 2001
From: Jun-te Kim <junte.kim@mealcraft.com>
Date: Sat, 13 Dec 2025 23:22:24 +0000
Subject: [PATCH] push

---
 .github/workflows/weekly-k8s-backup.yml       | 34 +++++++++++++++
 github_runner/install/values.yaml             | 20 +++++++++
 mist_infra/README.md                          |  8 ++++
 mist_infra/arc/update_arc.sh                  | 43 +++++++++++++++++++
 mist_infra/arc/values.yaml                    | 19 ++++++++
 .../cert_manager/install_cert_manager.sh      | 13 ++++++
 mist_infra/rbac/infra-deployer-rbac.yaml      |  0
 .../scripts/backup_k9s_storage_to_s3.sh       | 42 ++++++++++++++++++
 8 files changed, 179 insertions(+)
 create mode 100644 .github/workflows/weekly-k8s-backup.yml
 create mode 100644 github_runner/install/values.yaml
 create mode 100644 mist_infra/README.md
 create mode 100755 mist_infra/arc/update_arc.sh
 create mode 100644 mist_infra/arc/values.yaml
 create mode 100644 mist_infra/cert_manager/install_cert_manager.sh
 create mode 100644 mist_infra/rbac/infra-deployer-rbac.yaml
 create mode 100644 mist_infra/scripts/backup_k9s_storage_to_s3.sh

diff --git a/.github/workflows/weekly-k8s-backup.yml b/.github/workflows/weekly-k8s-backup.yml
new file mode 100644
index 0000000..8e075a3
--- /dev/null
+++ b/.github/workflows/weekly-k8s-backup.yml
@@ -0,0 +1,34 @@
+name: Weekly K8s Storage Backup
+
+on:
+  schedule:
+    # Sunday 02:30 UTC (quiet time, predictable)
+    - cron: "30 2 * * 0"
+  workflow_dispatch:
+
+jobs:
+  backup:
+    name: Backup /k8s_storage → S3
+    runs-on: [self-hosted, mist]
+    timeout-minutes: 180
+
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v4
+
+      - name: Sanity check mount
+        run: |
+          echo "Listing /k8s_storage:"
+          ls -lah /k8s_storage
+
+      - name: Verify AWS identity
+        run: aws sts get-caller-identity
+
+      - name: Run backup
+        run: |
+          bash scripts/backup_k8s_storage_to_s3.sh
+
+
+# example of restoring a back up
+# aws s3 cp s3://mist-backups/2025-03-09/k8s_storage_mist_2025-03-09_02-30-01.tar.gz .
+# sudo tar -xzf k8s_storage_*.tar.gz -C /home/kimjunte/k8s_storage
\ No newline at end of file
diff --git a/github_runner/install/values.yaml b/github_runner/install/values.yaml
new file mode 100644
index 0000000..d4d7662
--- /dev/null
+++ b/github_runner/install/values.yaml
@@ -0,0 +1,20 @@
+runner:
+  name: mist-runner
+  labels:
+    - mist
+    - self-hosted
+
+  envFrom:
+    - secretRef:
+        name: aws-secrets
+
+  volumeMounts:
+    - name: k8s-storage
+      mountPath: /k8s_storage
+      readOnly: true
+
+  volumes:
+    - name: k8s-storage
+      hostPath:
+        path: /home/kimjunte/k8s_storage
+        type: Directory
diff --git a/mist_infra/README.md b/mist_infra/README.md
new file mode 100644
index 0000000..45336ab
--- /dev/null
+++ b/mist_infra/README.md
@@ -0,0 +1,8 @@
+./scripts/bootstrap_microk8s.sh
+./mist_infra/cert_manager/install_cert_manager.sh
+./mist_infra/arc/update_arc.sh
+
+
+for each clusteR:
+
+kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.14.4/cert-manager.crds.yaml
diff --git a/mist_infra/arc/update_arc.sh b/mist_infra/arc/update_arc.sh
new file mode 100755
index 0000000..64a1b33
--- /dev/null
+++ b/mist_infra/arc/update_arc.sh
@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# ==========================================================
+# Update / Install GitHub Actions Runner Controller (ARC)
+#
+# - Safe to run multiple times
+# - Applies changes from arc/values.yaml
+# - Does NOT assume fresh cluster
+#
+# ==========================================================
+
+NAMESPACE="actions-runner-system"
+RELEASE_NAME="actions-runner-controller"
+CHART="actions-runner-controller/actions-runner-controller"
+VALUES_FILE="$(dirname "$0")/values.yaml"
+
+echo "=== Updating ARC (GitHub Actions Runner Controller) ==="
+
+echo "→ Ensuring namespace exists: $NAMESPACE"
+kubectl create namespace "$NAMESPACE" \
+  --dry-run=client -o yaml | kubectl apply -f -
+
+echo "→ Adding Helm repo (if missing)"
+helm repo add actions-runner-controller \
+  https://actions-runner-controller.github.io/actions-runner-controller \
+  >/dev/null 2>&1 || true
+
+helm repo update
+
+echo "→ Applying Helm upgrade"
+helm upgrade --install \
+  "$RELEASE_NAME" \
+  "$CHART" \
+  -n "$NAMESPACE" \
+  -f "$VALUES_FILE"
+
+echo
+echo "✅ ARC update complete"
+echo
+echo "Next steps:"
+echo "- kubectl get pods -n $NAMESPACE"
+echo "- kubectl get runners"
diff --git a/mist_infra/arc/values.yaml b/mist_infra/arc/values.yaml
new file mode 100644
index 0000000..45275c1
--- /dev/null
+++ b/mist_infra/arc/values.yaml
@@ -0,0 +1,19 @@
+runner:
+  labels:
+    - mist
+    - self-hosted
+
+  envFrom:
+    - secretRef:
+        name: aws-secrets
+
+  volumeMounts:
+    - name: k8s-storage
+      mountPath: /k8s_storage
+      readOnly: true
+
+  volumes:
+    - name: k8s-storage
+      hostPath:
+        path: /home/kimjunte/k8s_storage
+        type: Directory
diff --git a/mist_infra/cert_manager/install_cert_manager.sh b/mist_infra/cert_manager/install_cert_manager.sh
new file mode 100644
index 0000000..f20c228
--- /dev/null
+++ b/mist_infra/cert_manager/install_cert_manager.sh
@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.14.4/cert-manager.crds.yaml
+
+helm repo add jetstack https://charts.jetstack.io >/dev/null 2>&1 || true
+helm repo update
+
+kubectl create namespace cert-manager --dry-run=client -o yaml | kubectl apply -f -
+
+helm upgrade --install cert-manager jetstack/cert-manager \
+  --namespace cert-manager \
+  --version v1.14.4
diff --git a/mist_infra/rbac/infra-deployer-rbac.yaml b/mist_infra/rbac/infra-deployer-rbac.yaml
new file mode 100644
index 0000000..e69de29
diff --git a/mist_infra/scripts/backup_k9s_storage_to_s3.sh b/mist_infra/scripts/backup_k9s_storage_to_s3.sh
new file mode 100644
index 0000000..77c5136
--- /dev/null
+++ b/mist_infra/scripts/backup_k9s_storage_to_s3.sh
@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# ----------------------------------------------------------
+# Weekly full backup of all Kubernetes PV data
+# ----------------------------------------------------------
+
+SOURCE_DIR="/k8s_storage"
+TMP_DIR="/tmp/mist-backups"
+BUCKET="mist-backups"
+
+DATE="$(date -u +%Y-%m-%d)"
+TIMESTAMP="$(date -u +%Y-%m-%d_%H-%M-%S)"
+HOST="$(hostname)"
+
+ARCHIVE_NAME="k8s_storage_${HOST}_${TIMESTAMP}.tar.gz"
+ARCHIVE_PATH="${TMP_DIR}/${ARCHIVE_NAME}"
+
+echo "=== Mist weekly PV backup ==="
+echo "Source:   ${SOURCE_DIR}"
+echo "Archive:  ${ARCHIVE_PATH}"
+echo "Bucket:   s3://${BUCKET}/${DATE}/"
+
+mkdir -p "${TMP_DIR}"
+
+echo "→ Creating tarball"
+tar \
+  --numeric-owner \
+  --xattrs \
+  --acls \
+  -czf "${ARCHIVE_PATH}" \
+  -C "${SOURCE_DIR}" .
+
+echo "→ Uploading to S3"
+aws s3 cp \
+  "${ARCHIVE_PATH}" \
+  "s3://${BUCKET}/${DATE}/${ARCHIVE_NAME}"
+
+echo "→ Cleaning up local temp"
+rm -f "${ARCHIVE_PATH}"
+
+echo "✅ Backup complete"