push

.github/workflows/weekly-k8s-backup.yml

@@ -0,0 +1,34 @@
+name: Weekly K8s Storage Backup
+
+on:
+  schedule:
+    # Sunday 02:30 UTC (quiet time, predictable)
+    - cron: "30 2 * * 0"
+  workflow_dispatch:
+
+jobs:
+  backup:
+    name: Backup /k8s_storage → S3
+    runs-on: [self-hosted, mist]
+    timeout-minutes: 180
+
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v4
+
+      - name: Sanity check mount
+        run: |
+          echo "Listing /k8s_storage:"
+          ls -lah /k8s_storage
+
+      - name: Verify AWS identity
+        run: aws sts get-caller-identity
+
+      - name: Run backup
+        run: |
+          bash scripts/backup_k8s_storage_to_s3.sh
+
+
+# example of restoring a back up
+# aws s3 cp s3://mist-backups/2025-03-09/k8s_storage_mist_2025-03-09_02-30-01.tar.gz .
+# sudo tar -xzf k8s_storage_*.tar.gz -C /home/kimjunte/k8s_storage

github_runner/install/values.yaml

@@ -0,0 +1,20 @@
+runner:
+  name: mist-runner
+  labels:
+    - mist
+    - self-hosted
+
+  envFrom:
+    - secretRef:
+        name: aws-secrets
+
+  volumeMounts:
+    - name: k8s-storage
+      mountPath: /k8s_storage
+      readOnly: true
+
+  volumes:
+    - name: k8s-storage
+      hostPath:
+        path: /home/kimjunte/k8s_storage
+        type: Directory

mist_infra/README.md

@@ -0,0 +1,8 @@
+./scripts/bootstrap_microk8s.sh
+./mist_infra/cert_manager/install_cert_manager.sh
+./mist_infra/arc/update_arc.sh
+
+
+for each clusteR:
+
+kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.14.4/cert-manager.crds.yaml

mist_infra/arc/update_arc.sh

@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# ==========================================================
+# Update / Install GitHub Actions Runner Controller (ARC)
+#
+# - Safe to run multiple times
+# - Applies changes from arc/values.yaml
+# - Does NOT assume fresh cluster
+#
+# ==========================================================
+
+NAMESPACE="actions-runner-system"
+RELEASE_NAME="actions-runner-controller"
+CHART="actions-runner-controller/actions-runner-controller"
+VALUES_FILE="$(dirname "$0")/values.yaml"
+
+echo "=== Updating ARC (GitHub Actions Runner Controller) ==="
+
+echo "→ Ensuring namespace exists: $NAMESPACE"
+kubectl create namespace "$NAMESPACE" \
+  --dry-run=client -o yaml | kubectl apply -f -
+
+echo "→ Adding Helm repo (if missing)"
+helm repo add actions-runner-controller \
+  https://actions-runner-controller.github.io/actions-runner-controller \
+  >/dev/null 2>&1 || true
+
+helm repo update
+
+echo "→ Applying Helm upgrade"
+helm upgrade --install \
+  "$RELEASE_NAME" \
+  "$CHART" \
+  -n "$NAMESPACE" \
+  -f "$VALUES_FILE"
+
+echo
+echo "✅ ARC update complete"
+echo
+echo "Next steps:"
+echo "- kubectl get pods -n $NAMESPACE"
+echo "- kubectl get runners"

mist_infra/arc/values.yaml

@@ -0,0 +1,19 @@
+runner:
+  labels:
+    - mist
+    - self-hosted
+
+  envFrom:
+    - secretRef:
+        name: aws-secrets
+
+  volumeMounts:
+    - name: k8s-storage
+      mountPath: /k8s_storage
+      readOnly: true
+
+  volumes:
+    - name: k8s-storage
+      hostPath:
+        path: /home/kimjunte/k8s_storage
+        type: Directory

mist_infra/cert_manager/install_cert_manager.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.14.4/cert-manager.crds.yaml
+
+helm repo add jetstack https://charts.jetstack.io >/dev/null 2>&1 || true
+helm repo update
+
+kubectl create namespace cert-manager --dry-run=client -o yaml | kubectl apply -f -
+
+helm upgrade --install cert-manager jetstack/cert-manager \
+  --namespace cert-manager \
+  --version v1.14.4

mist_infra/scripts/backup_k9s_storage_to_s3.sh

@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# ----------------------------------------------------------
+# Weekly full backup of all Kubernetes PV data
+# ----------------------------------------------------------
+
+SOURCE_DIR="/k8s_storage"
+TMP_DIR="/tmp/mist-backups"
+BUCKET="mist-backups"
+
+DATE="$(date -u +%Y-%m-%d)"
+TIMESTAMP="$(date -u +%Y-%m-%d_%H-%M-%S)"
+HOST="$(hostname)"
+
+ARCHIVE_NAME="k8s_storage_${HOST}_${TIMESTAMP}.tar.gz"
+ARCHIVE_PATH="${TMP_DIR}/${ARCHIVE_NAME}"
+
+echo "=== Mist weekly PV backup ==="
+echo "Source:   ${SOURCE_DIR}"
+echo "Archive:  ${ARCHIVE_PATH}"
+echo "Bucket:   s3://${BUCKET}/${DATE}/"
+
+mkdir -p "${TMP_DIR}"
+
+echo "→ Creating tarball"
+tar \
+  --numeric-owner \
+  --xattrs \
+  --acls \
+  -czf "${ARCHIVE_PATH}" \
+  -C "${SOURCE_DIR}" .
+
+echo "→ Uploading to S3"
+aws s3 cp \
+  "${ARCHIVE_PATH}" \
+  "s3://${BUCKET}/${DATE}/${ARCHIVE_NAME}"
+
+echo "→ Cleaning up local temp"
+rm -f "${ARCHIVE_PATH}"
+
+echo "✅ Backup complete"