From 64d4bf9c0b4ec62454e1e5a859b0a59df6343c07 Mon Sep 17 00:00:00 2001 From: Daniel Sy Date: Wed, 30 Jul 2025 14:35:42 +0200 Subject: [PATCH 1/5] =?UTF-8?q?feat(manifest):=20=F0=9F=8E=89=20WIP=20Add?= =?UTF-8?q?=20CronJob=20and=20Secret=20for=20S3=20backups?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a new CronJob for scheduled S3 backups using rclone, along with a corresponding Secret for AWS credentials. This introduces automated backup functionality for the Forgejo server, enhancing data protection and recovery capabilities. --- .../manifests/forgejo-s3-backup-cronjob.yaml | 64 +++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 template/stacks/forgejo/forgejo-server/manifests/forgejo-s3-backup-cronjob.yaml diff --git a/template/stacks/forgejo/forgejo-server/manifests/forgejo-s3-backup-cronjob.yaml b/template/stacks/forgejo/forgejo-server/manifests/forgejo-s3-backup-cronjob.yaml new file mode 100644 index 0000000..769cd0d --- /dev/null +++ b/template/stacks/forgejo/forgejo-server/manifests/forgejo-s3-backup-cronjob.yaml @@ -0,0 +1,64 @@ +apiVersion: batch/v1 +kind: CronJob +metadata: + name: forgejo-s3-backup + namespace: gitea +spec: + schedule: "24 * * * *" + jobTemplate: + spec: + template: + spec: + containers: + - name: rclone + image: rclone/rclone:1.70 + imagePullPolicy: IfNotPresent + env: + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: forgejo-cloud-credentials + key: access-key + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: forgejo-cloud-credentials + key: secret-key + volumeMounts: + - name: rclone-config + mountPath: /etc/rclone + readOnly: true + command: + - /bin/sh + - -c + - | + sleep 7d + # rclone sync remote-source:packages remote-destination:packages --config /etc/rclone/config + restartPolicy: OnFailure + volumes: + - name: rclone-config + secret: + secretName: forgejo-s3-backup + +--- +apiVersion: v1 +kind: Secret +metadata: + name: forgejo-s3-backup + namespace: gitea +type: Opaque +stringData: + config: | + [remote-source] + type = s3 + provider = AWS + env_auth = true + endpoint = https://edp-forgejo-non-prod-observability.obs.eu-de.otc.t-systems.com + region = eu-de + + [remote-destination] + type = s3 + provider = AWS + env_auth = true + endpoint = https://edp-forgejo-backup-test-manu.obs.eu-de.otc.t-systems.com + region = eu-de -- 2.47.3 From a87633f2e560dcf2872209c57d49a84bbdf91a0a Mon Sep 17 00:00:00 2001 From: Richard Robert Reitz Date: Thu, 31 Jul 2025 15:24:39 +0200 Subject: [PATCH 2/5] fix(s3backup): doing a local backup first and then push it to remote, which is still on the same OBS store --- .../manifests/forgejo-s3-backup-cronjob.yaml | 43 ++++++++++++++----- 1 file changed, 33 insertions(+), 10 deletions(-) diff --git a/template/stacks/forgejo/forgejo-server/manifests/forgejo-s3-backup-cronjob.yaml b/template/stacks/forgejo/forgejo-server/manifests/forgejo-s3-backup-cronjob.yaml index 769cd0d..d7b78e6 100644 --- a/template/stacks/forgejo/forgejo-server/manifests/forgejo-s3-backup-cronjob.yaml +++ b/template/stacks/forgejo/forgejo-server/manifests/forgejo-s3-backup-cronjob.yaml @@ -4,7 +4,7 @@ metadata: name: forgejo-s3-backup namespace: gitea spec: - schedule: "24 * * * *" + schedule: "0 2 * * *" jobTemplate: spec: template: @@ -26,20 +26,41 @@ spec: key: secret-key volumeMounts: - name: rclone-config - mountPath: /etc/rclone + mountPath: /config/rclone readOnly: true + - name: backup-dir + mountPath: /backup_dir + readOnly: false command: - /bin/sh - -c - | - sleep 7d - # rclone sync remote-source:packages remote-destination:packages --config /etc/rclone/config + rm -Rf /backup_dir/backup || true + mkdir -p /backup_dir/backup + rclone sync remote-source:/edp-forgejo-non-prod-observability/packages /backup_dir/backup -v --ignore-checksum + rclone sync /backup_dir/backup remote-destination:/edp-forgejo-non-prod-observability/hackathon3 -v --ignore-checksum + rm -Rf /backup_dir/backup || true restartPolicy: OnFailure volumes: - name: rclone-config secret: secretName: forgejo-s3-backup - + - name: backup-dir + persistentVolumeClaim: + claimName: s3-temp-data +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: s3-temp-data + namespace: gitea +spec: + accessModes: + - ReadWriteOnce + volumeMode: Filesystem + resources: + requests: + storage: 50Gi --- apiVersion: v1 kind: Secret @@ -48,17 +69,19 @@ metadata: namespace: gitea type: Opaque stringData: - config: | + rclone.conf: | [remote-source] type = s3 - provider = AWS + provider = HuaweiOBS env_auth = true - endpoint = https://edp-forgejo-non-prod-observability.obs.eu-de.otc.t-systems.com + endpoint = obs.eu-de.otc.t-systems.com region = eu-de + acl = private [remote-destination] type = s3 - provider = AWS + provider = HuaweiOBS env_auth = true - endpoint = https://edp-forgejo-backup-test-manu.obs.eu-de.otc.t-systems.com + endpoint = obs.eu-de.otc.t-systems.com region = eu-de + acl = private -- 2.47.3 From 414054b466db30d0134785943dd2b7951bf858eb Mon Sep 17 00:00:00 2001 From: "Fritz-Leo.Ochsmann" Date: Thu, 31 Jul 2025 15:59:25 +0200 Subject: [PATCH 3/5] feat(forgejo): backup s3 directly to pvc --- .../manifests/forgejo-s3-backup-cronjob.yaml | 25 +++++-------------- .../stacks/forgejo/forgejo-server/values.yaml | 1 + 2 files changed, 7 insertions(+), 19 deletions(-) diff --git a/template/stacks/forgejo/forgejo-server/manifests/forgejo-s3-backup-cronjob.yaml b/template/stacks/forgejo/forgejo-server/manifests/forgejo-s3-backup-cronjob.yaml index d7b78e6..223188a 100644 --- a/template/stacks/forgejo/forgejo-server/manifests/forgejo-s3-backup-cronjob.yaml +++ b/template/stacks/forgejo/forgejo-server/manifests/forgejo-s3-backup-cronjob.yaml @@ -4,7 +4,7 @@ metadata: name: forgejo-s3-backup namespace: gitea spec: - schedule: "0 2 * * *" + schedule: "0 1 * * *" jobTemplate: spec: template: @@ -29,17 +29,13 @@ spec: mountPath: /config/rclone readOnly: true - name: backup-dir - mountPath: /backup_dir + mountPath: /backup readOnly: false command: - /bin/sh - -c - | - rm -Rf /backup_dir/backup || true - mkdir -p /backup_dir/backup - rclone sync remote-source:/edp-forgejo-non-prod-observability/packages /backup_dir/backup -v --ignore-checksum - rclone sync /backup_dir/backup remote-destination:/edp-forgejo-non-prod-observability/hackathon3 -v --ignore-checksum - rm -Rf /backup_dir/backup || true + rclone sync source:/${SOURCE_BUCKET}/packages /backup -v --ignore-checksum restartPolicy: OnFailure volumes: - name: rclone-config @@ -47,17 +43,16 @@ spec: secretName: forgejo-s3-backup - name: backup-dir persistentVolumeClaim: - claimName: s3-temp-data + claimName: s3-backup --- apiVersion: v1 kind: PersistentVolumeClaim metadata: - name: s3-temp-data + name: s3-backup namespace: gitea spec: accessModes: - ReadWriteOnce - volumeMode: Filesystem resources: requests: storage: 50Gi @@ -70,15 +65,7 @@ metadata: type: Opaque stringData: rclone.conf: | - [remote-source] - type = s3 - provider = HuaweiOBS - env_auth = true - endpoint = obs.eu-de.otc.t-systems.com - region = eu-de - acl = private - - [remote-destination] + [source] type = s3 provider = HuaweiOBS env_auth = true diff --git a/template/stacks/forgejo/forgejo-server/values.yaml b/template/stacks/forgejo/forgejo-server/values.yaml index 3b354fe..d8cd9dc 100644 --- a/template/stacks/forgejo/forgejo-server/values.yaml +++ b/template/stacks/forgejo/forgejo-server/values.yaml @@ -1,3 +1,4 @@ +# This is only used for deploying older versions of infra-catalogue where the bucket name is not an output of the terragrunt modules {{{- define "BUCKET_NAME" -}}} {{{- if (getenv "FORGEJO_BUCKET_NAME") -}}} {{{ getenv "FORGEJO_BUCKET_NAME" }}} -- 2.47.3 From ccd9d08ec2fed149de2dac4c539e04e51a90733c Mon Sep 17 00:00:00 2001 From: evdo Date: Fri, 1 Aug 2025 10:18:38 +0200 Subject: [PATCH 4/5] feat(forgejo): updated secret ref for a bucket name --- .../forgejo-server/manifests/forgejo-s3-backup-cronjob.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/template/stacks/forgejo/forgejo-server/manifests/forgejo-s3-backup-cronjob.yaml b/template/stacks/forgejo/forgejo-server/manifests/forgejo-s3-backup-cronjob.yaml index 223188a..ba0aebd 100644 --- a/template/stacks/forgejo/forgejo-server/manifests/forgejo-s3-backup-cronjob.yaml +++ b/template/stacks/forgejo/forgejo-server/manifests/forgejo-s3-backup-cronjob.yaml @@ -14,6 +14,11 @@ spec: image: rclone/rclone:1.70 imagePullPolicy: IfNotPresent env: + - name: SOURCE_BUCKET + valueFrom: + secretKeyRef: + name: forgejo-cloud-credentials + key: bucket-name - name: AWS_ACCESS_KEY_ID valueFrom: secretKeyRef: -- 2.47.3 From ddc7ed4905debd4c194d738eef00177d4fa548be Mon Sep 17 00:00:00 2001 From: "Fritz-Leo.Ochsmann" Date: Fri, 8 Aug 2025 15:24:04 +0200 Subject: [PATCH 5/5] chore: set default storage class to csi-disk driver --- .../forgejo-server/manifests/forgejo-s3-backup-cronjob.yaml | 4 ++++ template/stacks/forgejo/forgejo-server/values.yaml | 2 ++ .../observability/grafana-operator/manifests/grafana.yaml | 4 ++++ .../observability/victoria-k8s-stack/manifests/vlogs.yaml | 4 +++- template/stacks/observability/victoria-k8s-stack/values.yaml | 4 +++- 5 files changed, 16 insertions(+), 2 deletions(-) diff --git a/template/stacks/forgejo/forgejo-server/manifests/forgejo-s3-backup-cronjob.yaml b/template/stacks/forgejo/forgejo-server/manifests/forgejo-s3-backup-cronjob.yaml index ba0aebd..e5ea7df 100644 --- a/template/stacks/forgejo/forgejo-server/manifests/forgejo-s3-backup-cronjob.yaml +++ b/template/stacks/forgejo/forgejo-server/manifests/forgejo-s3-backup-cronjob.yaml @@ -55,7 +55,11 @@ kind: PersistentVolumeClaim metadata: name: s3-backup namespace: gitea + annotations: + everest.io/disk-volume-type: SATA + everest.io/crypt-key-id: {{{ .Env.PVC_KMS_KEY_ID }}} spec: + storageClassName: csi-disk accessModes: - ReadWriteOnce resources: diff --git a/template/stacks/forgejo/forgejo-server/values.yaml b/template/stacks/forgejo/forgejo-server/values.yaml index d8cd9dc..a7d7335 100644 --- a/template/stacks/forgejo/forgejo-server/values.yaml +++ b/template/stacks/forgejo/forgejo-server/values.yaml @@ -28,8 +28,10 @@ postgresql-ha: persistence: enabled: true size: 200Gi + storageClass: csi-disk annotations: everest.io/crypt-key-id: {{{ .Env.PVC_KMS_KEY_ID }}} + everest.io/disk-volume-type: SATA test: enabled: false diff --git a/template/stacks/observability/grafana-operator/manifests/grafana.yaml b/template/stacks/observability/grafana-operator/manifests/grafana.yaml index 87bc732..1c47357 100644 --- a/template/stacks/observability/grafana-operator/manifests/grafana.yaml +++ b/template/stacks/observability/grafana-operator/manifests/grafana.yaml @@ -6,7 +6,11 @@ metadata: dashboards: "grafana" spec: persistentVolumeClaim: + metadata: + annotations: + everest.io/disk-volume-type: SATA spec: + storageClassName: csi-disk accessModes: - ReadWriteOnce resources: diff --git a/template/stacks/observability/victoria-k8s-stack/manifests/vlogs.yaml b/template/stacks/observability/victoria-k8s-stack/manifests/vlogs.yaml index 4c6fbe9..c74f8d5 100644 --- a/template/stacks/observability/victoria-k8s-stack/manifests/vlogs.yaml +++ b/template/stacks/observability/victoria-k8s-stack/manifests/vlogs.yaml @@ -9,7 +9,9 @@ spec: storageMetadata: annotations: everest.io/crypt-key-id: {{{ .Env.PVC_KMS_KEY_ID }}} + everest.io/disk-volume-type: SATA storage: + storageClassName: csi-disk accessModes: - ReadWriteOnce resources: @@ -21,4 +23,4 @@ spec: cpu: 500m limits: memory: 10Gi - cpu: 2 \ No newline at end of file + cpu: 2 diff --git a/template/stacks/observability/victoria-k8s-stack/values.yaml b/template/stacks/observability/victoria-k8s-stack/values.yaml index db459f3..78c705d 100644 --- a/template/stacks/observability/victoria-k8s-stack/values.yaml +++ b/template/stacks/observability/victoria-k8s-stack/values.yaml @@ -289,7 +289,9 @@ vmsingle: storageMetadata: annotations: everest.io/crypt-key-id: {{{ .Env.PVC_KMS_KEY_ID }}} + everest.io/disk-volume-type: SATA storage: + storageClassName: csi-disk accessModes: - ReadWriteOnce resources: @@ -880,7 +882,7 @@ grafana: enabled: false # all values for grafana helm chart can be specified here persistence: - enabled: true + enabled: false type: pvc storageClassName: "default" grafana.ini: -- 2.47.3