From e0f6cc77dd43f8adc073f0b1cd5e40e2ee17f92d Mon Sep 17 00:00:00 2001
From: Patrick Sy <patrick.sy@telekom.de>
Date: Tue, 5 Aug 2025 11:32:28 +0200
Subject: [PATCH 1/3] fix(observability): Added missing encryption to grafana
 volume

---
 .../stacks/observability/grafana-operator/manifests/grafana.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/template/stacks/observability/grafana-operator/manifests/grafana.yaml b/template/stacks/observability/grafana-operator/manifests/grafana.yaml
index 1c47357..c5fa295 100644
--- a/template/stacks/observability/grafana-operator/manifests/grafana.yaml
+++ b/template/stacks/observability/grafana-operator/manifests/grafana.yaml
@@ -9,6 +9,7 @@ spec:
     metadata:
       annotations:
         everest.io/disk-volume-type: SATA
+        everest.io/crypt-key-id: {{{ .Env.PVC_KMS_KEY_ID }}}
     spec:
       storageClassName: csi-disk
       accessModes:

From 975bb6b982cf0db3510900198d3276ffaa9928e0 Mon Sep 17 00:00:00 2001
From: Patrick Sy <patrick.sy@telekom.de>
Date: Tue, 12 Aug 2025 14:07:38 +0200
Subject: [PATCH 2/3] feat(observability): Introduced alert for failed s3
 backup jobs

---
 .../victoria-k8s-stack/manifests/alerts.yaml          | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/template/stacks/observability/victoria-k8s-stack/manifests/alerts.yaml b/template/stacks/observability/victoria-k8s-stack/manifests/alerts.yaml
index f884bd9..8accb1e 100644
--- a/template/stacks/observability/victoria-k8s-stack/manifests/alerts.yaml
+++ b/template/stacks/observability/victoria-k8s-stack/manifests/alerts.yaml
@@ -16,3 +16,14 @@ spec:
           annotations:
             value: "{{ $value }}"
             description: 'forgejo is down in cluster environment {{ $labels.cluster_environment }}'
+    - name: forgejo-backup
+      rules:
+        - alert: forgejo s3 backup job failed
+          expr: max by(cluster_environment) (kube_job_status_failed{job_name=~"forgejo-s3-backup-.*"}) != 0
+          for: 30s
+          labels:
+            severity: major
+            job:  "{{ $labels.job }}"
+          annotations:
+            value: "{{ $value }}"
+            description: 'forgejo s3 backup job failed in cluster environment {{ $labels.cluster_environment }}'

From fb64314fb27e1db6020a1433a7fc177f41d24485 Mon Sep 17 00:00:00 2001
From: Patrick Sy <patrick.sy@telekom.de>
Date: Tue, 12 Aug 2025 14:20:01 +0200
Subject: [PATCH 3/3] feat(observability): Introduced alert priority for
 notifications

---
 .../observability/victoria-k8s-stack/manifests/alerts.yaml   | 4 ++--
 template/stacks/observability/victoria-k8s-stack/values.yaml | 5 +++--
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/template/stacks/observability/victoria-k8s-stack/manifests/alerts.yaml b/template/stacks/observability/victoria-k8s-stack/manifests/alerts.yaml
index 8accb1e..9419609 100644
--- a/template/stacks/observability/victoria-k8s-stack/manifests/alerts.yaml
+++ b/template/stacks/observability/victoria-k8s-stack/manifests/alerts.yaml
@@ -11,7 +11,7 @@ spec:
           expr: sum by(cluster_environment) (up{pod=~"forgejo-server-.*"}) < 1
           for: 30s
           labels:
-            severity: major
+            severity: critical
             job:  "{{ $labels.job }}"
           annotations:
             value: "{{ $value }}"
@@ -22,7 +22,7 @@ spec:
           expr: max by(cluster_environment) (kube_job_status_failed{job_name=~"forgejo-s3-backup-.*"}) != 0
           for: 30s
           labels:
-            severity: major
+            severity: critical
             job:  "{{ $labels.job }}"
           annotations:
             value: "{{ $value }}"
diff --git a/template/stacks/observability/victoria-k8s-stack/values.yaml b/template/stacks/observability/victoria-k8s-stack/values.yaml
index f3b5241..e421957 100644
--- a/template/stacks/observability/victoria-k8s-stack/values.yaml
+++ b/template/stacks/observability/victoria-k8s-stack/values.yaml
@@ -542,12 +542,13 @@ alertmanager:
   # -- (object) Alertmanager configuration
   config:
     route:
-      receiver: "outlook"
+      receiver: "blackhole"
       routes: 
         - matchers:
-            - alertname=~".*"
+            - severity=~"critical|major"
           receiver: outlook
     receivers:
+      - name: blackhole
       - name: outlook
         email_configs:
           - smarthost: 'mail.mms-support.de:465'