feat(observability): Introduced alert priority for notifications

template/stacks/observability/grafana-operator/manifests/grafana.yaml

@@ -9,6 +9,7 @@ spec:
     metadata:
       annotations:
         everest.io/disk-volume-type: SATA
+        everest.io/crypt-key-id: {{{ .Env.PVC_KMS_KEY_ID }}}
     spec:
       storageClassName: csi-disk
       accessModes:

template/stacks/observability/victoria-k8s-stack/manifests/alerts.yaml

@@ -11,8 +11,19 @@ spec:
           expr: sum by(cluster_environment) (up{pod=~"forgejo-server-.*"}) < 1
           for: 30s
           labels:
-            severity: major
+            severity: critical
             job:  "{{ $labels.job }}"
           annotations:
             value: "{{ $value }}"
             description: 'forgejo is down in cluster environment {{ $labels.cluster_environment }}'
+    - name: forgejo-backup
+      rules:
+        - alert: forgejo s3 backup job failed
+          expr: max by(cluster_environment) (kube_job_status_failed{job_name=~"forgejo-s3-backup-.*"}) != 0
+          for: 30s
+          labels:
+            severity: critical
+            job:  "{{ $labels.job }}"
+          annotations:
+            value: "{{ $value }}"
+            description: 'forgejo s3 backup job failed in cluster environment {{ $labels.cluster_environment }}'

template/stacks/observability/victoria-k8s-stack/values.yaml

@@ -542,12 +542,13 @@ alertmanager:
   # -- (object) Alertmanager configuration
   config:
     route:
-      receiver: "outlook"
+      receiver: "blackhole"
       routes: 
         - matchers:
-            - alertname=~".*"
+            - severity=~"critical|major"
           receiver: outlook
     receivers:
+      - name: blackhole
       - name: outlook
         email_configs:
           - smarthost: 'mail.mms-support.de:465'