feat(observability): Introduced alert priority for notifications

template/stacks/observability/victoria-k8s-stack/manifests/alerts.yaml

@@ -11,7 +11,7 @@ spec:
           expr: sum by(cluster_environment) (up{pod=~"forgejo-server-.*"}) < 1
           for: 30s
           labels:
-            severity: major
+            severity: critical
             job:  "{{ $labels.job }}"
           annotations:
             value: "{{ $value }}"
@@ -22,7 +22,7 @@ spec:
           expr: max by(cluster_environment) (kube_job_status_failed{job_name=~"forgejo-s3-backup-.*"}) != 0
           for: 30s
           labels:
-            severity: major
+            severity: critical
             job:  "{{ $labels.job }}"
           annotations:
             value: "{{ $value }}"

template/stacks/observability/victoria-k8s-stack/values.yaml

@@ -542,12 +542,13 @@ alertmanager:
   # -- (object) Alertmanager configuration
   config:
     route:
-      receiver: "outlook"
+      receiver: "blackhole"
       routes: 
         - matchers:
-            - alertname=~".*"
+            - severity=~"critical|major"
           receiver: outlook
     receivers:
+      - name: blackhole
       - name: outlook
         email_configs:
           - smarthost: 'mail.mms-support.de:465'