diff --git a/otc/observability.buildth.ing/stacks/observability/grafana-operator/manifests/grafana.yaml b/otc/observability.buildth.ing/stacks/observability/grafana-operator/manifests/grafana.yaml index af5fa6d..10ab68f 100644 --- a/otc/observability.buildth.ing/stacks/observability/grafana-operator/manifests/grafana.yaml +++ b/otc/observability.buildth.ing/stacks/observability/grafana-operator/manifests/grafana.yaml @@ -9,6 +9,7 @@ spec: metadata: annotations: everest.io/disk-volume-type: SATA + everest.io/crypt-key-id: c02a26f1-3c7e-486d-ba5a-266c321bb203 spec: storageClassName: csi-disk accessModes: diff --git a/otc/observability.buildth.ing/stacks/observability/victoria-k8s-stack/manifests/alerts.yaml b/otc/observability.buildth.ing/stacks/observability/victoria-k8s-stack/manifests/alerts.yaml index f884bd9..9419609 100644 --- a/otc/observability.buildth.ing/stacks/observability/victoria-k8s-stack/manifests/alerts.yaml +++ b/otc/observability.buildth.ing/stacks/observability/victoria-k8s-stack/manifests/alerts.yaml @@ -11,8 +11,19 @@ spec: expr: sum by(cluster_environment) (up{pod=~"forgejo-server-.*"}) < 1 for: 30s labels: - severity: major + severity: critical job: "{{ $labels.job }}" annotations: value: "{{ $value }}" description: 'forgejo is down in cluster environment {{ $labels.cluster_environment }}' + - name: forgejo-backup + rules: + - alert: forgejo s3 backup job failed + expr: max by(cluster_environment) (kube_job_status_failed{job_name=~"forgejo-s3-backup-.*"}) != 0 + for: 30s + labels: + severity: critical + job: "{{ $labels.job }}" + annotations: + value: "{{ $value }}" + description: 'forgejo s3 backup job failed in cluster environment {{ $labels.cluster_environment }}' diff --git a/otc/observability.buildth.ing/stacks/observability/victoria-k8s-stack/values.yaml b/otc/observability.buildth.ing/stacks/observability/victoria-k8s-stack/values.yaml index 554446a..a8ff717 100644 --- a/otc/observability.buildth.ing/stacks/observability/victoria-k8s-stack/values.yaml +++ b/otc/observability.buildth.ing/stacks/observability/victoria-k8s-stack/values.yaml @@ -542,12 +542,13 @@ alertmanager: # -- (object) Alertmanager configuration config: route: - receiver: "outlook" + receiver: "blackhole" routes: - matchers: - - alertname=~".*" + - severity=~"critical|major" receiver: outlook receivers: + - name: blackhole - name: outlook email_configs: - smarthost: 'mail.mms-support.de:465'