Configure Alertmanager email alerts

apps/monitoring/alertmanager-email-app.yaml

@@ -0,0 +1,23 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: alertmanager-email
+  namespace: argocd
+spec:
+  project: default
+
+  source:
+    repoURL: http://git.home.lan/jay/k8s-gitops.git
+    targetRevision: HEAD
+    path: apps/monitoring/alertmanager-email
+
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: monitoring
+
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+    syncOptions:
+      - CreateNamespace=false

apps/monitoring/alertmanager-email/alertmanager-secret.yaml

@@ -0,0 +1,64 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: alertmanager-monitoring-kube-prometheus-alertmanager
+  namespace: monitoring
+type: Opaque
+stringData:
+  alertmanager.yaml: |
+    global:
+      resolve_timeout: 5m
+      smtp_smarthost: 'mail.home.lan:25'
+      smtp_from: 'alertmanager@home.lan'
+      smtp_hello: 'kubernetes.home.lan'
+      smtp_require_tls: false
+
+    route:
+      receiver: jay-email
+      group_by:
+        - namespace
+        - alertname
+        - severity
+      group_wait: 30s
+      group_interval: 5m
+      repeat_interval: 4h
+      routes:
+        - receiver: "null"
+          matchers:
+            - alertname = "Watchdog"
+
+    receivers:
+      - name: "null"
+
+      - name: jay-email
+        email_configs:
+          - to: 'jay@home.lan'
+            send_resolved: true
+            headers:
+              From: 'Kubernetes Alertmanager <alertmanager@home.lan>'
+              To: 'Jay <jay@home.lan>'
+              Subject: '[Kubernetes] {{ .Status | toUpper }}: {{ .CommonLabels.alertname }}'
+
+    inhibit_rules:
+      - source_matchers:
+          - severity = critical
+        target_matchers:
+          - severity = warning
+        equal:
+          - namespace
+          - alertname
+
+      - source_matchers:
+          - severity = warning
+        target_matchers:
+          - severity = info
+        equal:
+          - namespace
+          - alertname
+
+      - source_matchers:
+          - alertname = InfoInhibitor
+        target_matchers:
+          - severity = info
+        equal:
+          - namespace

apps/monitoring/alertmanager-email/kustomization.yaml

@@ -0,0 +1,7 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - alertmanager-secret.yaml
+  - test-alert.yaml
+  - alertmanager-email

apps/monitoring/alertmanager-email/test-alert.yaml

@@ -0,0 +1,19 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: alertmanager-email-test
+  namespace: monitoring
+  labels:
+    release: monitoring
+spec:
+  groups:
+    - name: jay-email-test.rules
+      rules:
+        - alert: JayEmailTest
+          expr: vector(1)
+          for: 1m
+          labels:
+            severity: warning
+          annotations:
+            summary: "Kubernetes email alert test"
+            description: "This is a temporary test alert from Alertmanager to jay@home.lan."