Add dashboards.archlinux.org for public Grafana dashboards

Co-authored-by: Kristian Klausen <kristian@klausen.dk>

group_vars/all/root_access.yml

@@ -26,6 +26,7 @@ root_ssh_keys:
       - monitoring.archlinux.org
       - runner1.archlinux.org
       - runner2.archlinux.org
+      - dashboards.archlinux.org
 
 # run playbook 'playbooks/tasks/reencrypt-vault-key.yml' when this changes
 # before running it, make sure to gpg --lsign-key all of the below keys

host_vars/dashboards.archlinux.org

@@ -0,0 +1,3 @@
+---
+filesystem: btrfs
+ipv4_address: 157.90.255.107

hosts

@@ -142,6 +142,7 @@ repro2.pkgbuild.com
 runner1.archlinux.org
 md.archlinux.org
 man.archlinux.org
+dashboards.archlinux.org
 
 [kape_servers]
 asia.mirror.pkgbuild.com

playbooks/dashboards.archlinux.org.yml

@@ -0,0 +1,18 @@
+- name: setup public dashboards server
+  hosts: dashboards.archlinux.org
+  remote_user: root
+  roles:
+    - { role: firewalld }
+    - { role: common }
+    - { role: tools }
+    - { role: sshd }
+    - { role: root_ssh }
+    - { role: hardening }
+    - { role: borg_client, tags: ["borg"], when: "'borg_clients' in group_names" }
+    - { role: prometheus }
+    - { role: prometheus_exporters }
+    - { role: promtail }
+    - { role: certbot }
+    - { role: nginx }
+    - { role: grafana, grafana_domain: 'dashboards.archlinux.org' }
+    - { role: fail2ban }

roles/prometheus/defaults/main.yml

@@ -17,6 +17,7 @@ blackbox_targets:
     - https://bbs.archlinux.org
     - https://bugs.archlinux.org
     - https://conf.archlinux.org
+    - https://dashboards.archlinux.org/healthz
     - https://dev.archlinux.org
     - https://europe.archive.pkgbuild.com
     - https://europe.mirror.pkgbuild.com

roles/prometheus/files/node.rules.yml

@@ -112,7 +112,7 @@ groups:
           summary: "Prometheus too many restarts (instance {{ $labels.instance }})"
           description: "Prometheus has restarted more than twice in the last 15 minutes. It might be crashlooping.\n  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"
       - alert: PrometheusNotConnectedToAlertmanager
-        expr: prometheus_notifications_alertmanagers_discovered < 1
+        expr: prometheus_notifications_alertmanagers_discovered{instance!~"dashboards.archlinux.org"} < 1
         for: 5m
         labels:
           severity: critical

tf-stage1/archlinux.tf

@@ -102,6 +102,10 @@ locals {
       server_type = "cx31"
       domain      = "monitoring"
     }
+    "dashboards.archlinux.org" = {
+      server_type = "cx11"
+      domain      = "dashboards"
+    }
     "patchwork.archlinux.org" = {
       server_type = "cx11"
       domain      = "patchwork"