From d2b110d2501dff507728d57a403f9429612652a7 Mon Sep 17 00:00:00 2001
From: Sven-Hendrik Haase <svenstaro@gmail.com>
Date: Fri, 30 Apr 2021 18:54:03 +0200
Subject: [PATCH] Add dashboards.archlinux.org for public Grafana dashboards

Co-authored-by: Kristian Klausen <kristian@klausen.dk>
---
 group_vars/all/root_access.yml         |  1 +
 host_vars/dashboards.archlinux.org     |  3 +++
 hosts                                  |  1 +
 playbooks/dashboards.archlinux.org.yml | 18 ++++++++++++++++++
 roles/prometheus/defaults/main.yml     |  1 +
 roles/prometheus/files/node.rules.yml  |  2 +-
 tf-stage1/archlinux.tf                 |  4 ++++
 7 files changed, 29 insertions(+), 1 deletion(-)
 create mode 100644 host_vars/dashboards.archlinux.org
 create mode 100644 playbooks/dashboards.archlinux.org.yml

diff --git a/group_vars/all/root_access.yml b/group_vars/all/root_access.yml
index 3a77cf90..f88964be 100644
--- a/group_vars/all/root_access.yml
+++ b/group_vars/all/root_access.yml
@@ -26,6 +26,7 @@ root_ssh_keys:
       - monitoring.archlinux.org
       - runner1.archlinux.org
       - runner2.archlinux.org
+      - dashboards.archlinux.org
 
 # run playbook 'playbooks/tasks/reencrypt-vault-key.yml' when this changes
 # before running it, make sure to gpg --lsign-key all of the below keys
diff --git a/host_vars/dashboards.archlinux.org b/host_vars/dashboards.archlinux.org
new file mode 100644
index 00000000..827585ee
--- /dev/null
+++ b/host_vars/dashboards.archlinux.org
@@ -0,0 +1,3 @@
+---
+filesystem: btrfs
+ipv4_address: 157.90.255.107
diff --git a/hosts b/hosts
index a5af4aa2..bd6a829e 100644
--- a/hosts
+++ b/hosts
@@ -142,6 +142,7 @@ repro2.pkgbuild.com
 runner1.archlinux.org
 md.archlinux.org
 man.archlinux.org
+dashboards.archlinux.org
 
 [kape_servers]
 asia.mirror.pkgbuild.com
diff --git a/playbooks/dashboards.archlinux.org.yml b/playbooks/dashboards.archlinux.org.yml
new file mode 100644
index 00000000..81512f42
--- /dev/null
+++ b/playbooks/dashboards.archlinux.org.yml
@@ -0,0 +1,18 @@
+- name: setup public dashboards server
+  hosts: dashboards.archlinux.org
+  remote_user: root
+  roles:
+    - { role: firewalld }
+    - { role: common }
+    - { role: tools }
+    - { role: sshd }
+    - { role: root_ssh }
+    - { role: hardening }
+    - { role: borg_client, tags: ["borg"], when: "'borg_clients' in group_names" }
+    - { role: prometheus }
+    - { role: prometheus_exporters }
+    - { role: promtail }
+    - { role: certbot }
+    - { role: nginx }
+    - { role: grafana, grafana_domain: 'dashboards.archlinux.org' }
+    - { role: fail2ban }
diff --git a/roles/prometheus/defaults/main.yml b/roles/prometheus/defaults/main.yml
index 4bec05d9..bd833b8b 100644
--- a/roles/prometheus/defaults/main.yml
+++ b/roles/prometheus/defaults/main.yml
@@ -17,6 +17,7 @@ blackbox_targets:
     - https://bbs.archlinux.org
     - https://bugs.archlinux.org
     - https://conf.archlinux.org
+    - https://dashboards.archlinux.org/healthz
     - https://dev.archlinux.org
     - https://europe.archive.pkgbuild.com
     - https://europe.mirror.pkgbuild.com
diff --git a/roles/prometheus/files/node.rules.yml b/roles/prometheus/files/node.rules.yml
index 73766e2c..78a83ef5 100644
--- a/roles/prometheus/files/node.rules.yml
+++ b/roles/prometheus/files/node.rules.yml
@@ -112,7 +112,7 @@ groups:
           summary: "Prometheus too many restarts (instance {{ $labels.instance }})"
           description: "Prometheus has restarted more than twice in the last 15 minutes. It might be crashlooping.\n  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"
       - alert: PrometheusNotConnectedToAlertmanager
-        expr: prometheus_notifications_alertmanagers_discovered < 1
+        expr: prometheus_notifications_alertmanagers_discovered{instance!~"dashboards.archlinux.org"} < 1
         for: 5m
         labels:
           severity: critical
diff --git a/tf-stage1/archlinux.tf b/tf-stage1/archlinux.tf
index 2b84304c..230f1d6d 100644
--- a/tf-stage1/archlinux.tf
+++ b/tf-stage1/archlinux.tf
@@ -102,6 +102,10 @@ locals {
       server_type = "cx31"
       domain      = "monitoring"
     }
+    "dashboards.archlinux.org" = {
+      server_type = "cx11"
+      domain      = "dashboards"
+    }
     "patchwork.archlinux.org" = {
       server_type = "cx11"
       domain      = "patchwork"