Split storage box monitoring into new text collector

This was previously monitored as part of the borg text collector, but
now that it only runs after each backup (instead of hourly) the stats
from monitoring.archlinux.org do not remain accurate for long. Switch
back to hourly checks of the storage box's disk usage by adding a new
text collector just for this purpose.

roles/prometheus_exporters/files/borg-textcollector.sh

@@ -34,23 +34,6 @@ if [[ -f /usr/local/bin/borg ]]; then
   echo "# HELP borg_hetzner_repo_size_bytes amount of data stored in the repo in bytes" >> $TMP_FILE
   echo "# TYPE borg_hetzner_repo_size_bytes gauge" >> $TMP_FILE
   echo "borg_hetzner_repo_size_bytes $REPO_SIZE" >> $TMP_FILE
-
-  STORAGE_BOX_DF=$(sftp -P23 u236610.your-storagebox.de <<<df 2>/dev/null | tail -1)
-  STORAGE_BOX_SIZE=$(( 1024 * $(awk '{print $1}' <<<$STORAGE_BOX_DF) )) # KiB -> bytes
-  STORAGE_BOX_USED=$(( 1024 * $(awk '{print $2}' <<<$STORAGE_BOX_DF) )) # KiB -> bytes
-  STORAGE_BOX_FREE=$(( 1024 * $(awk '{print $3}' <<<$STORAGE_BOX_DF) )) # KiB -> bytes
-
-  echo "# HELP hetzner_storage_box_size_bytes storage box size in bytes (excl. snapshots)" >> $TMP_FILE
-  echo "# TYPE hetzner_storage_box_size_bytes gauge" >> $TMP_FILE
-  echo "hetzner_storage_box_size_bytes $STORAGE_BOX_SIZE" >> $TMP_FILE
-
-  echo "# HELP hetzner_storage_box_used_bytes storage box used space in bytes" >> $TMP_FILE
-  echo "# TYPE hetzner_storage_box_used_bytes gauge" >> $TMP_FILE
-  echo "hetzner_storage_box_used_bytes $STORAGE_BOX_USED" >> $TMP_FILE
-
-  echo "# HELP hetzner_storage_box_free_bytes storage box free space in bytes" >> $TMP_FILE
-  echo "# TYPE hetzner_storage_box_free_bytes gauge" >> $TMP_FILE
-  echo "hetzner_storage_box_free_bytes $STORAGE_BOX_FREE" >> $TMP_FILE
 fi
 
 mv -f $TMP_FILE $PROM_FILE

roles/prometheus_exporters/files/hetzner-textcollector.sh

@@ -0,0 +1,38 @@
+#!/usr/bin/bash
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+if (( $# != 1 )); then
+  echo "Missing textcollector directory argument"
+  exit 1
+fi
+
+TEXTFILE_COLLECTOR_DIR=${1}
+PROM_FILE=$TEXTFILE_COLLECTOR_DIR/hetzner.prom
+
+
+TMP_FILE=$PROM_FILE.$$
+[ -e $TMP_FILE ] && rm -f $TMP_FILE
+
+trap "rm -f $TMP_FILE" EXIT
+
+STORAGE_BOX_DF=$(sftp -P23 u236610.your-storagebox.de <<<df 2>/dev/null | tail -1)
+STORAGE_BOX_SIZE=$(( 1024 * $(awk '{print $1}' <<<$STORAGE_BOX_DF) )) # KiB -> bytes
+STORAGE_BOX_USED=$(( 1024 * $(awk '{print $2}' <<<$STORAGE_BOX_DF) )) # KiB -> bytes
+STORAGE_BOX_FREE=$(( 1024 * $(awk '{print $3}' <<<$STORAGE_BOX_DF) )) # KiB -> bytes
+
+echo "# HELP hetzner_storage_box_size_bytes storage box size in bytes (excl. snapshots)" >> $TMP_FILE
+echo "# TYPE hetzner_storage_box_size_bytes gauge" >> $TMP_FILE
+echo "hetzner_storage_box_size_bytes $STORAGE_BOX_SIZE" >> $TMP_FILE
+
+echo "# HELP hetzner_storage_box_used_bytes storage box used space in bytes" >> $TMP_FILE
+echo "# TYPE hetzner_storage_box_used_bytes gauge" >> $TMP_FILE
+echo "hetzner_storage_box_used_bytes $STORAGE_BOX_USED" >> $TMP_FILE
+
+echo "# HELP hetzner_storage_box_free_bytes storage box free space in bytes" >> $TMP_FILE
+echo "# TYPE hetzner_storage_box_free_bytes gauge" >> $TMP_FILE
+echo "hetzner_storage_box_free_bytes $STORAGE_BOX_FREE" >> $TMP_FILE
+
+mv -f $TMP_FILE $PROM_FILE

roles/prometheus_exporters/tasks/main.yml

@@ -53,6 +53,7 @@
     - arch-textcollector.sh
     - borg-textcollector.sh
     - borg-offsite-textcollector.sh
+    - hetzner-textcollector.sh
     - rebuilderd-textcollector.sh
     - rebuilderd-status-textcollector.py
     - archive-textcollector.sh
@@ -84,6 +85,18 @@
     - { name: borg-offsite, service: borg-backup-offsite }
   when: "'borg_clients' in group_names"
 
+- name: install hetzner textcollector service
+  template: src=prometheus-hetzner-textcollector.service.j2 dest=/etc/systemd/system/prometheus-hetzner-textcollector.service owner=root group=root mode=644
+  when: "inventory_hostname == 'monitoring.archlinux.org'"
+
+- name: install hetzner textcollector timer
+  template: src=prometheus-hetzner-textcollector.timer.j2 dest=/etc/systemd/system/prometheus-hetzner-textcollector.timer owner=root group=root mode=644
+  when: "inventory_hostname == 'monitoring.archlinux.org'"
+
+- name: enable and start prometheus hetzner textcollector timer
+  systemd: name=prometheus-hetzner-textcollector.timer enabled=yes daemon_reload=yes state=started
+  when: "inventory_hostname == 'monitoring.archlinux.org'"
+
 - name: install fail2ban textcollector service
   template: src=prometheus-fail2ban-textcollector.service.j2 dest=/etc/systemd/system/prometheus-fail2ban-textcollector.service owner=root group=root mode=644
 

roles/prometheus_exporters/templates/prometheus-hetzner-textcollector.service.j2

@@ -0,0 +1,34 @@
+[Unit]
+Description=Prometheus Hetzner Exporter TextCollector
+After=network-online.target
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/bin/hetzner-textcollector.sh {{ prometheus_textfile_dir }}
+
+NoNewPrivileges=true
+LockPersonality=true
+
+PrivateDevices=true
+PrivateTmp=true
+ProtectSystem=strict
+ProtectHome=read-only
+ReadWritePaths={{ prometheus_textfile_dir }}
+
+MemoryDenyWriteExecute=true
+RemoveIPC=true
+RestrictRealtime=true
+RestrictNamespaces=true
+RestrictSUIDSGID=true
+
+RestrictAddressFamilies=~AF_PACKET
+RestrictAddressFamilies=~AF_NETLINK
+
+ProtectHostname=true
+ProtectControlGroups=true
+ProtectKernelLogs=true
+ProtectKernelTunables=true
+ProtectKernelModules=true
+ProtectClock=true
+
+SystemCallArchitectures=native

roles/prometheus_exporters/templates/prometheus-hetzner-textcollector.timer.j2

@@ -0,0 +1,10 @@
+[Unit]
+Description=Prometheus Hetzner Exporter TextCollector Timer
+
+[Timer]
+OnUnitActiveSec=1h
+OnBootSec=15min
+RandomizedDelaySec=1min
+
+[Install]
+WantedBy=timers.target