1
1
mirror of https://gitlab.archlinux.org/archlinux/infrastructure.git synced 2024-11-15 16:03:08 +01:00

Split storage box monitoring into new text collector

This was previously monitored as part of the borg text collector, but
now that it only runs after each backup (instead of hourly) the stats
from monitoring.archlinux.org do not remain accurate for long. Switch
back to hourly checks of the storage box's disk usage by adding a new
text collector just for this purpose.
This commit is contained in:
Evangelos Foutras 2021-07-19 01:11:30 +03:00
parent 19199f6fb0
commit c844d0cb6c
No known key found for this signature in database
GPG Key ID: 51E8B148A9999C34
5 changed files with 95 additions and 17 deletions

@ -34,23 +34,6 @@ if [[ -f /usr/local/bin/borg ]]; then
echo "# HELP borg_hetzner_repo_size_bytes amount of data stored in the repo in bytes" >> $TMP_FILE
echo "# TYPE borg_hetzner_repo_size_bytes gauge" >> $TMP_FILE
echo "borg_hetzner_repo_size_bytes $REPO_SIZE" >> $TMP_FILE
STORAGE_BOX_DF=$(sftp -P23 u236610.your-storagebox.de <<<df 2>/dev/null | tail -1)
STORAGE_BOX_SIZE=$(( 1024 * $(awk '{print $1}' <<<$STORAGE_BOX_DF) )) # KiB -> bytes
STORAGE_BOX_USED=$(( 1024 * $(awk '{print $2}' <<<$STORAGE_BOX_DF) )) # KiB -> bytes
STORAGE_BOX_FREE=$(( 1024 * $(awk '{print $3}' <<<$STORAGE_BOX_DF) )) # KiB -> bytes
echo "# HELP hetzner_storage_box_size_bytes storage box size in bytes (excl. snapshots)" >> $TMP_FILE
echo "# TYPE hetzner_storage_box_size_bytes gauge" >> $TMP_FILE
echo "hetzner_storage_box_size_bytes $STORAGE_BOX_SIZE" >> $TMP_FILE
echo "# HELP hetzner_storage_box_used_bytes storage box used space in bytes" >> $TMP_FILE
echo "# TYPE hetzner_storage_box_used_bytes gauge" >> $TMP_FILE
echo "hetzner_storage_box_used_bytes $STORAGE_BOX_USED" >> $TMP_FILE
echo "# HELP hetzner_storage_box_free_bytes storage box free space in bytes" >> $TMP_FILE
echo "# TYPE hetzner_storage_box_free_bytes gauge" >> $TMP_FILE
echo "hetzner_storage_box_free_bytes $STORAGE_BOX_FREE" >> $TMP_FILE
fi
mv -f $TMP_FILE $PROM_FILE

@ -0,0 +1,38 @@
#!/usr/bin/bash
set -o errexit
set -o nounset
set -o pipefail
if (( $# != 1 )); then
echo "Missing textcollector directory argument"
exit 1
fi
TEXTFILE_COLLECTOR_DIR=${1}
PROM_FILE=$TEXTFILE_COLLECTOR_DIR/hetzner.prom
TMP_FILE=$PROM_FILE.$$
[ -e $TMP_FILE ] && rm -f $TMP_FILE
trap "rm -f $TMP_FILE" EXIT
STORAGE_BOX_DF=$(sftp -P23 u236610.your-storagebox.de <<<df 2>/dev/null | tail -1)
STORAGE_BOX_SIZE=$(( 1024 * $(awk '{print $1}' <<<$STORAGE_BOX_DF) )) # KiB -> bytes
STORAGE_BOX_USED=$(( 1024 * $(awk '{print $2}' <<<$STORAGE_BOX_DF) )) # KiB -> bytes
STORAGE_BOX_FREE=$(( 1024 * $(awk '{print $3}' <<<$STORAGE_BOX_DF) )) # KiB -> bytes
echo "# HELP hetzner_storage_box_size_bytes storage box size in bytes (excl. snapshots)" >> $TMP_FILE
echo "# TYPE hetzner_storage_box_size_bytes gauge" >> $TMP_FILE
echo "hetzner_storage_box_size_bytes $STORAGE_BOX_SIZE" >> $TMP_FILE
echo "# HELP hetzner_storage_box_used_bytes storage box used space in bytes" >> $TMP_FILE
echo "# TYPE hetzner_storage_box_used_bytes gauge" >> $TMP_FILE
echo "hetzner_storage_box_used_bytes $STORAGE_BOX_USED" >> $TMP_FILE
echo "# HELP hetzner_storage_box_free_bytes storage box free space in bytes" >> $TMP_FILE
echo "# TYPE hetzner_storage_box_free_bytes gauge" >> $TMP_FILE
echo "hetzner_storage_box_free_bytes $STORAGE_BOX_FREE" >> $TMP_FILE
mv -f $TMP_FILE $PROM_FILE

@ -53,6 +53,7 @@
- arch-textcollector.sh
- borg-textcollector.sh
- borg-offsite-textcollector.sh
- hetzner-textcollector.sh
- rebuilderd-textcollector.sh
- rebuilderd-status-textcollector.py
- archive-textcollector.sh
@ -84,6 +85,18 @@
- { name: borg-offsite, service: borg-backup-offsite }
when: "'borg_clients' in group_names"
- name: install hetzner textcollector service
template: src=prometheus-hetzner-textcollector.service.j2 dest=/etc/systemd/system/prometheus-hetzner-textcollector.service owner=root group=root mode=644
when: "inventory_hostname == 'monitoring.archlinux.org'"
- name: install hetzner textcollector timer
template: src=prometheus-hetzner-textcollector.timer.j2 dest=/etc/systemd/system/prometheus-hetzner-textcollector.timer owner=root group=root mode=644
when: "inventory_hostname == 'monitoring.archlinux.org'"
- name: enable and start prometheus hetzner textcollector timer
systemd: name=prometheus-hetzner-textcollector.timer enabled=yes daemon_reload=yes state=started
when: "inventory_hostname == 'monitoring.archlinux.org'"
- name: install fail2ban textcollector service
template: src=prometheus-fail2ban-textcollector.service.j2 dest=/etc/systemd/system/prometheus-fail2ban-textcollector.service owner=root group=root mode=644

@ -0,0 +1,34 @@
[Unit]
Description=Prometheus Hetzner Exporter TextCollector
After=network-online.target
[Service]
Type=oneshot
ExecStart=/usr/local/bin/hetzner-textcollector.sh {{ prometheus_textfile_dir }}
NoNewPrivileges=true
LockPersonality=true
PrivateDevices=true
PrivateTmp=true
ProtectSystem=strict
ProtectHome=read-only
ReadWritePaths={{ prometheus_textfile_dir }}
MemoryDenyWriteExecute=true
RemoveIPC=true
RestrictRealtime=true
RestrictNamespaces=true
RestrictSUIDSGID=true
RestrictAddressFamilies=~AF_PACKET
RestrictAddressFamilies=~AF_NETLINK
ProtectHostname=true
ProtectControlGroups=true
ProtectKernelLogs=true
ProtectKernelTunables=true
ProtectKernelModules=true
ProtectClock=true
SystemCallArchitectures=native

@ -0,0 +1,10 @@
[Unit]
Description=Prometheus Hetzner Exporter TextCollector Timer
[Timer]
OnUnitActiveSec=1h
OnBootSec=15min
RandomizedDelaySec=1min
[Install]
WantedBy=timers.target