From c238ebda47e5566aaaf5ee8a866171c39b87c531 Mon Sep 17 00:00:00 2001
From: Jelle van der Waa <jelle@archlinux.org>
Date: Thu, 25 Feb 2021 22:07:17 +0100
Subject: [PATCH] Add prometheus textcollector for Arch repositories

Export the repository size of our repositories so they can be monitored
and we have some useful data for repository growth in the future.
---
 .../grafana/files/dashboards/repository.json  | 145 ++++++++++++++++++
 .../files/repository-textcollector.sh         |  33 ++++
 roles/prometheus_exporters/tasks/main.yml     |  13 ++
 ...etheus-repository-textcollector.service.j2 |  37 +++++
 ...ometheus-repository-textcollector.timer.j2 |  11 ++
 5 files changed, 239 insertions(+)
 create mode 100644 roles/grafana/files/dashboards/repository.json
 create mode 100644 roles/prometheus_exporters/files/repository-textcollector.sh
 create mode 100644 roles/prometheus_exporters/templates/prometheus-repository-textcollector.service.j2
 create mode 100644 roles/prometheus_exporters/templates/prometheus-repository-textcollector.timer.j2

diff --git a/roles/grafana/files/dashboards/repository.json b/roles/grafana/files/dashboards/repository.json
new file mode 100644
index 00000000..9066be17
--- /dev/null
+++ b/roles/grafana/files/dashboards/repository.json
@@ -0,0 +1,145 @@
+{
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": "-- Grafana --",
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "type": "dashboard"
+      }
+    ]
+  },
+  "description": "Arch Repository Sizes",
+  "editable": true,
+  "gnetId": null,
+  "graphTooltip": 0,
+  "id": 32,
+  "links": [],
+  "panels": [
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": "Prometheus",
+      "description": "The size of all our repositories",
+      "fieldConfig": {
+        "defaults": {
+          "custom": {},
+          "unit": "decbytes"
+        },
+        "overrides": []
+      },
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 14,
+        "w": 24,
+        "x": 0,
+        "y": 0
+      },
+      "hiddenSeries": false,
+      "id": 2,
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": true,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
+      "percentage": false,
+      "pluginVersion": "7.4.2",
+      "pointradius": 2,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "repository_directory_size_bytes",
+          "interval": "",
+          "legendFormat": "{{ name }}",
+          "queryType": "randomWalk",
+          "refId": "A"
+        },
+        {
+          "expr": "sum(repository_directory_size_bytes)",
+          "hide": false,
+          "interval": "",
+          "legendFormat": "total",
+          "refId": "B"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeRegions": [],
+      "timeShift": null,
+      "title": "Repository sizes",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "$$hashKey": "object:68",
+          "format": "decbytes",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        },
+        {
+          "$$hashKey": "object:69",
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    }
+  ],
+  "schemaVersion": 27,
+  "style": "dark",
+  "tags": [],
+  "templating": {
+    "list": []
+  },
+  "time": {
+    "from": "now-6M",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "",
+  "title": "Arch Repositories",
+  "uid": "hXmUEbsGk",
+  "version": 1
+}
\ No newline at end of file
diff --git a/roles/prometheus_exporters/files/repository-textcollector.sh b/roles/prometheus_exporters/files/repository-textcollector.sh
new file mode 100644
index 00000000..d6c8e83d
--- /dev/null
+++ b/roles/prometheus_exporters/files/repository-textcollector.sh
@@ -0,0 +1,33 @@
+#!/bin/bash
+
+set -o errexit
+set -o nounset
+
+if (( $# != 1 )); then
+  echo "Missing textcollector directory argument"
+  exit 1
+fi
+
+TEXTFILE_COLLECTOR_DIR=${1}
+REPOS_DIR=/srv/ftp
+PROM_FILE=$TEXTFILE_COLLECTOR_DIR/repository.prom
+
+TMP_FILE=$PROM_FILE.$$
+[ -e $TMP_FILE ] && rm -f $TMP_FILE
+
+trap "rm -f $TMP_FILE" EXIT
+
+echo "# HELP repository_directory_size_bytes repository directory size in bytes" >> $TMP_FILE
+echo "# TYPE repository_directory_size_bytes gauge" >> $TMP_FILE
+
+for dir in $REPOS_DIR/*; do
+  # All Arch repositories should have an os dir, this excludes the archive and other non repo dirs.
+  if [ -d "$dir/os" ]; then
+    reponame=$(basename $dir)
+    directory_size=$(du -Lsb ${dir} | awk '{ print $1 }')
+
+    echo "repository_directory_size_bytes{name=\"${reponame}\"} $directory_size" >> $TMP_FILE
+  fi
+done
+
+mv -f $TMP_FILE $PROM_FILE
diff --git a/roles/prometheus_exporters/tasks/main.yml b/roles/prometheus_exporters/tasks/main.yml
index 5ea6b6fc..2fba0be9 100644
--- a/roles/prometheus_exporters/tasks/main.yml
+++ b/roles/prometheus_exporters/tasks/main.yml
@@ -55,6 +55,7 @@
     - rebuilderd-textcollector.sh
     - rebuilderd-status-textcollector.py
     - archive-textcollector.sh
+    - repository-textcollector.sh
     - btrfs-textcollector.sh
 
 - name: install arch textcollector service
@@ -98,6 +99,10 @@
   template: src=prometheus-archive-textcollector.service.j2 dest=/etc/systemd/system/prometheus-archive-textcollector.service owner=root group=root mode=644
   when: "'archive_mirrors' in group_names or inventory_hostname == 'gemini.archlinux.org'"
 
+- name: install rebuilderd textcollector service
+  template: src=prometheus-repository-textcollector.service.j2 dest=/etc/systemd/system/prometheus-repository-textcollector.service owner=root group=root mode=644
+  when: "inventory_hostname == 'gemini.archlinux.org'"
+
 - name: install rebuilderd textcollector timer
   template: src=prometheus-archive-textcollector.timer.j2 dest=/etc/systemd/system/prometheus-archive-textcollector.timer owner=root group=root mode=644
   when: "'archive_mirrors' in group_names or inventory_hostname == 'gemini.archlinux.org'"
@@ -106,6 +111,14 @@
   systemd: name=prometheus-archive-textcollector.timer enabled=yes daemon_reload=yes state=started
   when: "'archive_mirrors' in group_names or inventory_hostname == 'gemini.archlinux.org'"
 
+- name: install rebuilderd textcollector timer
+  template: src=prometheus-repository-textcollector.timer.j2 dest=/etc/systemd/system/prometheus-repository-textcollector.timer owner=root group=root mode=644
+  when: "inventory_hostname == 'gemini.archlinux.org'"
+
+- name: enable and start prometheus repository textcollector timer
+  systemd: name=prometheus-repository-textcollector.timer enabled=yes daemon_reload=yes state=started
+  when: "inventory_hostname == 'gemini.archlinux.org'"
+
 - name: install sudoers for btrfs
   copy: src=sudoers dest=/etc/sudoers.d/node_exporter owner=root group=root mode=0440
   when: filesystem == "btrfs"
diff --git a/roles/prometheus_exporters/templates/prometheus-repository-textcollector.service.j2 b/roles/prometheus_exporters/templates/prometheus-repository-textcollector.service.j2
new file mode 100644
index 00000000..d596c269
--- /dev/null
+++ b/roles/prometheus_exporters/templates/prometheus-repository-textcollector.service.j2
@@ -0,0 +1,37 @@
+[Unit]
+Description=Prometheus Arch Repository Exporter
+After=network.target
+
+[Service]
+Type=oneshot
+User=node_exporter
+ExecStart=/usr/local/bin/repository-textcollector.sh {{ prometheus_textfile_dir }}
+
+NoNewPrivileges=true
+LockPersonality=true
+CapabilityBoundingSet=
+UMask=077
+
+PrivateDevices=true
+PrivateTmp=true
+ProtectSystem=strict
+ProtectHome=true
+ReadWritePaths={{ prometheus_textfile_dir }}
+
+MemoryDenyWriteExecute=true
+RemoveIPC=true
+RestrictRealtime=true
+RestrictNamespaces=true
+RestrictSUIDSGID=true
+
+RestrictAddressFamilies=~AF_NETLINK
+RestrictAddressFamilies=~AF_PACKET
+
+ProtectHostname=true
+ProtectControlGroups=true
+ProtectKernelLogs=true
+ProtectKernelTunables=true
+ProtectKernelModules=true
+ProtectClock=true
+
+SystemCallArchitectures=native
diff --git a/roles/prometheus_exporters/templates/prometheus-repository-textcollector.timer.j2 b/roles/prometheus_exporters/templates/prometheus-repository-textcollector.timer.j2
new file mode 100644
index 00000000..a5d2788f
--- /dev/null
+++ b/roles/prometheus_exporters/templates/prometheus-repository-textcollector.timer.j2
@@ -0,0 +1,11 @@
+[Unit]
+Description=Prometheus Arch Repository Exporter TextCollector Timer
+
+[Timer]
+# 24 hours
+OnUnitActiveSec=1440m
+OnBootSec=15min
+RandomizedDelaySec=1min
+
+[Install]
+WantedBy=timers.target