infrastructure/roles/prometheus/templates/prometheus.yml.j2

#jinja2: lstrip_blocks: True
{% if not prometheus_receive_only %}
global:
  scrape_interval: 60s

# Load rules once and periodically evaluate them according to the global 'evaluation_interval'.
rule_files:
  - node.rules.yml

# Alertmanager configuration
alerting:
  alertmanagers:
    - static_configs:
      - targets:
        - localhost:9093

remote_write:
  {% for relabel_config in prometheus_remote_write_relabel_configs %}
  - url: http://{{ prometheus_domain }}:9090/api/v1/write
    write_relabel_configs:
      - source_labels: [{{ relabel_config.label | join(',') }}]
        regex: "{{ relabel_config.regex }}"
        action: keep
    basic_auth:
      username: {{ vault_prometheus_user }}
      password: {{ vault_prometheus_passwd }}

  {% endfor %}
scrape_configs:
  - job_name: prometheus
    static_configs:
      - targets: ['127.0.0.1:9090']
        labels:
          instance: "{{ inventory_hostname }}"

  - job_name: prometheus-domain
    scheme: http
    basic_auth:
      username: {{ vault_prometheus_user }}
      password: {{ vault_prometheus_passwd }}
    static_configs:
      - targets: ['{{ prometheus_domain }}:9090']
        labels:
          job: prometheus
          instance: "dashboards.archlinux.org"

  - job_name: loki
    static_configs:
      - targets: ['127.0.0.1:3100']
        labels:
          instance: "{{ inventory_hostname }}"

  - job_name: tempo
    static_configs:
      - targets: ['127.0.0.1:3200']
        labels:
          instance: "{{ inventory_hostname }}"

  - job_name: 'node_exporter'
    static_configs:
      {% for host in groups['all'] %}
      - targets: ['{{ hostvars[host]['wireguard_address'] }}:{{ prometheus_exporter_port }}']
        labels:
          instance: "{{ host }}"
      {% endfor %}

  - job_name: 'promtail'
    static_configs:
      {% for host in groups['all'] %}
      - targets: ['{{ hostvars[host]['wireguard_address'] }}:9080']
        labels:
          instance: "{{ host }}"
      {% endfor %}

  - job_name: 'debuginfod'
    static_configs:
      - targets: ['{{ hostvars['debuginfod.archlinux.org']['wireguard_address'] }}:8002']
        labels:
          instance: "debuginfod.archlinux.org"

  - job_name: 'powerdns'
    static_configs:
      {% for host in groups['geo_mirrors'] + [geo_acme_dns_challenge_ns] %}
      - targets: ['{{ hostvars[host]['wireguard_address'] }}:8081']
        labels:
          instance: "{{ host }}"
      {% endfor %}

  - job_name: 'gitlab_runner_exporter'
    static_configs:
      {% for host in groups['gitlab_runners'] %}
      - targets: ['{{ hostvars[host]['wireguard_address'] }}:{{ gitlab_runner_exporter_port }}']
        labels:
          instance: "{{ host }}"
      {% endfor %}

  - job_name: 'keycloak'
    scheme: https
    metrics_path: "/realms/master/metrics"
    basic_auth:
       username: "{{ vault_keycloak_nginx_user }}"
       password: "{{ vault_keycloak_nginx_passwd }}"
    static_configs:
      - targets: ['accounts.archlinux.org:443']
        labels:
          instance: "accounts.archlinux.org"

  - job_name: 'gitlab_exporter'
    scheme: https
    metrics_path: "-/metrics"
    params:
      token: ["{{ vault_gitlab_prometheus_token }}"]
    static_configs:
      - targets: ['gitlab.archlinux.org:443']
        labels:
          instance: "gitlab.archlinux.org"

  - job_name: 'mysqld_exporter'
    static_configs:
      {% for host in groups['mysql_servers'] %}
      - targets: ['{{ hostvars[host]['wireguard_address'] }}:{{ prometheus_mysqld_exporter_port }}']
        labels:
          instance: "{{ host }}"
      {% endfor %}

  - job_name: 'memcached_exporter'
    static_configs:
      {% for host in groups['memcached'] %}
      - targets: ['{{ hostvars[host]['wireguard_address'] }}:{{ prometheus_memcached_exporter_port }}']
        labels:
          instance: "{{ host }}"
      {% endfor %}

  - job_name: hedgedoc
    scheme: https
    authorization:
      credentials: {{ vault_hedgedoc_metrics_token }}
    static_configs:
      - targets: ['md.archlinux.org:443']

  - job_name: aurweb
    scheme: https
    static_configs:
      - targets: ['aur.archlinux.org:443']

  - job_name: archweb
    scheme: https
    authorization:
      credentials: {{ vault_archweb_metrics_token }}
    static_configs:
      - targets: ['archlinux.org:443']

  - job_name: goaurrpc
    metrics_path: /rpc/metrics
    scheme: https
    authorization:
      credentials: {{ vault_goaurrpc_metrics_token }}
    static_configs:
      - targets: ['aur.archlinux.org:443']

  - job_name: grafana
    scheme: https
    authorization:
      credentials: {{ vault_grafana_metrics_token }}
    static_configs:
      - targets: ['monitoring.archlinux.org:443', 'dashboards.archlinux.org:443']
  {% for module in blackbox_targets.keys() %}

  - job_name: 'blackbox_{{ module }}'
    metrics_path: /probe
    scrape_interval: 60s
    params:
      module: [{{ module }}]
    static_configs:
      - targets:
        {% for target in blackbox_targets[module] | select('string') %}
        - {{ target }}
        {% endfor %}
      {% for vhost in blackbox_targets[module] | select('mapping') if module == 'http_prometheus' %}
      - targets:
          {% for target in vhost.targets %}
          - http{{ 's' if vhost.secure }}://{{ target }}
          {% endfor %}
        labels:
          vhost: {{ vhost.hostname }}
      {% endfor %}
    relabel_configs:
      - source_labels: [__address__]
        target_label: __param_target
      - source_labels: [__param_target]
        target_label: instance
      - target_label: __address__
        replacement: 127.0.0.1:9115
      {% if module == 'http_prometheus' %}
      - source_labels: [vhost]
        target_label: __param_hostname
      {% endif %}
  {% endfor %}
  {% for endpoint in matrix_metrics_endpoints %}

  - job_name: 'synapse_{{ endpoint }}'
    scrape_interval: 15s
    metrics_path: "/metrics/synapse.{{ endpoint }}"
    scheme: https
    authorization:
      credentials: {{ vault_matrix_metrics_token }}
    static_configs:
      - targets: ["matrix.archlinux.org:443"]
  {% endfor %}
{% endif %}