1
1
Fork 0
mirror of https://gitlab.archlinux.org/archlinux/infrastructure.git synced 2024-05-27 07:46:05 +02:00

Remove zabbix-agent, zabbix roles

Zabbix has been replaced by Prometheus for monitoring our services.
This commit is contained in:
Jelle van der Waa 2020-10-15 16:31:39 +02:00
parent 556a0522d2
commit 4bc660d6be
No known key found for this signature in database
GPG Key ID: C06086337C50773E
52 changed files with 15 additions and 1064 deletions

15
docs/grafana.md Normal file
View File

@ -0,0 +1,15 @@
# Grafana
Our Grafana is hosted on https://monitoring.archlinux.org and is accessible for
all Arch Linux Staff, editing rights are restricted to users with the Devops
Role.
Dashboards and datasources are automatically provisioned by Grafana with Grafana's build in [provisioning configuration](https://grafana.com/docs/grafana/latest/administration/provisioning/).
## Adding a new Dashboard
A new dashboard can be configured in our Grafana instance to try it out and if satisfactory checked in to Git as following:
* Export the dashboard to json (top left, share dashboard => exporter => save to file).
* Save the json file in `roles/grafana/files/dashboards'
* Git add the file and run the grafana playbook

View File

@ -4,9 +4,6 @@ configure_firewall: true
# module when deploying firewalld tasks
ansible_python_interpreter: /usr/bin/python3
zabbix_agent_templates:
- Template OS Linux
# this is used by the maintenance role to get the ip address
# of the machine running the playbook
maintenance_remote_machine: "{{ hostvars[inventory_hostname]['ansible_env'].SSH_CLIENT.split(' ')[0] }}"

View File

@ -4,8 +4,3 @@ archweb_db_host: 'apollo.archlinux.org'
# raise tcp window limits to 32MiB
tcp_rmem: "10240 87380 33554432"
tcp_wmem: "10240 87380 33554432"
zabbix_agent_templates:
- Template OS Linux
- Template App Syncrepo
- Template App Syncrepo Arch32

View File

@ -1,8 +1,5 @@
---
zabbix_agent_templates:
- Template OS Linux
repro_users:
holger:
name: "Holger Levsen"

View File

@ -1,11 +1,2 @@
---
filesystem: btrfs
zabbix_agent_templates:
- Template OS Linux
- Template App Borg Backup
- Template App HTTP Service
- Template App HTTPS Service
- Template App Nginx
- Template App SSH Service
- Template App PostgreSQL

View File

@ -14,24 +14,6 @@ system_disks:
kanboard_version: "v1.2.14"
# TODO use a list of enabled roles or groups to enable each template and also use the same to enable each role for a machine? duplicating and manually tracking this stuff sucks, but maybe we want to deploy roles without monitoring? maybe not?
zabbix_agent_templates:
- Template OS Linux
- Template App Borg Backup
- Template App HTTP Service
- Template App HTTPS Service
- Template App MySQL
- Template App Nginx
- Template App SMTP Service
- Template App SSH Service
- Template App Zabbix Server
- Template App PostgreSQL
- Template App Memcached
zabbix_agent_host_groups:
- Linux servers
- Zabbix servers
fail2ban_jails:
sshd: true
postfix: true

View File

@ -1,11 +1,3 @@
---
filesystem: btrfs
memcached_socket: "/var/run/aurweb.sock"
zabbix_agent_templates:
- Template OS Linux
- Template App Borg Backup
- Template App HTTP Service
- Template App HTTPS Service
- Template App MySQL
- Template App Nginx

View File

@ -1,11 +1,3 @@
---
filesystem: btrfs
memcached_socket: "/var/run/aurweb.sock"
zabbix_agent_templates:
- Template OS Linux
- Template App Borg Backup
- Template App HTTP Service
- Template App HTTPS Service
- Template App MySQL
- Template App Nginx

View File

@ -1,9 +1,2 @@
---
filesystem: btrfs
zabbix_agent_templates:
- Template OS Linux
- Template App Borg Backup
- Template App HTTP Service
- Template App HTTPS Service
- Template App MySQL
- Template App Nginx

View File

@ -1,9 +1,2 @@
---
filesystem: btrfs
zabbix_agent_templates:
- Template OS Linux
- Template App Borg Backup
- Template App HTTP Service
- Template App HTTPS Service
- Template App MySQL
- Template App Nginx

View File

@ -14,7 +14,3 @@ system_disks:
- /dev/nvme1n1
archbuild_fs: 'btrfs'
zabbix_agent_templates:
- Template OS Linux
- Template Btrfs

View File

@ -17,8 +17,3 @@ system_disks:
raid_level: "raid6"
archive_domain: archive.archlinux.org
zabbix_agent_templates:
- Template OS Linux
- Template App Borg Backup
- Template App Nginx

View File

@ -1,10 +1,3 @@
---
filesystem: btrfs
zabbix_agent_templates:
- Template OS Linux
- Template App Borg Backup
- Template App HTTP Service
- Template App HTTPS Service
- Template App Nginx
- Template App SSH Service
gitlab_backupdir: /srv/gitlab/data/backups

View File

@ -1,8 +1,2 @@
---
filesystem: btrfs
zabbix_agent_templates:
- Template OS Linux
- Template App Borg Backup
- Template App HTTP Service
- Template App HTTPS Service
- Template App Nginx

View File

@ -7,10 +7,3 @@ system_disks:
vault_mariadb_users:
root: "{{encrypted_mariadb_users_root_password}}"
zabbix_agent: "{{encrypted_mariadb_users_zabbix_password}}"
zabbix_agent_templates:
- Template OS Linux
- Template App Mailman
- Template App MySQL
- Template App Nginx

View File

@ -1,11 +1,2 @@
---
filesystem: btrfs
zabbix_agent_templates:
- Template OS Linux
- Template App Borg Backup
- Template App HTTP Service
- Template App HTTPS Service
- Template App Nginx
- Template App SSH Service
- Template App PostgreSQL

View File

@ -2,5 +2,3 @@
mirror_domain: mirror.pkgbuild.com
archweb_mirrorcheck_locations: [12, 13]
filesystem: btrfs
zabbix_agent_templates:
- Template OS Linux

View File

@ -1,8 +1,5 @@
---
filesystem: btrfs
zabbix_agent_templates:
- Template OS Linux
- Template App Borg Backup
arch_users:
demize:
name: "Johannes Löthberg"

View File

@ -1,7 +1,2 @@
---
filesystem: btrfs
zabbix_agent_templates:
- Template OS Linux
- Template App Borg Backup
- Template App PostgreSQL

View File

@ -1,8 +1,3 @@
---
filesystem: btrfs
zabbix_agent_templates:
- Template OS Linux
- Template App Borg Backup
- Template App HTTP Service
- Template App HTTPS Service

View File

@ -12,6 +12,3 @@ network_interface: "en*"
system_disks:
- /dev/nvme0n1
- /dev/nvme1n1
zabbix_agent_templates:
- Template OS Linux

View File

@ -1,4 +1,2 @@
---
filesystem: "btrfs"
zabbix_agent_templates:
- Template OS Linux

View File

@ -1,6 +1,2 @@
---
filesystem: btrfs
zabbix_agent_templates:
- Template OS Linux
- Template App Borg Backup
- Template App PostgreSQL

View File

@ -50,7 +50,6 @@
- { role: mailman, mailman_domain: "lists.archlinux.org" }
- { role: patchwork }
- { role: kanboard }
- { role: zabbix_server, tags: ["zabbix"] }
- { role: grafana }
- { role: archwiki }
- { role: conf_archlinux }

View File

@ -1,4 +1,3 @@
---
mariadb_skip_name_resolve: False
mariadb_skip_networking: False
@ -25,7 +24,5 @@ mariadb_innodb_stats_sample_pages: '32'
mariadb_innodb_thread_concurrency: '8'
mariadb_innodb_file_per_table: False
zabbix_agent_mysql_user: 'zabbix_agent'
mysql_backup_dir: '/root/backup-mysql'
mysql_backup_defaults: '/root/.backup-my.cnf'

View File

@ -38,10 +38,3 @@
- name: create client configuration for root
template: src=client.cnf.j2 dest=/root/.my.cnf owner=root group=root mode=0644
no_log: true
- name: configure zabbix-agent user
# FIXME: "zabbix_agent" is hardcoded in the password variable: {{ vault_mariadb_users.zabbix_agent }}
# NOTE: the zabbix-agent role uses {{zabbix_agent_mysql_password}} in the my.cnf.j2 template
mysql_user: user={{ zabbix_agent_mysql_user }} host=localhost password={{ vault_mariadb_users.zabbix_agent }}
# TODO: implement in ansible: grant process on *.* to 'zabbix_agent'@'localhost';

View File

@ -1,9 +0,0 @@
---
zabbix_agent_server: zabbix.archlinux.org
zabbix_agent_ip: ""
zabbix_agent_dns: "{{inventory_hostname}}"
zabbix_agent_useip: 0
# TODO set this as a default here once the value has been set correctly for each host. Otherwise we might remove templates from a host by accident
#zabbix_agent_template: ['Template OS Linux']
zabbix_agent_host_groups:
- Linux servers

View File

@ -1,55 +0,0 @@
#!/usr/bin/python
import argparse
import json
import os
from memcache import Client
SOCKET_DIR = '/run/memcached'
def stats(socket):
client = Client([f'unix://{SOCKET_DIR}/{socket}'])
stats = client.get_stats()
if not stats:
print(json.dumps({}))
else:
data = stats[0][1]
data['hit_rate'] = "0"
data['cache_rate'] = "0"
# Calculate hit rate
try:
data['hit_rate'] = str(round(float(data['get_hits']) / float(data['cmd_get']) * 100))
data['cache_rate'] = str(round(float(data['bytes']) / float(data['limit_maxbytes']) * 100))
except ZeroDivisionError:
pass
print(json.dumps(data))
def discover():
sockets = []
for loc in os.listdir(SOCKET_DIR):
if not loc.endswith('.sock'):
continue
sockets.append({"{#SOCKET}": loc})
print(json.dumps({"data": sockets}))
def main():
parser = argparse.ArgumentParser(description='memcached statistics dumptool (in json)')
parser.add_argument('--discover', action='store_true', help='discover memcached sockets')
parser.add_argument('--socket', type=str, help='the memcached socket location')
args = parser.parse_args()
if args.discover:
discover()
if args.socket:
stats(args.socket)
if __name__ == "__main__":
main()

View File

@ -1,32 +0,0 @@
#!/usr/bin/env perl
use strict;
use warnings;
use DBI;
use JSON;
my $config = $ARGV[0];
my $mode = $ARGV[1];
my $dbname = $ARGV[2];
if (!($mode eq "dbsize" or $mode eq "discover")) {
print STDERR "mode is not dbsize or discover\n";
die;
}
if ($mode eq "dbsize" and not defined $dbname) {
print STDERR "dbsize requires dbname to be defined\n";
die;
}
my $db = DBI->connect("DBI:mysql:mysql_read_default_file=$config");
if ($mode eq "dbsize") {
print encode_json($db->selectrow_hashref("select SUM(ROUND(((DATA_LENGTH + INDEX_LENGTH) / 1024 / 1024 ), 2)) AS dbsize from information_schema.tables where table_schema = ?", undef, $dbname));
} elsif ($mode eq "discover") {
print encode_json({
data => $db->selectall_arrayref("SELECT table_schema '{#TABLE_SCHEMA}' FROM information_schema.tables GROUP BY table_schema;", {Slice => {}})
});
} else {
die "unhandeled mode";
}

View File

@ -1,33 +0,0 @@
#!/usr/bin/env perl
use strict;
use warnings;
use DBI;
use JSON;
my $config = $ARGV[0];
my $mode = $ARGV[1];
my $user = $ARGV[2];
if (!($mode eq "userstat" or $mode eq "discover")) {
print STDERR "mode is not userstat or discover\n";
die;
}
if ($mode eq "userstat" and not defined $user) {
print STDERR "userstat requires user to be defined\n";
die;
}
my $db = DBI->connect("DBI:mysql:mysql_read_default_file=$config");
if ($mode eq "userstat") {
$db->{FetchHashKeyName} = "NAME_lc";
print encode_json($db->selectrow_hashref("select * from information_schema.user_statistics where user = ?", undef, $user));
} elsif ($mode eq "discover") {
print encode_json({
data => $db->selectall_arrayref("select user '{#USERNAME}' from information_schema.user_statistics", {Slice=>{}})
});
} else {
die "unhandeled mode";
}

View File

@ -1,159 +0,0 @@
#!/usr/bin/env perl
use strict;
use warnings;
use v5.16;
use autodie;
use Clone 'clone';
use IO::Handle;
use JSON;
use Statistics::Descriptive;
use Time::HiRes qw(time sleep);
my $interval = 30;
my $devmode = 0;
my @nginx_log_file_paths = glob("/var/log/nginx/*-access.log /var/log/nginx/*/access.log");
my $hostname_regex = qr/^.*(archlinux\.org|pkgbuild\.com)$/;
@nginx_log_file_paths = ("./test-access.log") if $devmode;
my $logfile;
#$SIG{PIPE} = 'IGNORE';
$SIG{__DIE__} = sub {
print STDERR "Should be dying now\n";
#close $logfile;
POSIX::_exit(2);
print STDERR "Called exit()\n";
};
sub trim {
my $str = shift;
$str =~ s/^\s+//;
$str =~ s/\s+$//;
return $str;
}
sub send_zabbix {
my ($key, $value) = @_;
state $zabbix_sender;
if (not defined $zabbix_sender) {
open $zabbix_sender, "|-", "zabbix_sender -c /etc/zabbix/zabbix_agentd.conf --real-time -i - >/dev/null" unless $devmode;
open $zabbix_sender, "|-", "cat" if $devmode;
$zabbix_sender->autoflush();
}
my $ret = syswrite $zabbix_sender, (sprintf "- %s %s\n", $key, $value);
if (not defined $ret) {
print STDERR "Writing failed: '$!' Restarting zabbix_sender\n";
undef $zabbix_sender;
send_zabbix(@_);
}
}
sub main {
die "No log files found" if @nginx_log_file_paths == 0;
open $logfile, "-|", qw(tail -n0 -q -F), @nginx_log_file_paths;
my $last_send_time = 0;
my $value_template = {
# counters since prog start
status => {
200 => 0,
404 => 0,
500 => 0,
502 => 0,
504 => 0,
other => 0,
},
# counter since prog start
request_count => 0,
cached_request_count => 0,
# calculated values since last send
request_time => {
max => 0,
average => 0,
median => 0,
}
};
my $values_per_host = {};
my $stat_per_host = {};
my $modified_hostlist = 0;
while (my $line = <$logfile>) {
#print "Got line: ".$line."\n";
$line = trim($line);
# json log format
if ($line =~ m/^{.*}$/) {
update_stats_for_line($values_per_host, $stat_per_host, $value_template, \$modified_hostlist, decode_json($line));
}
# main log format
if ($line =~ m/(?<remote_addr>\S+) (?<host>\S+) (?<remote_user>\S+) \[(?<time_local>.*?)\]\s+"(?<request>.*?)" (?<status>\S+) (?<body_bytes_sent>\S+) "(?<http_referer>.*?)" "(?<http_user_agent>.*?)" "(?<http_x_forwarded_for>\S+)"(?: (?<request_time>[\d\.]+|-))?/) {
update_stats_for_line($values_per_host, $stat_per_host, $value_template, \$modified_hostlist, \%+);
}
# reduced log format
if ($line =~ m/(?<host>\S+) \[(?<time_local>.*?)\]\s+"(?<request>.*?)" (?<status>\S+) (?<body_bytes_sent>\S+) "(?<http_referer>.*?)" "(?<http_user_agent>.*?)"/) {
update_stats_for_line($values_per_host, $stat_per_host, $value_template, \$modified_hostlist, \%+);
}
my $now = time;
if ($now >= $last_send_time + $interval) {
send_zabbix('nginx.discover', encode_json({data => [ map { { "{#VHOSTNAME}" => $_ } } keys %{$values_per_host} ]})) if $modified_hostlist;
$modified_hostlist = 0;
for my $host (keys %{$values_per_host}) {
my $stat = $stat_per_host->{$host};
my $values = $values_per_host->{$host};
$values->{request_time}->{max} = $stat->max() // 0.0;
$values->{request_time}->{average} = $stat->mean() // 0.0;
$values->{request_time}->{median} = $stat->median() // 0.0;
if ($stat->count() == 0) {
print STDERR "clearing stats for '$host'\n" if $devmode;
delete $values_per_host->{$host};
delete $stat_per_host->{$host};
$modified_hostlist = 1;
}
$stat->clear();
send_zabbix(sprintf('nginx.values[%s]', $host), encode_json($values));
}
$last_send_time = $now;
}
}
}
sub update_stats_for_line {
my ($values_per_host, $stat_per_host, $value_template, $modified_hostlist, $line_values) = @_;
my %val = %$line_values;
my $host = $val{host};
return unless $host =~ m/$hostname_regex/;
if (not defined $values_per_host->{$host}) {
$values_per_host->{$host} = clone($value_template);
$stat_per_host->{$host} = Statistics::Descriptive::Full->new();
$$modified_hostlist = 1;
}
my $stat = $stat_per_host->{$host};
my $values = $values_per_host->{$host};
$stat->add_data($val{request_time}) if defined($val{request_time}) and $val{request_time} != 0;
$values->{request_count}++;
$values->{cached_request_count}++ if defined($val{request_time}) and $val{request_time} == 0;
my $status_key = defined $values->{status}->{$val{status}} ? $val{status} : "other";
$values->{status}->{$status_key}++;
}
main();

View File

@ -1,10 +0,0 @@
[Unit]
Description=nginx zabbix monitoring
After=nginx.service
[Service]
ExecStart=/usr/local/bin/nginx-zabbix.pl
Restart=on-failure
[Install]
WantedBy=multi-user.target

View File

@ -1,31 +0,0 @@
#!/usr/bin/env perl
use strict;
use warnings;
use DBI;
use JSON;
my $mode = $ARGV[0];
my $dbname = $ARGV[1];
if (!($mode eq "dbsize" or $mode eq "discover")) {
print STDERR "mode is not dbsize or discover\n";
die;
}
if ($mode eq "dbsize" and not defined $dbname) {
print STDERR "dbsize requires dbname to be defined\n";
die;
}
my $db = DBI->connect("DBI:Pg:");
if ($mode eq "dbsize") {
print encode_json($db->selectrow_hashref("SELECT pg_database_size(?) AS dbsize;", undef, $dbname));
} elsif ($mode eq "discover") {
print encode_json({
data => $db->selectall_arrayref("SELECT datname \"{#DATNAME}\" FROM pg_database;", {Slice => {}})
});
} else {
die "unhandled mode";
}

View File

@ -1,31 +0,0 @@
#!/usr/bin/env perl
use strict;
use warnings;
use DBI;
use JSON;
my $mode = $ARGV[0];
my $dbname = $ARGV[1];
if (!($mode eq "userstat" or $mode eq "discover")) {
print STDERR "mode is not userstat or discover\n";
die;
}
if ($mode eq "userstat" and not defined $dbname) {
print STDERR "userstat requires dbname to be defined\n";
die;
}
my $db = DBI->connect("DBI:Pg:");
if ($mode eq "userstat") {
print encode_json($db->selectrow_hashref("SELECT tup_fetched,tup_returned,tup_inserted,tup_deleted,tup_updated,deadlocks,blks_hit,blks_read from pg_stat_database where datname = ?;", undef, $dbname));
} elsif ($mode eq "discover") {
print encode_json({
data => $db->selectall_arrayref("SELECT datname \"{#DATNAME}\" FROM pg_database;", {Slice => {}})
});
} else {
die "unhandled mode";
}

View File

@ -1,74 +0,0 @@
#!/usr/bin/python
import dbus
import json
import re
# regex are ancored to beginning and end of string already
unit_whitelist_regexes = [
r'archive-uploader.service',
r'borg-backup.service',
r'createlinks.service',
r'dovecot.service',
r'fcgiwrap-.*.service',
r'grafana.service',
r'mailman.service',
r'mariadb.service',
r'matrix-appservice-irc.service',
r'mysqld.service',
r'nginx.service',
r'nginx-zabbix.service',
r'opendkim.service',
r'php-fpm.service',
r'php-fpm@.*.service',
r'postfix.service',
r'postfwd.service',
r'postgresql.service',
r'quassel.service',
r'security-tracker-update.service',
r'syslog-ng@.*.service',
r'rspamd.service',
r'sshd.service',
r'svnserve.service',
r'synapse.service',
r'system-rsyncd.slice',
r'unbound.service',
r'uwsgi@.*.service',
r'zabbix-agent.service',
r'zabbix-server-pgsql.service',
r'fail2ban.service',
]
bus = dbus.SystemBus()
systemd1 = bus.get_object('org.freedesktop.systemd1', '/org/freedesktop/systemd1')
systemd1_manager = dbus.Interface(systemd1, dbus_interface='org.freedesktop.systemd1.Manager')
units = systemd1_manager.ListUnits()
discovered = []
def get_unit_prop_interface(obj, interface, propname):
try:
return obj.Get(interface, propname)
except dbus.exceptions.DBusException as err:
if err.get_dbus_name() == 'org.freedesktop.DBus.Error.UnknownProperty':
return None
else:
raise
def get_unit_prop(obj, propname):
interfaces = ['org.freedesktop.systemd1.Service', 'org.freedesktop.systemd1.Slice']
for interface in interfaces:
ret = get_unit_prop_interface(obj, interface, propname)
if ret:
return ret
return None
combined = '|'.join(unit_whitelist_regexes)
unit_whitelist_compiled = re.compile('^('+combined+')$')
for u in units:
if unit_whitelist_compiled.match(u[0]):
discovered.append({"{#UNITNAME}": u[0].replace("@", "--AT--")})
print(json.dumps({"data": discovered}));

View File

@ -1,22 +0,0 @@
#!/usr/bin/python
import dbus
ignore = set(["syncrepo.service", "syncrepo_arch32.service"])
bus = dbus.SystemBus()
systemd1 = bus.get_object("org.freedesktop.systemd1", "/org/freedesktop/systemd1")
systemd1_manager = dbus.Interface(
systemd1, dbus_interface="org.freedesktop.systemd1.Manager"
)
units = systemd1_manager.ListUnits()
for unit in filter(
lambda u: u[3] == "failed"
and u[0] not in ignore
and not u[0].startswith("user@")
and not u[0].startswith("user-runtime-dir@"),
units,
):
print(unit[0])

View File

@ -1,31 +0,0 @@
#!/usr/bin/env perl
use strict;
use warnings;
use autodie;
use JSON::XS;
use POSIX;
my $json = decode_json(scalar(`sudo /usr/local/bin/borg list --json --format '{start}{end}' --sort-by timestamp --last 1`));
print encode_json({
last_archive => {
is_checkpoint => $json->{archives}[0]->{archive} =~ m/.checkpoint$/ ? 1 : 0,
start => parse_time($json->{archives}[0]->{start}),
end => parse_time($json->{archives}[0]->{end}),
age => time() - parse_time($json->{archives}[0]->{end}),
},
repository => {
last_modified => parse_time($json->{repository}->{last_modified}),
},
});
sub parse_time {
my ($string) = @_;
if ($string =~ m/^(?<year>....)-(?<month>..)-(?<day>..)T(?<hour>..):(?<minute>..):(?<second>..)(?:\.\d+)?$/) {
my $time = POSIX::mktime($+{second},$+{minute},$+{hour},$+{day},$+{month}-1,$+{year}-1900);
return $time;
}
# return undef to ensure that the value is set in the json
return undef;
}

View File

@ -1,2 +0,0 @@
#!/bin/bash
btrfs subvol list -d / | wc -l

View File

@ -1,7 +0,0 @@
---
- name: restart zabbix agent
service: name=zabbix-agent state=restarted
- name: restart nginx-zabbix.service
service: name=nginx-zabbix.service state=restarted

View File

@ -1,128 +0,0 @@
---
# If you do not have python-zabbix-api installed (** ON YOUR LOCAL MACHINE **),
# this will fail. Please install it; it is needed for tasks later in this role.
- name: install python-zabbix-api package on localhost
pacman: name=python-zabbix-api
run_once: true
delegate_to: localhost
- name: install packages
pacman: name=zabbix-agent,python-dbus,perl-json,arch-audit
- name: install packages for borg monitoring
pacman: name=perl-json
when: "'borg_clients' in group_names"
- name: install packages for nginx monitoring
pacman: name=perl-json,perl-clone,perl-statistics-descriptive
when: "'nginx' in group_names"
- name: install mysql userstats monitoring packages
pacman: name=perl-dbi,perl-dbd-mysql,perl-json
when: "'mysql_servers' in group_names"
- name: install postgresql userstats monitoring packages
pacman: name=perl-dbi,perl-dbd-pg,perl-json
when: "'postgresql_servers' in group_names"
- name: install memcached monitoring packages
pacman: name=python-memcached
when: "'memcached' in group_names"
- name: adding existing zabbix-agent to memcached group
user:
name: zabbix-agent
groups: memcached
append: yes
when: "'memcached' in group_names"
- name: set zabbix directory permissions in /etc
file: path=/etc/zabbix state=directory owner=root group=root mode=755
- name: create host PSK
shell: umask 077; openssl rand -hex 64 > /etc/zabbix/zabbix_agentd.psk creates=/etc/zabbix/zabbix_agentd.psk
- name: fix permissions of PSK file
file: path=/etc/zabbix/zabbix_agentd.psk owner=zabbix-agent group=zabbix-agent mode=600
- name: fetch PSK
command: cat /etc/zabbix/zabbix_agentd.psk
check_mode: no
register: zabbix_agent_psk
changed_when: false
- name: Set host config in zabbix
zabbix_host:
server_url: "https://{{ zabbix_agent_server }}"
login_user: "{{ vault_zabbix_admin_user }}"
login_password: "{{ vault_zabbix_admin_password }}"
host_name: "{{ inventory_hostname }}"
host_groups: "{{ zabbix_agent_host_groups }}"
visible_name: "{{ inventory_hostname }}"
link_templates: "{{ zabbix_agent_templates }}"
status: enabled
state: present
inventory_mode: disabled
interfaces:
- type: 1
main: 1
useip: "{{ zabbix_agent_useip }}"
ip: "{{ zabbix_agent_ip }}"
dns: "{{ inventory_hostname }}"
port: 10050
tls_psk_identity: "PSK{{ inventory_hostname }}"
tls_accept: 2
tls_connect: 2
tls_psk: "{{ zabbix_agent_psk.stdout }}"
delegate_to: localhost
- name: install agent config
template: src=zabbix_agentd.conf dest=/etc/zabbix/zabbix_agentd.conf owner=zabbix-agent group=zabbix-agent mode=600
notify:
- restart zabbix agent
- name: Create mysql config for agent checks
template: src=my.cnf.j2 dest=/etc/zabbix/zabbix_agentd.my.cnf owner=zabbix-agent group=zabbix-agent mode=600
when: zabbix_agent_mysql_user is defined
- name: Install helper scripts
copy: src={{ item }} dest=/usr/local/bin/{{ item }} mode=0755 owner=root group=root
with_items:
- systemd-units-failed.py
- mysql-user-stats.pl
- systemd-discover-accounting-units.py
- zabbix-borg-timestamps.pl
- mysql-dbsize-stats.pl
- postgresql-dbsize-stats.pl
- postgresql-user-stats.pl
- memcached-stats.py
- zabbix-btrfs-subvolume.sh
- name: Install helper scripts
copy: src=nginx-zabbix.pl dest=/usr/local/bin/nginx-zabbix.pl mode=0755 owner=root group=root
notify:
- restart nginx-zabbix.service
when: "'nginx' in group_names"
- name: install sudo config
template: src=zabbix-agent-sudoers.conf.j2 dest=/etc/sudoers.d/zabbix-agent-sudoers mode=0440 owner=root group=root
- name: copy nginx-zabbix.service
copy: src=nginx-zabbix.service dest=/etc/systemd/system/nginx-zabbix.service owner=root group=root mode=0644
notify:
- restart nginx-zabbix.service
when: "'nginx' in group_names"
- name: start nginx-zabbix.service
service: name=nginx-zabbix.service enabled=yes state=started
when: "'nginx' in group_names"
- name: run zabbix agent service
service: name=zabbix-agent enabled=yes state=started
- name: open firewall holes
ansible.posix.firewalld: service=zabbix-agent permanent=true state=enabled immediate=yes
when: configure_firewall
tags:
- firewall

View File

@ -1,3 +0,0 @@
[client]
user={{zabbix_agent_mysql_user}}
password={{zabbix_agent_mysql_password}}

View File

@ -1,2 +0,0 @@
zabbix-agent ALL=(root) NOPASSWD: /usr/local/bin/borg list --json --format {start}{end} --sort-by timestamp --last 1, /usr/local/bin/mysql-dbsize-stats.pl, /usr/local/bin/memcached-stats.py, /usr/local/bin/zabbix-btrfs-subvolume.sh
zabbix-agent ALL=(postgres) NOPASSWD: /usr/local/bin/postgresql-dbsize-stats.pl, /usr/local/bin/postgresql-user-stats.pl

View File

@ -1,75 +0,0 @@
LogType=system
{% if inventory_hostname == "apollo.archlinux.org" %}
Server={{zabbix_agent_server}},fe80::921b:eff:fec4:41fe
{% else %}
Server={{zabbix_agent_server}}
{% endif %}
ServerActive={{zabbix_agent_server}}
Hostname={{inventory_hostname|lower}}
#Include=/etc/zabbix/zabbix_agentd.conf.d/*.conf
TLSConnect=psk
TLSAccept=psk
TLSPSKIdentity=PSK{{inventory_hostname|lower}}
TLSPSKFile=/etc/zabbix/zabbix_agentd.psk
UserParameter=mysql.ping,mysqladmin --defaults-file="/etc/zabbix/zabbix_agentd.my.cnf" ping|grep alive|wc -l
UserParameter=mysql.version,mysql --defaults-file="/etc/zabbix/zabbix_agentd.my.cnf" -V
UserParameter=mysql.status[*],echo "show global status where Variable_name='$1';" | mysql --defaults-file="/etc/zabbix/zabbix_agentd.my.cnf" -N | awk '{print $$2}'
UserParameter=mysql.userstats[*],/usr/local/bin/mysql-user-stats.pl "/etc/zabbix/zabbix_agentd.my.cnf" userstat '$1'
UserParameter=mysql.userstats_discover,/usr/local/bin/mysql-user-stats.pl "/etc/zabbix/zabbix_agentd.my.cnf" discover
UserParameter=mysql.dbsize[*],sudo /usr/local/bin/mysql-dbsize-stats.pl "/root/.my.cnf" dbsize '$1'
UserParameter=mysql.dbsize_discover,sudo /usr/local/bin/mysql-dbsize-stats.pl "/root/.my.cnf" discover
UserParameter=postgresql.dbsize[*],sudo -u postgres /usr/local/bin/postgresql-dbsize-stats.pl dbsize '$1'
UserParameter=postgresql.dbsize_discover,sudo -u postgres /usr/local/bin/postgresql-dbsize-stats.pl discover
UserParameter=postgresql.userstats[*],sudo -u postgres /usr/local/bin/postgresql-user-stats.pl userstat '$1'
UserParameter=postgresql.userstats_discover,sudo -u postgres /usr/local/bin/postgresql-user-stats.pl discover
UserParameter=systemd.unit.is-active[*],systemctl is-active --quiet '$1' && echo 1 || echo 0
UserParameter=systemd.unit.is-failed[*],systemctl is-failed --quiet '$1' && echo 1 || echo 0
UserParameter=systemd.unit.is-enabled[*],systemctl is-enabled --quiet '$1' 2>/dev/null && echo 1 || echo 0
UserParameter=systemd.failed_units,/usr/local/bin/systemd-units-failed.py
UserParameter=systemd.accounting_discover,/usr/local/bin/systemd-discover-accounting-units.py
UserParameter=systemd.unit_data[*],systemctl show -- "$(echo '$1' | sed 's#--AT--#@#g')" | perl -MJSON -ne 'BEGIN{my %h}; my ($$k,$$v) = split(/=/,$$_,2); chomp($$h{$$k} = ($v =~ s/\[not set\]/0/r)); END {print encode_json(\%h)};'
UserParameter=borg_backup.data,/usr/local/bin/zabbix-borg-timestamps.pl
UserParameter=postfix.pfmailq,mailq | grep -v "Mail queue is empty" | grep -c '^[0-9A-Z]'
UserParameter=postfix[*],/usr/local/bin/postfix-zabbix-stats.sh $1
{% raw %}
UserParameter=custom.vfs.discover_disks, awk 'BEGIN {print "{"; print "\"data\":["; ORS=""} { if($3 !~ "^(sd[a-z]+[0-9]+|dm-.*|md[0-9]+|loop[0-9]+|sr[0-9]+)$" ) { { if (NR!=1) {print ",\n"}; print " { \"{#DEVICENAME}\":\"" $3 "\" }"} } } END {print "\n ]\n}\n"}' /proc/diskstats
{% endraw %}
# reads completed successfully
UserParameter=custom.vfs.dev.read.ops[*],awk -v disk="$1" '{ if ($$3 == disk) print $$4; }' /proc/diskstats
# sectors read
UserParameter=custom.vfs.dev.read.sectors[*],awk -v disk="$1" '{ if ($$3 == disk) print $$6; }' /proc/diskstats
# time spent reading (ms)
UserParameter=custom.vfs.dev.read.ms[*],awk -v disk="$1" '{ if ($$3 == disk) print $$7; }' /proc/diskstats
# writes completed
UserParameter=custom.vfs.dev.write.ops[*],awk -v disk="$1" '{ if ($$3 == disk) print $$8; }' /proc/diskstats
# sectors written
UserParameter=custom.vfs.dev.write.sectors[*],awk -v disk="$1" '{ if ($$3 == disk) print $$10; }' /proc/diskstats
# time spent writing (ms)
UserParameter=custom.vfs.dev.write.ms[*],awk -v disk="$1" '{ if ($$3 == disk) print $$11; }' /proc/diskstats
# I/Os currently in progress
UserParameter=custom.vfs.dev.io.active[*],awk -v disk="$1" '{ if ($$3 == disk) print $$12; }' /proc/diskstats
# time spent doing I/Os (ms)
UserParameter=custom.vfs.dev.io.ms[*],awk -v disk="$1" '{ if ($$3 == disk) print $$13; }' /proc/diskstats
UserParameter=linux.pressure,perl -MJSON -E 'my %h; for my $filename (@ARGV) {open my $f, "<", $filename; while (my $line = <$f>) {if ($line =~ m/^(\S+).*total=(\d+)$/) {my ($type, $value) = ($1, $2); $h{$filename =~ s#/proc/pressure/##r}->{$type}->{total} = $value;} } } print encode_json(\%h);' /proc/pressure/cpu /proc/pressure/io /proc/pressure/memory
# Upgradable vulnerable packages, omitting testing packages
UserParameter=arch_audit.data,arch-audit -u | grep -v "from testing repos"
UserParameter=arch_archive.db_count,echo 'select count(*) from files;' | sqlite3 /home/archive/archive-uploader.sqlite
UserParameter=memcached.stats[*],sudo /usr/local/bin/memcached-stats.py --socket '$1'
UserParameter=memcached.stats_discover,/usr/local/bin/memcached-stats.py --discover
#UserParameter=postgresql.dbsize_discover,sudo -u postgres /usr/local/bin/postgresql-dbsize-stats.pl discover
UserParameter=btrfs.subvolume_count,sudo /usr/local/bin/zabbix-btrfs-subvolume.sh

View File

@ -1,8 +0,0 @@
---
zabbix_db_name: zabbix-server
zabbix_db_user: zabbix-server
zabbix_domain: zabbix.archlinux.org
zabbix_web_dir: /usr/share/webapps/zabbix
zabbix_pgpass: /var/lib/zabbix-server/.pgpass

View File

@ -1,7 +0,0 @@
---
- name: restart zabbix server
service: name=zabbix-server-pgsql state=restarted
- name: restart php-fpm@zabbix-web
service: name=php-fpm@zabbix-web state=restarted

View File

@ -1,81 +0,0 @@
---
- name: install packages
pacman: name=zabbix-server,zabbix-frontend-php
- name: set zabbix directory permissions in /etc
file: path=/etc/zabbix state=directory owner=root group=root mode=755
- name: install server config
template: src=zabbix_server.conf dest=/etc/zabbix/zabbix_server.conf owner=zabbix-server group=zabbix-server mode=600
notify:
- restart zabbix server
- name: install nginx config
template: src=nginx.d.conf.j2 dest=/etc/nginx/nginx.d/zabbix.conf owner=root group=root mode=644
notify:
- reload nginx
tags: ['nginx']
- name: make nginx log dir
file: path=/var/log/nginx/{{ zabbix_domain }} state=directory owner=root group=root mode=0755
- name: create zabbix db user
postgresql_user: name={{ zabbix_db_user }} password={{ vault_zabbix_db_password }} encrypted=true
become: yes
become_user: postgres
become_method: su
- name: create zabbix db
postgresql_db: db="{{ zabbix_db_name }}" owner={{ zabbix_db_user }}
become: yes
become_user: postgres
become_method: su
- name: create zabbix .pgpass
template: src=pgpass.j2 dest="{{ zabbix_pgpass }}" owner=zabbix-server group=zabbix-server mode=600
no_log: true
- name: check if zabbix db contains tables
shell: set -o pipefail && psql -U {{ zabbix_db_user }} {{ zabbix_db_name }} -c '\d' | grep -P '\(\d+ rows\)'
become_user: zabbix-server
become: yes
register: table_is_populated
failed_when: ( table_is_populated.rc not in [ 0, 1 ] )
changed_when: table_is_populated.rc != 0
- name: install database schema
shell: psql -U "{{ zabbix_db_user }}" "{{ zabbix_db_name }}" < "/usr/share/zabbix-server/postgresql/{{ item }}"
become_user: zabbix-server
become: yes
when: table_is_populated.rc != 0
with_items:
- schema.sql
- images.sql
- data.sql
- name: make zabbix web user
user: name=zabbix-web shell=/bin/false home="{{ zabbix_web_dir }}" createhome=no
- name: configure php-fpm
template:
src=php-fpm.conf.j2 dest="/etc/php/php-fpm.d/zabbix-web.conf"
owner=root group=root mode=0644
notify:
- restart php-fpm@zabbix-web
- name: install zabbix web config
template: src=zabbix.conf.php.j2 dest=/usr/share/webapps/zabbix/conf/zabbix.conf.php owner=zabbix-web group=zabbix-web mode=600
- name: run zabbix server service
service: name=zabbix-server-pgsql enabled=yes state=started
- name: start and enable systemd socket
service: name=php-fpm@zabbix-web.socket state=started enabled=true
- name: open firewall holes
ansible.posix.firewalld: service=zabbix-server permanent=true state=enabled immediate=yes
when: configure_firewall
tags:
- firewall

View File

@ -1,20 +0,0 @@
{
"tcp": {
"listen": "localhost:12345"
},
"irc": {
"server": "chat.freenode.net:6697",
"tls": true,
"tls_skip_verify": false,
"nick": "archzabbix",
"realname": "Arch Zabbix Notifier",
"server_pass": "",
"identify_pass": "{{ vault_zabbix_irc_password }}",
"sasl_login": "archzabbix",
"sasl_pass": "{{ vault_zabbix_irc_password }}",
"channels": ["#archlinux-devops"]
},
"commands": {
}
}

View File

@ -1,50 +0,0 @@
upstream zabbix {
server unix:///run/php-fpm/zabbix-web.socket;
}
server {
listen 80;
listen [::]:80;
server_name {{ zabbix_domain }};
access_log /var/log/nginx/{{ zabbix_domain }}/access.log reduced;
error_log /var/log/nginx/{{ zabbix_domain }}/error.log;
include snippets/letsencrypt.conf;
location / {
access_log off;
return 301 https://$server_name$request_uri;
}
}
server {
listen 443 ssl http2;
listen [::]:443 ssl http2;
server_name {{ zabbix_domain }};
access_log /var/log/nginx/{{ zabbix_domain }}/access.log reduced;
error_log /var/log/nginx/{{ zabbix_domain }}/error.log;
ssl_certificate /etc/letsencrypt/live/{{ zabbix_domain }}/fullchain.pem;
ssl_certificate_key /etc/letsencrypt/live/{{ zabbix_domain }}/privkey.pem;
ssl_trusted_certificate /etc/letsencrypt/live/{{ zabbix_domain }}/chain.pem;
root {{zabbix_web_dir}};
index index.php;
location / {
try_files $uri $uri/ /index.php$is_args$args;
}
location ~ \.php$ {
access_log /var/log/nginx/{{ zabbix_domain }}/access.log main;
try_files $uri =404;
fastcgi_split_path_info ^(.+\.php)(/.+)$;
fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name;
include fastcgi_params;
fastcgi_pass zabbix;
}
}

View File

@ -1,2 +0,0 @@
# hostname:port:database:username:password
localhost:*:{{zabbix_db_name}}:{{zabbix_db_user}}:{{vault_zabbix_db_password}}

View File

@ -1,24 +0,0 @@
[global]
error_log = syslog
daemonize = no
[zabbix-web]
listen = /run/php-fpm/zabbix-web.socket
listen.owner = zabbix-web
listen.group = http
listen.mode = 0660
pm = dynamic
pm.max_children = 20
pm.start_servers = 1
pm.min_spare_servers = 1
pm.max_spare_servers = 3
pm.max_requests = 2000
php_admin_value[open_basedir] = {{zabbix_web_dir}}:/tmp
php_admin_value[opcache.memory_consumption] = 128
php_admin_value[opcache.interned_strings_buffer] = 8
php_admin_value[opcache.max_accelerated_files] = 200
php_admin_value[opcache.revalidate_freq] = 60
php_admin_value[opcache.fast_shutdown] = 1
php_admin_value[disable_functions] = passthru, exec, proc_open, shell_exec, system, popen

View File

@ -1,19 +0,0 @@
<?php
// Zabbix GUI configuration file.
global $DB;
$DB['TYPE'] = 'POSTGRESQL';
$DB['SERVER'] = 'localhost';
$DB['PORT'] = '0';
$DB['DATABASE'] = '{{zabbix_db_name}}';
$DB['USER'] = '{{zabbix_db_user}}';
$DB['PASSWORD'] = '{{vault_zabbix_db_password}}';
// Schema name. Used for IBM DB2 and PostgreSQL.
$DB['SCHEMA'] = '';
$ZBX_SERVER = '{{zabbix_domain}}';
$ZBX_SERVER_PORT = '10051';
$ZBX_SERVER_NAME = 'Arch Linux Zabbix';
$IMAGE_FORMAT_DEFAULT = IMAGE_FORMAT_PNG;

View File

@ -1,8 +0,0 @@
LogType=system
DBName={{zabbix_db_name}}
DBUser={{zabbix_db_user}}
DBPassword={{vault_zabbix_db_password}}
Timeout=4
LogSlowQueries=3000
StartPollers=30
AlertScriptsPath=/usr/local/bin