Combine sync{archive,debug,riscv} into mirrorsync

These roles are very similar and can be merged into a single new role.

Note: The archive mirror is changed from a 4-hour sync to minutely for
conformity with the other two mirrors. In practice this doesn't matter
as it was already taking over 4 hours to finish and was starting again
right after its previous run.

group_vars/geo_mirrors/misc.yml

@@ -1,3 +1,2 @@
 certbot_dns_support: true
 geo_mirror_domain: geo.mirror.pkgbuild.com
-riscv_mirror_domain: riscv.mirror.pkgbuild.com

group_vars/mirrors/mirrorsync.yml

@@ -0,0 +1,26 @@
+mirrorsync_mirrors:
+  archive:
+    hosts: "{{ groups['archive_mirrors'] }}"
+    source: rsync://rsync.archlinux.org/archive
+    target: /srv/archive
+    last_update_url: https://archive.archlinux.org/repos/last/lastupdate
+    last_update_dst: lastupdate
+    delay_updates: false
+  debug:
+    hosts: "{{ groups['geo_mirrors'] + ['debuginfod.archlinux.org'] }}"
+    source: rsync://rsync.archlinux.org/debug_packages
+    target: /srv/ftp
+    last_update_url: https://rsync.archlinux.org/lastupdate
+    last_update_dst: lastupdate
+    rsync_options:
+      - --include="*-debug/***"
+      - --include="pool/"
+      - --include="pool/*-debug/***"
+      - --exclude="*"
+  riscv:
+    hosts: "{{ groups['geo_mirrors'] }}"
+    mirror_domain: riscv.mirror.pkgbuild.com
+    source: rsync://archriscv.felixc.at/archriscv
+    target: /srv/riscv
+    last_update_url: https://archriscv.felixc.at/.status/lastupdate.txt
+    last_update_dst: .status/lastupdate.txt

host_vars/debuginfod.archlinux.org/mirrorsync.yml

@@ -0,0 +1 @@
+../../group_vars/mirrors/mirrorsync.yml

host_vars/mirror.pkgbuild.com/misc

@@ -1,5 +1,4 @@
 mirror_domain: mirror.pkgbuild.com
-mirror_debug_packages: false
 archweb_mirrorcheck_locations: [20, 21]
 filesystem: btrfs
 

playbooks/archive-mirrors.yml

@@ -10,7 +10,7 @@
     - { role: root_ssh }
     - { role: certbot }
     - { role: nginx }
-    - { role: syncarchive }
+    - { role: mirrorsync }
     - { role: archive_web }
     - { role: prometheus_exporters }
     - { role: promtail }

playbooks/debuginfod.archlinux.org.yml

@@ -11,6 +11,6 @@
     - { role: certbot }
     - { role: nginx }
     - { role: debuginfod }
-    - { role: syncdebug }
+    - { role: mirrorsync }
     - { role: prometheus_exporters }
     - { role: promtail }

playbooks/mirrors.yml

@@ -10,8 +10,7 @@
     - { role: certbot }
     - { role: nginx }
     - { role: syncrepo, tags: ['nginx'] }
-    - { role: syncdebug, when: mirror_debug_packages is not defined or mirror_debug_packages }
-    - { role: syncriscv, when: riscv_mirror_domain is defined }
+    - { role: mirrorsync }
     - { role: archweb, when: archweb_mirrorcheck_locations is defined, archweb_site: false, archweb_services: false, archweb_mirrorcheck: true }
     - { role: prometheus_exporters }
     - { role: promtail }

roles/mirrorsync/tasks/main.yml

@@ -0,0 +1,18 @@
+- name: Install rsync
+  pacman: name=rsync state=present
+
+- name: Set up synchronization
+  include_tasks: sync.yml
+  loop: "{{ mirrorsync_mirrors | dict2items }}"
+  loop_control:
+    label: "{{ item.key }}"
+  when: inventory_hostname in item.value.hosts
+
+- name: Set up nginx
+  include_tasks: web.yml
+  loop: "{{ mirrorsync_mirrors | dict2items }}"
+  loop_control:
+    label: "{{ item.key }}"
+  when:
+    - item.value.mirror_domain is defined
+    - inventory_hostname in item.value.hosts

roles/mirrorsync/tasks/sync.yml

@@ -0,0 +1,11 @@
+- name: Install sync script for {{ item.key }}
+  template: src=mirrorsync.j2 dest=/usr/local/bin/sync{{ item.key }} owner=root group=root mode=0755
+
+- name: Install systemd service for {{ item.key }}
+  template: src=mirrorsync.service.j2 dest=/etc/systemd/system/sync{{ item.key }}.service owner=root group=root mode=0644
+
+- name: Install systemd timer for {{ item.key }}
+  template: src=mirrorsync.timer.j2 dest=/etc/systemd/system/sync{{ item.key }}.timer owner=root group=root mode=0644
+
+- name: Start and enable timer for {{ item.key }}
+  systemd: name=sync{{ item.key }}.timer enabled=yes state=started daemon_reload=yes

roles/mirrorsync/tasks/web.yml

@@ -0,0 +1,14 @@
+- name: Create ssl cert for {{ item.value.mirror_domain }}
+  include_role:
+    name: certificate
+  vars:
+    domains: ["{{ item.value.mirror_domain }}"]
+    challenge: "DNS-01"
+
+- name: Configure nginx for {{ item.value.mirror_domain }}
+  template: src=nginx.d.conf.j2 dest=/etc/nginx/nginx.d/{{ item.key }}.conf owner=root group=root mode=0644
+  notify: Reload nginx
+  tags: ['nginx']
+
+- name: Make nginx log dir for {{ item.value.mirror_domain }}
+  file: path=/var/log/nginx/{{ item.value.mirror_domain }} state=directory owner=root group=root mode=0755

roles/mirrorsync/templates/mirrorsync.j2

@@ -0,0 +1,36 @@
+#!/bin/bash
+
+target="{{ item.value.target }}"
+lock="/run/lock/sync{{ item.key }}.lck"
+source_url='{{ item.value.source }}'
+lastupdate_url='{{ item.value.last_update_url }}'
+
+[ ! -d "${target}" ] && mkdir -p "${target}"
+
+exec 9>"${lock}"
+flock -n 9 || exit
+
+{% set delay_updates = '--delay-updates' if item.value.delay_updates | default(true) %}
+rsync_cmd() {
+	local -a cmd=(rsync -rlptH --safe-links --delete-delay {{ delay_updates }}
+		"--timeout=600" "--contimeout=60" --no-motd)
+
+	if stty &>/dev/null; then
+		cmd+=(-h -v --progress)
+	else
+		cmd+=("--info=name1")
+	fi
+
+	"${cmd[@]}" "$@"
+}
+
+# if we are called without a tty (cronjob) only run when there are changes
+if ! tty -s && [[ -f "$target/{{ item.value.last_update_dst }}" ]] && diff -b <(curl -Ls "$lastupdate_url") "$target/{{ item.value.last_update_dst }}" >/dev/null; then
+	exit 0
+fi
+
+rsync_cmd \
+{% for opt in item.value.rsync_options | default() %}
+	{{ opt }} \
+{% endfor %}
+	"${source_url}" "${target}"

roles/mirrorsync/templates/mirrorsync.service.j2

@@ -1,12 +1,12 @@
 [Unit]
-Description=Synchronize debug packages
-RequiresMountsFor=/srv/ftp
+Description=Synchronize {{ item.key }} mirror
+RequiresMountsFor={{ item.value.target }}
 Wants=network-online.target
 After=network-online.target
 
 [Service]
 Type=oneshot
-ExecStart=/usr/local/bin/syncdebug
+ExecStart=/usr/local/bin/sync{{ item.key }}
 Nice=19
 IOSchedulingClass=best-effort
 IOSchedulingPriority=7

roles/mirrorsync/templates/mirrorsync.timer.j2

@@ -1,5 +1,5 @@
 [Unit]
-Description=Minutely RISC-V mirror sync
+Description=Minutely {{ item.key }} mirror sync
 
 [Timer]
 OnCalendar=minutely

roles/mirrorsync/templates/nginx.d.conf.j2

@@ -0,0 +1,22 @@
+server {
+    listen       80;
+    listen       [::]:80;
+    listen       443 ssl http2;
+    listen       [::]:443 ssl http2;
+    server_name  {{ item.value.mirror_domain }};
+    root         /srv/riscv;
+
+    access_log   /var/log/nginx/{{ item.value.mirror_domain }}/access.log reduced;
+    access_log   /var/log/nginx/{{ item.value.mirror_domain }}/access.log.json json_reduced;
+    error_log    /var/log/nginx/{{ item.value.mirror_domain }}/error.log;
+
+    include snippets/letsencrypt.conf;
+
+    ssl_certificate      /etc/letsencrypt/live/{{ item.value.mirror_domain }}/fullchain.pem;
+    ssl_certificate_key  /etc/letsencrypt/live/{{ item.value.mirror_domain }}/privkey.pem;
+    ssl_trusted_certificate /etc/letsencrypt/live/{{ item.value.mirror_domain }}/chain.pem;
+
+    add_header X-Served-By "{{ inventory_hostname }}";
+
+    autoindex on;
+}

roles/syncarchive/files/syncarchive

@@ -1,34 +0,0 @@
-#!/bin/bash
-
-target="/srv/archive"
-lock="/var/lock/syncarchive.lck"
-source_url='rsync://rsync.archlinux.org/archive'
-lastupdate_url='https://archive.archlinux.org/repos/last/lastupdate'
-
-[ ! -d "${target}" ] && mkdir -p "${target}"
-
-exec 9>"${lock}"
-flock -n 9 || exit
-
-rsync_cmd() {
-	local -a cmd=(rsync -rlptH --safe-links --delete-delay
-		"--timeout=600" "--contimeout=60" --no-motd)
-
-	if stty &>/dev/null; then
-		cmd+=(-h -v --progress)
-	else
-		cmd+=("--info=name1")
-	fi
-
-	"${cmd[@]}" "$@"
-}
-
-# if we are called without a tty (cronjob) only run when there are changes
-if ! tty -s && [[ -f "$target/lastupdate" ]] && diff -b <(curl -Ls "$lastupdate_url") "$target/lastupdate" >/dev/null; then
-	exit 0
-fi
-
-rsync_cmd \
-    --exclude=".well-known" \
-	"${source_url}" \
-	"${target}"

roles/syncarchive/files/syncarchive.service

@@ -1,12 +0,0 @@
-[Unit]
-Description=Synchronize package archive mirror
-RequiresMountsFor=/srv/archive
-Wants=network-online.target
-After=network-online.target
-
-[Service]
-Type=oneshot
-ExecStart=/usr/local/bin/syncarchive
-Nice=19
-IOSchedulingClass=best-effort
-IOSchedulingPriority=7

roles/syncarchive/files/syncarchive.timer

@@ -1,10 +0,0 @@
-[Unit]
-Description=Sync archive every 4 hours
-
-[Timer]
-OnCalendar=00/4:00
-AccuracySec=1m
-Persistent=true
-
-[Install]
-WantedBy=timers.target

roles/syncarchive/tasks/main.yml

@@ -1,20 +0,0 @@
-- name: Install rsync
-  pacman: name=rsync state=present
-
-- name: Install syncarchive script
-  copy: src=syncarchive dest=/usr/local/bin/syncarchive owner=root group=root mode=0755
-
-- name: Install syncarchive units
-  copy: src={{ item }} dest=/etc/systemd/system/{{ item }} owner=root group=root mode=0644
-  with_items:
-    - syncarchive.timer
-    - syncarchive.service
-
-- name: Start and enable syncarchive units
-  systemd:
-    name: "{{ item }}"
-    enabled: true
-    state: started
-    daemon_reload: true
-  with_items:
-    - syncarchive.timer

roles/syncdebug/files/syncdebug

@@ -1,37 +0,0 @@
-#!/bin/bash
-
-target="/srv/ftp"
-lock="/var/lock/syncdebug.lck"
-source_url='rsync://rsync.archlinux.org/debug_packages'
-lastupdate_url='https://rsync.archlinux.org/lastupdate'
-
-[ ! -d "${target}" ] && mkdir -p "${target}"
-
-exec 9>"${lock}"
-flock -n 9 || exit
-
-rsync_cmd() {
-	local -a cmd=(rsync -rlptH --safe-links --delete-delay --delay-updates
-		"--timeout=600" "--contimeout=60" --no-motd)
-
-	if stty &>/dev/null; then
-		cmd+=(-h -v --progress)
-	else
-		cmd+=("--info=name1")
-	fi
-
-	"${cmd[@]}" "$@"
-}
-
-# if we are called without a tty (cronjob) only run when there are changes
-if ! tty -s && [[ -f "$target/lastupdate" ]] && diff -b <(curl -Ls "$lastupdate_url") "$target/lastupdate" >/dev/null; then
-	exit 0
-fi
-
-rsync_cmd \
-	--include="*-debug/***" \
-	--include="pool/" \
-	--include="pool/*-debug/***" \
-	--exclude="*" \
-	"${source_url}" \
-	"${target}"

roles/syncdebug/files/syncdebug.timer

@@ -1,10 +0,0 @@
-[Unit]
-Description=Sync debug packages every minute
-
-[Timer]
-OnCalendar=minutely
-AccuracySec=1m
-Persistent=true
-
-[Install]
-WantedBy=timers.target

roles/syncdebug/tasks/main.yml

@@ -1,20 +0,0 @@
-- name: Install rsync
-  pacman: name=rsync state=present
-
-- name: Install syncdebug script
-  copy: src=syncdebug dest=/usr/local/bin/syncdebug owner=root group=root mode=0755
-
-- name: Install syncdebug units
-  copy: src={{ item }} dest=/etc/systemd/system/{{ item }} owner=root group=root mode=0644
-  with_items:
-    - syncdebug.timer
-    - syncdebug.service
-
-- name: Start and enable syncdebug units
-  systemd:
-    name: "{{ item }}"
-    enabled: true
-    state: started
-    daemon_reload: true
-  with_items:
-    - syncdebug.timer

roles/syncriscv/files/syncriscv

@@ -1,31 +0,0 @@
-#!/bin/bash
-
-target="/srv/riscv"
-lock="/var/lock/syncriscv.lck"
-source_url='rsync://archriscv.felixc.at/archriscv'
-lastupdate_url='https://archriscv.felixc.at/.status/lastupdate.txt'
-
-[ ! -d "${target}" ] && mkdir -p "${target}"
-
-exec 9>"${lock}"
-flock -n 9 || exit
-
-rsync_cmd() {
-	local -a cmd=(rsync -rlptH --safe-links --delete-delay --delay-updates
-		"--timeout=600" "--contimeout=60" --no-motd)
-
-	if stty &>/dev/null; then
-		cmd+=(-h -v --progress)
-	else
-		cmd+=("--info=name1")
-	fi
-
-	"${cmd[@]}" "$@"
-}
-
-# if we are called without a tty (cronjob) only run when there are changes
-if ! tty -s && [[ -f "$target/.status/lastupdate.txt" ]] && diff -b <(curl -Ls "$lastupdate_url") "$target/.status/lastupdate.txt" >/dev/null; then
-	exit 0
-fi
-
-rsync_cmd "${source_url}" "${target}"

roles/syncriscv/files/syncriscv.service

@@ -1,12 +0,0 @@
-[Unit]
-Description=Synchronize RISC-V mirror
-RequiresMountsFor=/srv/riscv
-Wants=network-online.target
-After=network-online.target
-
-[Service]
-Type=oneshot
-ExecStart=/usr/local/bin/syncriscv
-Nice=19
-IOSchedulingClass=best-effort
-IOSchedulingPriority=7

roles/syncriscv/tasks/main.yml

@@ -1,29 +0,0 @@
-- name: Create ssl cert
-  include_role:
-    name: certificate
-  vars:
-    domains: ["{{ riscv_mirror_domain }}"]
-    challenge: "DNS-01"
-
-- name: Install rsync
-  pacman: name=rsync state=present
-
-- name: Install syncriscv script
-  copy: src=syncriscv dest=/usr/local/bin/syncriscv owner=root group=root mode=0755
-
-- name: Install syncriscv units
-  copy: src={{ item }} dest=/etc/systemd/system/{{ item }} owner=root group=root mode=0644
-  with_items:
-    - syncriscv.timer
-    - syncriscv.service
-
-- name: Start and enable syncriscv timer
-  systemd: name=syncriscv.timer enabled=yes state=started daemon_reload=yes
-
-- name: Set up nginx
-  template: src=nginx.d.conf.j2 dest=/etc/nginx/nginx.d/riscv.conf owner=root group=root mode=0644
-  notify: Reload nginx
-  tags: ['nginx']
-
-- name: Make nginx log dir
-  file: path=/var/log/nginx/{{ riscv_mirror_domain }} state=directory owner=root group=root mode=0755

roles/syncriscv/templates/nginx.d.conf.j2

@@ -1,22 +0,0 @@
-server {
-    listen       80;
-    listen       [::]:80;
-    listen       443 ssl http2;
-    listen       [::]:443 ssl http2;
-    server_name  {{ riscv_mirror_domain }};
-    root         /srv/riscv;
-
-    access_log   /var/log/nginx/{{ riscv_mirror_domain }}/access.log reduced;
-    access_log   /var/log/nginx/{{ riscv_mirror_domain }}/access.log.json json_reduced;
-    error_log    /var/log/nginx/{{ riscv_mirror_domain }}/error.log;
-
-    include snippets/letsencrypt.conf;
-
-    ssl_certificate      /etc/letsencrypt/live/{{ riscv_mirror_domain }}/fullchain.pem;
-    ssl_certificate_key  /etc/letsencrypt/live/{{ riscv_mirror_domain }}/privkey.pem;
-    ssl_trusted_certificate /etc/letsencrypt/live/{{ riscv_mirror_domain }}/chain.pem;
-
-    add_header X-Served-By "{{ inventory_hostname }}";
-
-    autoindex on;
-}