diff --git a/group_vars/geo_mirrors/misc.yml b/group_vars/geo_mirrors/misc.yml index ddfaa2e0..32dec742 100644 --- a/group_vars/geo_mirrors/misc.yml +++ b/group_vars/geo_mirrors/misc.yml @@ -1,3 +1,2 @@ certbot_dns_support: true geo_mirror_domain: geo.mirror.pkgbuild.com -riscv_mirror_domain: riscv.mirror.pkgbuild.com diff --git a/group_vars/mirrors/mirrorsync.yml b/group_vars/mirrors/mirrorsync.yml new file mode 100644 index 00000000..b6083439 --- /dev/null +++ b/group_vars/mirrors/mirrorsync.yml @@ -0,0 +1,26 @@ +mirrorsync_mirrors: + archive: + hosts: "{{ groups['archive_mirrors'] }}" + source: rsync://rsync.archlinux.org/archive + target: /srv/archive + last_update_url: https://archive.archlinux.org/repos/last/lastupdate + last_update_dst: lastupdate + delay_updates: false + debug: + hosts: "{{ groups['geo_mirrors'] + ['debuginfod.archlinux.org'] }}" + source: rsync://rsync.archlinux.org/debug_packages + target: /srv/ftp + last_update_url: https://rsync.archlinux.org/lastupdate + last_update_dst: lastupdate + rsync_options: + - --include="*-debug/***" + - --include="pool/" + - --include="pool/*-debug/***" + - --exclude="*" + riscv: + hosts: "{{ groups['geo_mirrors'] }}" + mirror_domain: riscv.mirror.pkgbuild.com + source: rsync://archriscv.felixc.at/archriscv + target: /srv/riscv + last_update_url: https://archriscv.felixc.at/.status/lastupdate.txt + last_update_dst: .status/lastupdate.txt diff --git a/host_vars/debuginfod.archlinux.org/mirrorsync.yml b/host_vars/debuginfod.archlinux.org/mirrorsync.yml new file mode 120000 index 00000000..4897f3f3 --- /dev/null +++ b/host_vars/debuginfod.archlinux.org/mirrorsync.yml @@ -0,0 +1 @@ +../../group_vars/mirrors/mirrorsync.yml \ No newline at end of file diff --git a/host_vars/mirror.pkgbuild.com/misc b/host_vars/mirror.pkgbuild.com/misc index ea0bc5a8..fabd834f 100644 --- a/host_vars/mirror.pkgbuild.com/misc +++ b/host_vars/mirror.pkgbuild.com/misc @@ -1,5 +1,4 @@ mirror_domain: mirror.pkgbuild.com -mirror_debug_packages: false archweb_mirrorcheck_locations: [20, 21] filesystem: btrfs diff --git a/playbooks/archive-mirrors.yml b/playbooks/archive-mirrors.yml index b5730996..593d6a07 100644 --- a/playbooks/archive-mirrors.yml +++ b/playbooks/archive-mirrors.yml @@ -10,7 +10,7 @@ - { role: root_ssh } - { role: certbot } - { role: nginx } - - { role: syncarchive } + - { role: mirrorsync } - { role: archive_web } - { role: prometheus_exporters } - { role: promtail } diff --git a/playbooks/debuginfod.archlinux.org.yml b/playbooks/debuginfod.archlinux.org.yml index 92dbfc06..04564f18 100644 --- a/playbooks/debuginfod.archlinux.org.yml +++ b/playbooks/debuginfod.archlinux.org.yml @@ -11,6 +11,6 @@ - { role: certbot } - { role: nginx } - { role: debuginfod } - - { role: syncdebug } + - { role: mirrorsync } - { role: prometheus_exporters } - { role: promtail } diff --git a/playbooks/mirrors.yml b/playbooks/mirrors.yml index 3d1dbda5..1b2b8822 100644 --- a/playbooks/mirrors.yml +++ b/playbooks/mirrors.yml @@ -10,8 +10,7 @@ - { role: certbot } - { role: nginx } - { role: syncrepo, tags: ['nginx'] } - - { role: syncdebug, when: mirror_debug_packages is not defined or mirror_debug_packages } - - { role: syncriscv, when: riscv_mirror_domain is defined } + - { role: mirrorsync } - { role: archweb, when: archweb_mirrorcheck_locations is defined, archweb_site: false, archweb_services: false, archweb_mirrorcheck: true } - { role: prometheus_exporters } - { role: promtail } diff --git a/roles/mirrorsync/tasks/main.yml b/roles/mirrorsync/tasks/main.yml new file mode 100644 index 00000000..78aacb65 --- /dev/null +++ b/roles/mirrorsync/tasks/main.yml @@ -0,0 +1,18 @@ +- name: Install rsync + pacman: name=rsync state=present + +- name: Set up synchronization + include_tasks: sync.yml + loop: "{{ mirrorsync_mirrors | dict2items }}" + loop_control: + label: "{{ item.key }}" + when: inventory_hostname in item.value.hosts + +- name: Set up nginx + include_tasks: web.yml + loop: "{{ mirrorsync_mirrors | dict2items }}" + loop_control: + label: "{{ item.key }}" + when: + - item.value.mirror_domain is defined + - inventory_hostname in item.value.hosts diff --git a/roles/mirrorsync/tasks/sync.yml b/roles/mirrorsync/tasks/sync.yml new file mode 100644 index 00000000..93de9173 --- /dev/null +++ b/roles/mirrorsync/tasks/sync.yml @@ -0,0 +1,11 @@ +- name: Install sync script for {{ item.key }} + template: src=mirrorsync.j2 dest=/usr/local/bin/sync{{ item.key }} owner=root group=root mode=0755 + +- name: Install systemd service for {{ item.key }} + template: src=mirrorsync.service.j2 dest=/etc/systemd/system/sync{{ item.key }}.service owner=root group=root mode=0644 + +- name: Install systemd timer for {{ item.key }} + template: src=mirrorsync.timer.j2 dest=/etc/systemd/system/sync{{ item.key }}.timer owner=root group=root mode=0644 + +- name: Start and enable timer for {{ item.key }} + systemd: name=sync{{ item.key }}.timer enabled=yes state=started daemon_reload=yes diff --git a/roles/mirrorsync/tasks/web.yml b/roles/mirrorsync/tasks/web.yml new file mode 100644 index 00000000..4dd84681 --- /dev/null +++ b/roles/mirrorsync/tasks/web.yml @@ -0,0 +1,14 @@ +- name: Create ssl cert for {{ item.value.mirror_domain }} + include_role: + name: certificate + vars: + domains: ["{{ item.value.mirror_domain }}"] + challenge: "DNS-01" + +- name: Configure nginx for {{ item.value.mirror_domain }} + template: src=nginx.d.conf.j2 dest=/etc/nginx/nginx.d/{{ item.key }}.conf owner=root group=root mode=0644 + notify: Reload nginx + tags: ['nginx'] + +- name: Make nginx log dir for {{ item.value.mirror_domain }} + file: path=/var/log/nginx/{{ item.value.mirror_domain }} state=directory owner=root group=root mode=0755 diff --git a/roles/mirrorsync/templates/mirrorsync.j2 b/roles/mirrorsync/templates/mirrorsync.j2 new file mode 100755 index 00000000..e2c7f2d2 --- /dev/null +++ b/roles/mirrorsync/templates/mirrorsync.j2 @@ -0,0 +1,36 @@ +#!/bin/bash + +target="{{ item.value.target }}" +lock="/run/lock/sync{{ item.key }}.lck" +source_url='{{ item.value.source }}' +lastupdate_url='{{ item.value.last_update_url }}' + +[ ! -d "${target}" ] && mkdir -p "${target}" + +exec 9>"${lock}" +flock -n 9 || exit + +{% set delay_updates = '--delay-updates' if item.value.delay_updates | default(true) %} +rsync_cmd() { + local -a cmd=(rsync -rlptH --safe-links --delete-delay {{ delay_updates }} + "--timeout=600" "--contimeout=60" --no-motd) + + if stty &>/dev/null; then + cmd+=(-h -v --progress) + else + cmd+=("--info=name1") + fi + + "${cmd[@]}" "$@" +} + +# if we are called without a tty (cronjob) only run when there are changes +if ! tty -s && [[ -f "$target/{{ item.value.last_update_dst }}" ]] && diff -b <(curl -Ls "$lastupdate_url") "$target/{{ item.value.last_update_dst }}" >/dev/null; then + exit 0 +fi + +rsync_cmd \ +{% for opt in item.value.rsync_options | default() %} + {{ opt }} \ +{% endfor %} + "${source_url}" "${target}" diff --git a/roles/syncdebug/files/syncdebug.service b/roles/mirrorsync/templates/mirrorsync.service.j2 similarity index 52% rename from roles/syncdebug/files/syncdebug.service rename to roles/mirrorsync/templates/mirrorsync.service.j2 index 48d1c40f..08e5b875 100644 --- a/roles/syncdebug/files/syncdebug.service +++ b/roles/mirrorsync/templates/mirrorsync.service.j2 @@ -1,12 +1,12 @@ [Unit] -Description=Synchronize debug packages -RequiresMountsFor=/srv/ftp +Description=Synchronize {{ item.key }} mirror +RequiresMountsFor={{ item.value.target }} Wants=network-online.target After=network-online.target [Service] Type=oneshot -ExecStart=/usr/local/bin/syncdebug +ExecStart=/usr/local/bin/sync{{ item.key }} Nice=19 IOSchedulingClass=best-effort IOSchedulingPriority=7 diff --git a/roles/syncriscv/files/syncriscv.timer b/roles/mirrorsync/templates/mirrorsync.timer.j2 similarity index 67% rename from roles/syncriscv/files/syncriscv.timer rename to roles/mirrorsync/templates/mirrorsync.timer.j2 index b944e784..97e9123e 100644 --- a/roles/syncriscv/files/syncriscv.timer +++ b/roles/mirrorsync/templates/mirrorsync.timer.j2 @@ -1,5 +1,5 @@ [Unit] -Description=Minutely RISC-V mirror sync +Description=Minutely {{ item.key }} mirror sync [Timer] OnCalendar=minutely diff --git a/roles/mirrorsync/templates/nginx.d.conf.j2 b/roles/mirrorsync/templates/nginx.d.conf.j2 new file mode 100644 index 00000000..f0b1696d --- /dev/null +++ b/roles/mirrorsync/templates/nginx.d.conf.j2 @@ -0,0 +1,22 @@ +server { + listen 80; + listen [::]:80; + listen 443 ssl http2; + listen [::]:443 ssl http2; + server_name {{ item.value.mirror_domain }}; + root /srv/riscv; + + access_log /var/log/nginx/{{ item.value.mirror_domain }}/access.log reduced; + access_log /var/log/nginx/{{ item.value.mirror_domain }}/access.log.json json_reduced; + error_log /var/log/nginx/{{ item.value.mirror_domain }}/error.log; + + include snippets/letsencrypt.conf; + + ssl_certificate /etc/letsencrypt/live/{{ item.value.mirror_domain }}/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/{{ item.value.mirror_domain }}/privkey.pem; + ssl_trusted_certificate /etc/letsencrypt/live/{{ item.value.mirror_domain }}/chain.pem; + + add_header X-Served-By "{{ inventory_hostname }}"; + + autoindex on; +} diff --git a/roles/syncarchive/files/syncarchive b/roles/syncarchive/files/syncarchive deleted file mode 100755 index 50957f57..00000000 --- a/roles/syncarchive/files/syncarchive +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/bash - -target="/srv/archive" -lock="/var/lock/syncarchive.lck" -source_url='rsync://rsync.archlinux.org/archive' -lastupdate_url='https://archive.archlinux.org/repos/last/lastupdate' - -[ ! -d "${target}" ] && mkdir -p "${target}" - -exec 9>"${lock}" -flock -n 9 || exit - -rsync_cmd() { - local -a cmd=(rsync -rlptH --safe-links --delete-delay - "--timeout=600" "--contimeout=60" --no-motd) - - if stty &>/dev/null; then - cmd+=(-h -v --progress) - else - cmd+=("--info=name1") - fi - - "${cmd[@]}" "$@" -} - -# if we are called without a tty (cronjob) only run when there are changes -if ! tty -s && [[ -f "$target/lastupdate" ]] && diff -b <(curl -Ls "$lastupdate_url") "$target/lastupdate" >/dev/null; then - exit 0 -fi - -rsync_cmd \ - --exclude=".well-known" \ - "${source_url}" \ - "${target}" diff --git a/roles/syncarchive/files/syncarchive.service b/roles/syncarchive/files/syncarchive.service deleted file mode 100644 index c7a1347a..00000000 --- a/roles/syncarchive/files/syncarchive.service +++ /dev/null @@ -1,12 +0,0 @@ -[Unit] -Description=Synchronize package archive mirror -RequiresMountsFor=/srv/archive -Wants=network-online.target -After=network-online.target - -[Service] -Type=oneshot -ExecStart=/usr/local/bin/syncarchive -Nice=19 -IOSchedulingClass=best-effort -IOSchedulingPriority=7 diff --git a/roles/syncarchive/files/syncarchive.timer b/roles/syncarchive/files/syncarchive.timer deleted file mode 100644 index b4f2548f..00000000 --- a/roles/syncarchive/files/syncarchive.timer +++ /dev/null @@ -1,10 +0,0 @@ -[Unit] -Description=Sync archive every 4 hours - -[Timer] -OnCalendar=00/4:00 -AccuracySec=1m -Persistent=true - -[Install] -WantedBy=timers.target diff --git a/roles/syncarchive/tasks/main.yml b/roles/syncarchive/tasks/main.yml deleted file mode 100644 index a6518940..00000000 --- a/roles/syncarchive/tasks/main.yml +++ /dev/null @@ -1,20 +0,0 @@ -- name: Install rsync - pacman: name=rsync state=present - -- name: Install syncarchive script - copy: src=syncarchive dest=/usr/local/bin/syncarchive owner=root group=root mode=0755 - -- name: Install syncarchive units - copy: src={{ item }} dest=/etc/systemd/system/{{ item }} owner=root group=root mode=0644 - with_items: - - syncarchive.timer - - syncarchive.service - -- name: Start and enable syncarchive units - systemd: - name: "{{ item }}" - enabled: true - state: started - daemon_reload: true - with_items: - - syncarchive.timer diff --git a/roles/syncdebug/files/syncdebug b/roles/syncdebug/files/syncdebug deleted file mode 100755 index 8af79662..00000000 --- a/roles/syncdebug/files/syncdebug +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/bash - -target="/srv/ftp" -lock="/var/lock/syncdebug.lck" -source_url='rsync://rsync.archlinux.org/debug_packages' -lastupdate_url='https://rsync.archlinux.org/lastupdate' - -[ ! -d "${target}" ] && mkdir -p "${target}" - -exec 9>"${lock}" -flock -n 9 || exit - -rsync_cmd() { - local -a cmd=(rsync -rlptH --safe-links --delete-delay --delay-updates - "--timeout=600" "--contimeout=60" --no-motd) - - if stty &>/dev/null; then - cmd+=(-h -v --progress) - else - cmd+=("--info=name1") - fi - - "${cmd[@]}" "$@" -} - -# if we are called without a tty (cronjob) only run when there are changes -if ! tty -s && [[ -f "$target/lastupdate" ]] && diff -b <(curl -Ls "$lastupdate_url") "$target/lastupdate" >/dev/null; then - exit 0 -fi - -rsync_cmd \ - --include="*-debug/***" \ - --include="pool/" \ - --include="pool/*-debug/***" \ - --exclude="*" \ - "${source_url}" \ - "${target}" diff --git a/roles/syncdebug/files/syncdebug.timer b/roles/syncdebug/files/syncdebug.timer deleted file mode 100644 index e6d7d4a8..00000000 --- a/roles/syncdebug/files/syncdebug.timer +++ /dev/null @@ -1,10 +0,0 @@ -[Unit] -Description=Sync debug packages every minute - -[Timer] -OnCalendar=minutely -AccuracySec=1m -Persistent=true - -[Install] -WantedBy=timers.target diff --git a/roles/syncdebug/tasks/main.yml b/roles/syncdebug/tasks/main.yml deleted file mode 100644 index 16903b39..00000000 --- a/roles/syncdebug/tasks/main.yml +++ /dev/null @@ -1,20 +0,0 @@ -- name: Install rsync - pacman: name=rsync state=present - -- name: Install syncdebug script - copy: src=syncdebug dest=/usr/local/bin/syncdebug owner=root group=root mode=0755 - -- name: Install syncdebug units - copy: src={{ item }} dest=/etc/systemd/system/{{ item }} owner=root group=root mode=0644 - with_items: - - syncdebug.timer - - syncdebug.service - -- name: Start and enable syncdebug units - systemd: - name: "{{ item }}" - enabled: true - state: started - daemon_reload: true - with_items: - - syncdebug.timer diff --git a/roles/syncriscv/files/syncriscv b/roles/syncriscv/files/syncriscv deleted file mode 100755 index b1116bf3..00000000 --- a/roles/syncriscv/files/syncriscv +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash - -target="/srv/riscv" -lock="/var/lock/syncriscv.lck" -source_url='rsync://archriscv.felixc.at/archriscv' -lastupdate_url='https://archriscv.felixc.at/.status/lastupdate.txt' - -[ ! -d "${target}" ] && mkdir -p "${target}" - -exec 9>"${lock}" -flock -n 9 || exit - -rsync_cmd() { - local -a cmd=(rsync -rlptH --safe-links --delete-delay --delay-updates - "--timeout=600" "--contimeout=60" --no-motd) - - if stty &>/dev/null; then - cmd+=(-h -v --progress) - else - cmd+=("--info=name1") - fi - - "${cmd[@]}" "$@" -} - -# if we are called without a tty (cronjob) only run when there are changes -if ! tty -s && [[ -f "$target/.status/lastupdate.txt" ]] && diff -b <(curl -Ls "$lastupdate_url") "$target/.status/lastupdate.txt" >/dev/null; then - exit 0 -fi - -rsync_cmd "${source_url}" "${target}" diff --git a/roles/syncriscv/files/syncriscv.service b/roles/syncriscv/files/syncriscv.service deleted file mode 100644 index 2426f9ff..00000000 --- a/roles/syncriscv/files/syncriscv.service +++ /dev/null @@ -1,12 +0,0 @@ -[Unit] -Description=Synchronize RISC-V mirror -RequiresMountsFor=/srv/riscv -Wants=network-online.target -After=network-online.target - -[Service] -Type=oneshot -ExecStart=/usr/local/bin/syncriscv -Nice=19 -IOSchedulingClass=best-effort -IOSchedulingPriority=7 diff --git a/roles/syncriscv/tasks/main.yml b/roles/syncriscv/tasks/main.yml deleted file mode 100644 index 9c8612bc..00000000 --- a/roles/syncriscv/tasks/main.yml +++ /dev/null @@ -1,29 +0,0 @@ -- name: Create ssl cert - include_role: - name: certificate - vars: - domains: ["{{ riscv_mirror_domain }}"] - challenge: "DNS-01" - -- name: Install rsync - pacman: name=rsync state=present - -- name: Install syncriscv script - copy: src=syncriscv dest=/usr/local/bin/syncriscv owner=root group=root mode=0755 - -- name: Install syncriscv units - copy: src={{ item }} dest=/etc/systemd/system/{{ item }} owner=root group=root mode=0644 - with_items: - - syncriscv.timer - - syncriscv.service - -- name: Start and enable syncriscv timer - systemd: name=syncriscv.timer enabled=yes state=started daemon_reload=yes - -- name: Set up nginx - template: src=nginx.d.conf.j2 dest=/etc/nginx/nginx.d/riscv.conf owner=root group=root mode=0644 - notify: Reload nginx - tags: ['nginx'] - -- name: Make nginx log dir - file: path=/var/log/nginx/{{ riscv_mirror_domain }} state=directory owner=root group=root mode=0755 diff --git a/roles/syncriscv/templates/nginx.d.conf.j2 b/roles/syncriscv/templates/nginx.d.conf.j2 deleted file mode 100644 index 07a116d8..00000000 --- a/roles/syncriscv/templates/nginx.d.conf.j2 +++ /dev/null @@ -1,22 +0,0 @@ -server { - listen 80; - listen [::]:80; - listen 443 ssl http2; - listen [::]:443 ssl http2; - server_name {{ riscv_mirror_domain }}; - root /srv/riscv; - - access_log /var/log/nginx/{{ riscv_mirror_domain }}/access.log reduced; - access_log /var/log/nginx/{{ riscv_mirror_domain }}/access.log.json json_reduced; - error_log /var/log/nginx/{{ riscv_mirror_domain }}/error.log; - - include snippets/letsencrypt.conf; - - ssl_certificate /etc/letsencrypt/live/{{ riscv_mirror_domain }}/fullchain.pem; - ssl_certificate_key /etc/letsencrypt/live/{{ riscv_mirror_domain }}/privkey.pem; - ssl_trusted_certificate /etc/letsencrypt/live/{{ riscv_mirror_domain }}/chain.pem; - - add_header X-Served-By "{{ inventory_hostname }}"; - - autoindex on; -}