Combine sync{archive,debug,riscv} into mirrorsync

These roles are very similar and can be merged into a single new role.

Note: The archive mirror is changed from a 4-hour sync to minutely for
conformity with the other two mirrors. In practice this doesn't matter
as it was already taking over 4 hours to finish and was starting again
right after its previous run.

group_vars/geo_mirrors/misc.yml

 certbot_dns_support: true
 geo_mirror_domain: geo.mirror.pkgbuild.com
-riscv_mirror_domain: riscv.mirror.pkgbuild.com

group_vars/mirrors/mirrorsync.yml

+mirrorsync_mirrors:
+  archive:
+    hosts: "{{ groups['archive_mirrors'] }}"
+    source: rsync://rsync.archlinux.org/archive
+    target: /srv/archive
+    last_update_url: https://archive.archlinux.org/repos/last/lastupdate
+    last_update_dst: lastupdate
+    delay_updates: false
+  debug:
+    hosts: "{{ groups['geo_mirrors'] + ['debuginfod.archlinux.org'] }}"
+    source: rsync://rsync.archlinux.org/debug_packages
+    target: /srv/ftp
+    last_update_url: https://rsync.archlinux.org/lastupdate
+    last_update_dst: lastupdate
+    rsync_options:
+      - --include="*-debug/***"
+      - --include="pool/"
+      - --include="pool/*-debug/***"
+      - --exclude="*"
+  riscv:
+    hosts: "{{ groups['geo_mirrors'] }}"
+    mirror_domain: riscv.mirror.pkgbuild.com
+    source: rsync://archriscv.felixc.at/archriscv
+    target: /srv/riscv
+    last_update_url: https://archriscv.felixc.at/.status/lastupdate.txt
+    last_update_dst: .status/lastupdate.txt

host_vars/debuginfod.archlinux.org/mirrorsync.yml

+../../group_vars/mirrors/mirrorsync.yml
\ No newline at end of file

host_vars/mirror.pkgbuild.com/misc

 mirror_domain: mirror.pkgbuild.com
-mirror_debug_packages: false
 archweb_mirrorcheck_locations: [20, 21]
 filesystem: btrfs
 

playbooks/archive-mirrors.yml

@@ -10,7 +10,7 @@
     - { role: root_ssh }
     - { role: certbot }
     - { role: nginx }
-    - { role: syncarchive }
+    - { role: mirrorsync }
     - { role: archive_web }
     - { role: prometheus_exporters }
     - { role: promtail }

playbooks/debuginfod.archlinux.org.yml

@@ -11,6 +11,6 @@
     - { role: certbot }
     - { role: nginx }
     - { role: debuginfod }
-    - { role: syncdebug }
+    - { role: mirrorsync }
     - { role: prometheus_exporters }
     - { role: promtail }

playbooks/mirrors.yml

@@ -10,8 +10,7 @@
     - { role: certbot }
     - { role: nginx }
     - { role: syncrepo, tags: ['nginx'] }
-    - { role: syncdebug, when: mirror_debug_packages is not defined or mirror_debug_packages }
-    - { role: syncriscv, when: riscv_mirror_domain is defined }
+    - { role: mirrorsync }
     - { role: archweb, when: archweb_mirrorcheck_locations is defined, archweb_site: false, archweb_services: false, archweb_mirrorcheck: true }
     - { role: prometheus_exporters }
     - { role: promtail }

roles/mirrorsync/tasks/main.yml

+- name: Install rsync
+  pacman: name=rsync state=present
+
+- name: Set up synchronization
+  include_tasks: sync.yml
+  loop: "{{ mirrorsync_mirrors | dict2items }}"
+  loop_control:
+    label: "{{ item.key }}"
+  when: inventory_hostname in item.value.hosts
+
+- name: Set up nginx
+  include_tasks: web.yml
+  loop: "{{ mirrorsync_mirrors | dict2items }}"
+  loop_control:
+    label: "{{ item.key }}"
+  when:
+    - item.value.mirror_domain is defined
+    - inventory_hostname in item.value.hosts

roles/mirrorsync/tasks/sync.yml

+- name: Install sync script for {{ item.key }}
+  template: src=mirrorsync.j2 dest=/usr/local/bin/sync{{ item.key }} owner=root group=root mode=0755
+
+- name: Install systemd service for {{ item.key }}
+  template: src=mirrorsync.service.j2 dest=/etc/systemd/system/sync{{ item.key }}.service owner=root group=root mode=0644
+
+- name: Install systemd timer for {{ item.key }}
+  template: src=mirrorsync.timer.j2 dest=/etc/systemd/system/sync{{ item.key }}.timer owner=root group=root mode=0644
+
+- name: Start and enable timer for {{ item.key }}
+  systemd: name=sync{{ item.key }}.timer enabled=yes state=started daemon_reload=yes

roles/mirrorsync/tasks/web.yml

+- name: Create ssl cert for {{ item.value.mirror_domain }}
+  include_role:
+    name: certificate
+  vars:
+    domains: ["{{ item.value.mirror_domain }}"]
+    challenge: "DNS-01"
+
+- name: Configure nginx for {{ item.value.mirror_domain }}
+  template: src=nginx.d.conf.j2 dest=/etc/nginx/nginx.d/{{ item.key }}.conf owner=root group=root mode=0644
+  notify: Reload nginx
+  tags: ['nginx']
+
+- name: Make nginx log dir for {{ item.value.mirror_domain }}
+  file: path=/var/log/nginx/{{ item.value.mirror_domain }} state=directory owner=root group=root mode=0755

roles/syncarchive/files/syncarchive → roles/mirrorsync/templates/mirrorsync.j2

 #!/bin/bash
 
-target="/srv/archive"
-lock="/var/lock/syncarchive.lck"
-source_url='rsync://rsync.archlinux.org/archive'
-lastupdate_url='https://archive.archlinux.org/repos/last/lastupdate'
+target="{{ item.value.target }}"
+lock="/run/lock/sync{{ item.key }}.lck"
+source_url='{{ item.value.source }}'
+lastupdate_url='{{ item.value.last_update_url }}'
 
 [ ! -d "${target}" ] && mkdir -p "${target}"
 
 exec 9>"${lock}"
 flock -n 9 || exit
 
+{% set delay_updates = '--delay-updates' if item.value.delay_updates | default(true) %}
 rsync_cmd() {
-	local -a cmd=(rsync -rlptH --safe-links --delete-delay
+	local -a cmd=(rsync -rlptH --safe-links --delete-delay {{ delay_updates }}
 		"--timeout=600" "--contimeout=60" --no-motd)
 
 	if stty &>/dev/null; then
@@ -24,11 +25,12 @@ rsync_cmd() {
 }
 
 # if we are called without a tty (cronjob) only run when there are changes
-if ! tty -s && [[ -f "$target/lastupdate" ]] && diff -b <(curl -Ls "$lastupdate_url") "$target/lastupdate" >/dev/null; then
+if ! tty -s && [[ -f "$target/{{ item.value.last_update_dst }}" ]] && diff -b <(curl -Ls "$lastupdate_url") "$target/{{ item.value.last_update_dst }}" >/dev/null; then
 	exit 0
 fi
 
 rsync_cmd \
-    --exclude=".well-known" \
-	"${source_url}" \
-	"${target}"
+{% for opt in item.value.rsync_options | default() %}
+	{{ opt }} \
+{% endfor %}
+	"${source_url}" "${target}"

roles/syncdebug/files/syncdebug.service → roles/mirrorsync/templates/mirrorsync.service.j2

 [Unit]
-Description=Synchronize debug packages
-RequiresMountsFor=/srv/ftp
+Description=Synchronize {{ item.key }} mirror
+RequiresMountsFor={{ item.value.target }}
 Wants=network-online.target
 After=network-online.target
 
 [Service]
 Type=oneshot
-ExecStart=/usr/local/bin/syncdebug
+ExecStart=/usr/local/bin/sync{{ item.key }}
 Nice=19
 IOSchedulingClass=best-effort
 IOSchedulingPriority=7

roles/syncriscv/files/syncriscv.timer → roles/mirrorsync/templates/mirrorsync.timer.j2

 [Unit]
-Description=Minutely RISC-V mirror sync
+Description=Minutely {{ item.key }} mirror sync
 
 [Timer]
 OnCalendar=minutely

roles/syncriscv/templates/nginx.d.conf.j2 → roles/mirrorsync/templates/nginx.d.conf.j2

@@ -3,18 +3,18 @@ server {
     listen       [::]:80;
     listen       443 ssl http2;
     listen       [::]:443 ssl http2;
-    server_name  {{ riscv_mirror_domain }};
+    server_name  {{ item.value.mirror_domain }};
     root         /srv/riscv;
 
-    access_log   /var/log/nginx/{{ riscv_mirror_domain }}/access.log reduced;
-    access_log   /var/log/nginx/{{ riscv_mirror_domain }}/access.log.json json_reduced;
-    error_log    /var/log/nginx/{{ riscv_mirror_domain }}/error.log;
+    access_log   /var/log/nginx/{{ item.value.mirror_domain }}/access.log reduced;
+    access_log   /var/log/nginx/{{ item.value.mirror_domain }}/access.log.json json_reduced;
+    error_log    /var/log/nginx/{{ item.value.mirror_domain }}/error.log;
 
     include snippets/letsencrypt.conf;
 
-    ssl_certificate      /etc/letsencrypt/live/{{ riscv_mirror_domain }}/fullchain.pem;
-    ssl_certificate_key  /etc/letsencrypt/live/{{ riscv_mirror_domain }}/privkey.pem;
-    ssl_trusted_certificate /etc/letsencrypt/live/{{ riscv_mirror_domain }}/chain.pem;
+    ssl_certificate      /etc/letsencrypt/live/{{ item.value.mirror_domain }}/fullchain.pem;
+    ssl_certificate_key  /etc/letsencrypt/live/{{ item.value.mirror_domain }}/privkey.pem;
+    ssl_trusted_certificate /etc/letsencrypt/live/{{ item.value.mirror_domain }}/chain.pem;
 
     add_header X-Served-By "{{ inventory_hostname }}";
 

roles/syncarchive/files/syncarchive.service

-[Unit]
-Description=Synchronize package archive mirror
-RequiresMountsFor=/srv/archive
-Wants=network-online.target
-After=network-online.target
-
-[Service]
-Type=oneshot
-ExecStart=/usr/local/bin/syncarchive
-Nice=19
-IOSchedulingClass=best-effort
-IOSchedulingPriority=7

roles/syncarchive/files/syncarchive.timer

-[Unit]
-Description=Sync archive every 4 hours
-
-[Timer]
-OnCalendar=00/4:00
-AccuracySec=1m
-Persistent=true
-
-[Install]
-WantedBy=timers.target

roles/syncarchive/tasks/main.yml

-- name: Install rsync
-  pacman: name=rsync state=present
-
-- name: Install syncarchive script
-  copy: src=syncarchive dest=/usr/local/bin/syncarchive owner=root group=root mode=0755
-
-- name: Install syncarchive units
-  copy: src={{ item }} dest=/etc/systemd/system/{{ item }} owner=root group=root mode=0644
-  with_items:
-    - syncarchive.timer
-    - syncarchive.service
-
-- name: Start and enable syncarchive units
-  systemd:
-    name: "{{ item }}"
-    enabled: true
-    state: started
-    daemon_reload: true
-  with_items:
-    - syncarchive.timer

roles/syncdebug/files/syncdebug

-#!/bin/bash
-
-target="/srv/ftp"
-lock="/var/lock/syncdebug.lck"
-source_url='rsync://rsync.archlinux.org/debug_packages'
-lastupdate_url='https://rsync.archlinux.org/lastupdate'
-
-[ ! -d "${target}" ] && mkdir -p "${target}"
-
-exec 9>"${lock}"
-flock -n 9 || exit
-
-rsync_cmd() {
-	local -a cmd=(rsync -rlptH --safe-links --delete-delay --delay-updates
-		"--timeout=600" "--contimeout=60" --no-motd)
-
-	if stty &>/dev/null; then
-		cmd+=(-h -v --progress)
-	else
-		cmd+=("--info=name1")
-	fi
-
-	"${cmd[@]}" "$@"
-}
-
-# if we are called without a tty (cronjob) only run when there are changes
-if ! tty -s && [[ -f "$target/lastupdate" ]] && diff -b <(curl -Ls "$lastupdate_url") "$target/lastupdate" >/dev/null; then
-	exit 0
-fi
-
-rsync_cmd \
-	--include="*-debug/***" \
-	--include="pool/" \
-	--include="pool/*-debug/***" \
-	--exclude="*" \
-	"${source_url}" \
-	"${target}"

roles/syncdebug/files/syncdebug.timer

-[Unit]
-Description=Sync debug packages every minute
-
-[Timer]
-OnCalendar=minutely
-AccuracySec=1m
-Persistent=true
-
-[Install]
-WantedBy=timers.target

roles/syncdebug/tasks/main.yml

-- name: Install rsync
-  pacman: name=rsync state=present
-
-- name: Install syncdebug script
-  copy: src=syncdebug dest=/usr/local/bin/syncdebug owner=root group=root mode=0755
-
-- name: Install syncdebug units
-  copy: src={{ item }} dest=/etc/systemd/system/{{ item }} owner=root group=root mode=0644
-  with_items:
-    - syncdebug.timer
-    - syncdebug.service
-
-- name: Start and enable syncdebug units
-  systemd:
-    name: "{{ item }}"
-    enabled: true
-    state: started
-    daemon_reload: true
-  with_items:
-    - syncdebug.timer