Skip to content
Snippets Groups Projects
Verified Commit 43eb814b authored by Evangelos Foutras's avatar Evangelos Foutras :smiley_cat:
Browse files

Combine sync{archive,debug,riscv} into mirrorsync

These roles are very similar and can be merged into a single new role.

Note: The archive mirror is changed from a 4-hour sync to minutely for
conformity with the other two mirrors. In practice this doesn't matter
as it was already taking over 4 hours to finish and was starting again
right after its previous run.
parent 93407f75
No related branches found
No related tags found
1 merge request!632Combine sync{archive,debug,riscv} into mirrorsync
Pipeline #31845 passed
Showing
with 135 additions and 119 deletions
certbot_dns_support: true
geo_mirror_domain: geo.mirror.pkgbuild.com
riscv_mirror_domain: riscv.mirror.pkgbuild.com
mirrorsync_mirrors:
archive:
hosts: "{{ groups['archive_mirrors'] }}"
source: rsync://rsync.archlinux.org/archive
target: /srv/archive
last_update_url: https://archive.archlinux.org/repos/last/lastupdate
last_update_dst: lastupdate
delay_updates: false
debug:
hosts: "{{ groups['geo_mirrors'] + ['debuginfod.archlinux.org'] }}"
source: rsync://rsync.archlinux.org/debug_packages
target: /srv/ftp
last_update_url: https://rsync.archlinux.org/lastupdate
last_update_dst: lastupdate
rsync_options:
- --include="*-debug/***"
- --include="pool/"
- --include="pool/*-debug/***"
- --exclude="*"
riscv:
hosts: "{{ groups['geo_mirrors'] }}"
mirror_domain: riscv.mirror.pkgbuild.com
source: rsync://archriscv.felixc.at/archriscv
target: /srv/riscv
last_update_url: https://archriscv.felixc.at/.status/lastupdate.txt
last_update_dst: .status/lastupdate.txt
../../group_vars/mirrors/mirrorsync.yml
\ No newline at end of file
mirror_domain: mirror.pkgbuild.com
mirror_debug_packages: false
archweb_mirrorcheck_locations: [20, 21]
filesystem: btrfs
......
......@@ -10,7 +10,7 @@
- { role: root_ssh }
- { role: certbot }
- { role: nginx }
- { role: syncarchive }
- { role: mirrorsync }
- { role: archive_web }
- { role: prometheus_exporters }
- { role: promtail }
......@@ -11,6 +11,6 @@
- { role: certbot }
- { role: nginx }
- { role: debuginfod }
- { role: syncdebug }
- { role: mirrorsync }
- { role: prometheus_exporters }
- { role: promtail }
......@@ -10,8 +10,7 @@
- { role: certbot }
- { role: nginx }
- { role: syncrepo, tags: ['nginx'] }
- { role: syncdebug, when: mirror_debug_packages is not defined or mirror_debug_packages }
- { role: syncriscv, when: riscv_mirror_domain is defined }
- { role: mirrorsync }
- { role: archweb, when: archweb_mirrorcheck_locations is defined, archweb_site: false, archweb_services: false, archweb_mirrorcheck: true }
- { role: prometheus_exporters }
- { role: promtail }
......
- name: Install rsync
pacman: name=rsync state=present
- name: Set up synchronization
include_tasks: sync.yml
loop: "{{ mirrorsync_mirrors | dict2items }}"
loop_control:
label: "{{ item.key }}"
when: inventory_hostname in item.value.hosts
- name: Set up nginx
include_tasks: web.yml
loop: "{{ mirrorsync_mirrors | dict2items }}"
loop_control:
label: "{{ item.key }}"
when:
- item.value.mirror_domain is defined
- inventory_hostname in item.value.hosts
- name: Install sync script for {{ item.key }}
template: src=mirrorsync.j2 dest=/usr/local/bin/sync{{ item.key }} owner=root group=root mode=0755
- name: Install systemd service for {{ item.key }}
template: src=mirrorsync.service.j2 dest=/etc/systemd/system/sync{{ item.key }}.service owner=root group=root mode=0644
- name: Install systemd timer for {{ item.key }}
template: src=mirrorsync.timer.j2 dest=/etc/systemd/system/sync{{ item.key }}.timer owner=root group=root mode=0644
- name: Start and enable timer for {{ item.key }}
systemd: name=sync{{ item.key }}.timer enabled=yes state=started daemon_reload=yes
- name: Create ssl cert for {{ item.value.mirror_domain }}
include_role:
name: certificate
vars:
domains: ["{{ item.value.mirror_domain }}"]
challenge: "DNS-01"
- name: Configure nginx for {{ item.value.mirror_domain }}
template: src=nginx.d.conf.j2 dest=/etc/nginx/nginx.d/{{ item.key }}.conf owner=root group=root mode=0644
notify: Reload nginx
tags: ['nginx']
- name: Make nginx log dir for {{ item.value.mirror_domain }}
file: path=/var/log/nginx/{{ item.value.mirror_domain }} state=directory owner=root group=root mode=0755
#!/bin/bash
target="/srv/archive"
lock="/var/lock/syncarchive.lck"
source_url='rsync://rsync.archlinux.org/archive'
lastupdate_url='https://archive.archlinux.org/repos/last/lastupdate'
target="{{ item.value.target }}"
lock="/run/lock/sync{{ item.key }}.lck"
source_url='{{ item.value.source }}'
lastupdate_url='{{ item.value.last_update_url }}'
[ ! -d "${target}" ] && mkdir -p "${target}"
exec 9>"${lock}"
flock -n 9 || exit
{% set delay_updates = '--delay-updates' if item.value.delay_updates | default(true) %}
rsync_cmd() {
local -a cmd=(rsync -rlptH --safe-links --delete-delay
local -a cmd=(rsync -rlptH --safe-links --delete-delay {{ delay_updates }}
"--timeout=600" "--contimeout=60" --no-motd)
if stty &>/dev/null; then
......@@ -24,11 +25,12 @@ rsync_cmd() {
}
# if we are called without a tty (cronjob) only run when there are changes
if ! tty -s && [[ -f "$target/lastupdate" ]] && diff -b <(curl -Ls "$lastupdate_url") "$target/lastupdate" >/dev/null; then
if ! tty -s && [[ -f "$target/{{ item.value.last_update_dst }}" ]] && diff -b <(curl -Ls "$lastupdate_url") "$target/{{ item.value.last_update_dst }}" >/dev/null; then
exit 0
fi
rsync_cmd \
--exclude=".well-known" \
"${source_url}" \
"${target}"
{% for opt in item.value.rsync_options | default() %}
{{ opt }} \
{% endfor %}
"${source_url}" "${target}"
[Unit]
Description=Synchronize debug packages
RequiresMountsFor=/srv/ftp
Description=Synchronize {{ item.key }} mirror
RequiresMountsFor={{ item.value.target }}
Wants=network-online.target
After=network-online.target
[Service]
Type=oneshot
ExecStart=/usr/local/bin/syncdebug
ExecStart=/usr/local/bin/sync{{ item.key }}
Nice=19
IOSchedulingClass=best-effort
IOSchedulingPriority=7
[Unit]
Description=Minutely RISC-V mirror sync
Description=Minutely {{ item.key }} mirror sync
[Timer]
OnCalendar=minutely
......
......@@ -3,18 +3,18 @@ server {
listen [::]:80;
listen 443 ssl http2;
listen [::]:443 ssl http2;
server_name {{ riscv_mirror_domain }};
server_name {{ item.value.mirror_domain }};
root /srv/riscv;
access_log /var/log/nginx/{{ riscv_mirror_domain }}/access.log reduced;
access_log /var/log/nginx/{{ riscv_mirror_domain }}/access.log.json json_reduced;
error_log /var/log/nginx/{{ riscv_mirror_domain }}/error.log;
access_log /var/log/nginx/{{ item.value.mirror_domain }}/access.log reduced;
access_log /var/log/nginx/{{ item.value.mirror_domain }}/access.log.json json_reduced;
error_log /var/log/nginx/{{ item.value.mirror_domain }}/error.log;
include snippets/letsencrypt.conf;
ssl_certificate /etc/letsencrypt/live/{{ riscv_mirror_domain }}/fullchain.pem;
ssl_certificate_key /etc/letsencrypt/live/{{ riscv_mirror_domain }}/privkey.pem;
ssl_trusted_certificate /etc/letsencrypt/live/{{ riscv_mirror_domain }}/chain.pem;
ssl_certificate /etc/letsencrypt/live/{{ item.value.mirror_domain }}/fullchain.pem;
ssl_certificate_key /etc/letsencrypt/live/{{ item.value.mirror_domain }}/privkey.pem;
ssl_trusted_certificate /etc/letsencrypt/live/{{ item.value.mirror_domain }}/chain.pem;
add_header X-Served-By "{{ inventory_hostname }}";
......
[Unit]
Description=Synchronize package archive mirror
RequiresMountsFor=/srv/archive
Wants=network-online.target
After=network-online.target
[Service]
Type=oneshot
ExecStart=/usr/local/bin/syncarchive
Nice=19
IOSchedulingClass=best-effort
IOSchedulingPriority=7
[Unit]
Description=Sync archive every 4 hours
[Timer]
OnCalendar=00/4:00
AccuracySec=1m
Persistent=true
[Install]
WantedBy=timers.target
- name: Install rsync
pacman: name=rsync state=present
- name: Install syncarchive script
copy: src=syncarchive dest=/usr/local/bin/syncarchive owner=root group=root mode=0755
- name: Install syncarchive units
copy: src={{ item }} dest=/etc/systemd/system/{{ item }} owner=root group=root mode=0644
with_items:
- syncarchive.timer
- syncarchive.service
- name: Start and enable syncarchive units
systemd:
name: "{{ item }}"
enabled: true
state: started
daemon_reload: true
with_items:
- syncarchive.timer
#!/bin/bash
target="/srv/ftp"
lock="/var/lock/syncdebug.lck"
source_url='rsync://rsync.archlinux.org/debug_packages'
lastupdate_url='https://rsync.archlinux.org/lastupdate'
[ ! -d "${target}" ] && mkdir -p "${target}"
exec 9>"${lock}"
flock -n 9 || exit
rsync_cmd() {
local -a cmd=(rsync -rlptH --safe-links --delete-delay --delay-updates
"--timeout=600" "--contimeout=60" --no-motd)
if stty &>/dev/null; then
cmd+=(-h -v --progress)
else
cmd+=("--info=name1")
fi
"${cmd[@]}" "$@"
}
# if we are called without a tty (cronjob) only run when there are changes
if ! tty -s && [[ -f "$target/lastupdate" ]] && diff -b <(curl -Ls "$lastupdate_url") "$target/lastupdate" >/dev/null; then
exit 0
fi
rsync_cmd \
--include="*-debug/***" \
--include="pool/" \
--include="pool/*-debug/***" \
--exclude="*" \
"${source_url}" \
"${target}"
[Unit]
Description=Sync debug packages every minute
[Timer]
OnCalendar=minutely
AccuracySec=1m
Persistent=true
[Install]
WantedBy=timers.target
- name: Install rsync
pacman: name=rsync state=present
- name: Install syncdebug script
copy: src=syncdebug dest=/usr/local/bin/syncdebug owner=root group=root mode=0755
- name: Install syncdebug units
copy: src={{ item }} dest=/etc/systemd/system/{{ item }} owner=root group=root mode=0644
with_items:
- syncdebug.timer
- syncdebug.service
- name: Start and enable syncdebug units
systemd:
name: "{{ item }}"
enabled: true
state: started
daemon_reload: true
with_items:
- syncdebug.timer
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment