prepare the split of repo and archive server

related to #531



Co-authored-by: Kristian Klausen <kristian@klausen.dk>
Signed-off-by: Christian Heusel <christian@heusel.eu>

group_vars/mirrors/mirrorsync.yml

@@ -18,7 +18,7 @@ mirrorsync_mirrors:
       - --include="pool/*-debug/***"
       - --exclude="*"
   repo:
-    hosts: "{{ groups['mirrors'] + ['build.archlinux.org'] }}"
+    hosts: "{{ (groups['mirrors'] + ['build.archlinux.org']) | difference(['repos.archlinux.org']) }}"
     source: rsync://rsync.archlinux.org/ftp_tier1
     target: /srv/ftp
     last_update_url: https://rsync.archlinux.org/lastupdate
@@ -28,6 +28,13 @@ mirrorsync_mirrors:
       - --exclude="/other"
       - --exclude="/sources"
       - --exclude="*-debug/"
+  kitchensink:
+    hosts: "repos.archlinux.org"
+    source: rsync://repos.archlinux.org/kitchensink_tier1
+    target: /srv/ftp
+    last_update_url: https://repos.archlinux.org/lastupdate
+    last_update_dst: lastupdate
+    save_lastsync: true
   riscv:
     hosts: "{{ groups['geo_mirrors'] }}"
     mirror_domain: riscv.mirror.pkgbuild.com

playbooks/repos.archlinux.org.yml

 - name: Setup repos.archlinux.org
   hosts: repos.archlinux.org
   remote_user: root
+  vars:
+    archweb_db_host: "{{ hostvars['archlinux.org']['wireguard_address'] }}"
   roles:
     - { role: common }
     - { role: tools }
@@ -10,7 +12,10 @@
     - { role: root_ssh }
     - { role: borg_client, tags: ['borg'] }
     - { role: sudo, tags: ['archusers'] }
+    # - { role: certbot }
+    - { role: nginx }
     - { role: fail2ban }
     - { role: mirrorsync }
     - { role: prometheus_exporters }
     - { role: promtail }
+    - { role: repo_archive_split_temp, repos_rsync_domain: "rsync.archlinux.org" }

roles/dbscripts/templates/rsyncd.conf.proto.j2

@@ -80,7 +80,7 @@ hosts deny = *
 [kitchensink_tier1]
 	path = /srv/ftp
 	comment = ftp area (everything, including very old versions)
-	hosts allow = @@ALLOWHOSTS_TIER1@@ {{ hostvars['gemini.archlinux.org']['ipv4_address'] }} {{ hostvars['gemini.archlinux.org']['ipv6_address'] }}
+	hosts allow = @@ALLOWHOSTS_TIER1@@ {{ hostvars['gemini.archlinux.org']['ipv4_address'] }} {{ hostvars['gemini.archlinux.org']['ipv6_address'] }} {{ hostvars['repos.archlinux.org']['ipv4_address'] }} {{ hostvars['repos.archlinux.org']['ipv6_address'] }}
 	max connections = 0
 
 [kitchensink_auth]

roles/repo_archive_split_temp/defaults/main.yml

+../../archweb/defaults/main.yml
\ No newline at end of file

roles/repo_archive_split_temp/files

+../dbscripts/files
\ No newline at end of file

roles/repo_archive_split_temp/tasks/main.yml

+- name: Install rsync and some perl stuff
+  pacman: name=rsync,perl-dbd-pg,perl-timedate state=present
+
+- name: Create ssl cert
+  include_role:
+    name: certificate
+  vars:
+    domains: ["{{ repos_rsync_domain }}"]
+
+- name: Make nginx log dir
+  file: path=/var/log/nginx/{{ repos_rsync_domain }} state=directory owner=root group=root mode=0755
+
+- name: Set up nginx
+  template: src=nginx.d.conf.j2 dest=/etc/nginx/nginx.d/dbscripts.conf owner=root group=root mode=0644
+  notify:
+    - Reload nginx
+  tags:
+    - nginx
+
+- name: Create rsyncd-conf-genscripts
+  file: path=/etc/rsyncd-conf-genscripts state=directory owner=root group=root mode=0700
+
+- name: Install rsync.conf.proto
+  template: src=rsyncd.conf.proto.j2 dest=/etc/rsyncd-conf-genscripts/rsyncd.conf.proto owner=root group=root mode=0644
+  tags:
+    - rsynd-conf-proto
+
+- name: Configure gen_rsyncd.conf.pl
+  template: src=gen_rsyncd.conf.pl dest=/etc/rsyncd-conf-genscripts/gen_rsyncd.conf.pl owner=root group=root mode=0700
+  no_log: true
+
+- name: Generate mirror config
+  command: /etc/rsyncd-conf-genscripts/gen_rsyncd.conf.pl
+  register: gen_rsyncd
+  changed_when: "gen_rsyncd.rc == 0"
+
+- name: Start and enable rsync
+  service: name=rsyncd.socket enabled=yes state=started
+
+- name: Open firewall holes for rsync
+  ansible.posix.firewalld: service=rsyncd permanent=true state=enabled immediate=yes
+  when: configure_firewall
+  tags:
+    - firewall
+
+- name: Install systemd timers
+  copy: src={{ item }} dest=/etc/systemd/system/{{ item }} owner=root group=root mode=0644
+  with_items:
+    - gen_rsyncd.timer
+    - gen_rsyncd.service
+
+- name: Activate systemd timers
+  service: name={{ item }} enabled=yes state=started
+  with_items:
+    - gen_rsyncd.timer

roles/repo_archive_split_temp/templates/gen_rsyncd.conf.pl

+../../dbscripts/templates/gen_rsyncd.conf.pl
\ No newline at end of file

roles/repo_archive_split_temp/templates/nginx.d.conf.j2

+server {
+    listen       80;
+    listen       [::]:80;
+    listen       443 ssl http2;
+    listen       [::]:443 ssl http2;
+    server_name  {{ repos_rsync_domain }};
+    root         /srv/ftp;
+
+    include snippets/letsencrypt.conf;
+
+    ssl_certificate      /etc/letsencrypt/live/{{ repos_rsync_domain }}/fullchain.pem;
+    ssl_certificate_key  /etc/letsencrypt/live/{{ repos_rsync_domain }}/privkey.pem;
+    ssl_trusted_certificate /etc/letsencrypt/live/{{ repos_rsync_domain }}/chain.pem;
+
+    access_log   /var/log/nginx/{{ repos_rsync_domain }}/access.log reduced;
+    access_log   /var/log/nginx/{{ repos_rsync_domain }}/access.log.json json_reduced;
+
+    location = /lastupdate {
+        allow all;
+    }
+
+    location / {
+        deny all;
+    }
+}

roles/repo_archive_split_temp/templates/rsyncd.conf.proto.j2

+../../dbscripts/templates/rsyncd.conf.proto.j2
\ No newline at end of file