Fall back to archive.org when not found for archive.*

For our archive.archlinux.org website instead of always redirecting to
archive.org try if we have the file and fallback redirecting to
archive.org. This reduces the load on archive.org and makes our
reproducible builds downloads faster then archive.org. This also set's
up another archive mirror on archive.ger.mirror.pkgbuild.com

Signed-off-by: Jelle van der Waa <jelle@archlinux.org>

host_vars/ger.mirror.pkgbuild.com

@@ -3,6 +3,7 @@ mirror_domain: ger.mirror.pkgbuild.com
 archweb_mirrorcheck_locations: [11]
 arch32_mirror_domain: ger.mirror.archlinux32.org
 network_interface: "eno2"
+archive_domain: archive.ger.mirror.pkgbuild.com
 ipv4_address: "159.122.70.8"
 ipv4_netmask: "/26"
 ipv4_gateway: "159.122.70.1"

playbooks/archive-mirrors.yml

@@ -12,3 +12,4 @@
     - { role: unbound }
     - { role: syncarchive, tags: ['nginx'] }
     - { role: zabbix-agent, tags: ["zabbix"] }
+    - { role: archive, archive_nginx_only: true, archive_dir: "/srv/archive"}

roles/archive/defaults/main.yml

@@ -4,3 +4,4 @@ archive_user_name: 'archive'
 archive_user_home: '/home/archive'
 archive_repo: '{{ archive_user_home }}/archive-uploader'
 archive_uploader_version: 'v0.15.3'
+archive_nginx_only: false

roles/archive/tasks/main.yml

@@ -10,6 +10,7 @@
     owner: archive
     group: archive
     mode: 0755
+  when: not archive_nginx_only
 
 - name: setup archive configuration
   template:
@@ -18,12 +19,15 @@
     owner: root
     group: root
     mode: 0644
+  when: not archive_nginx_only
 
 - name: setup archive timer
   systemd: name=archive.timer enabled=yes state=started
+  when: not archive_nginx_only
 
 - name: setup archive-hardlink timer
   systemd: name=archive-hardlink.timer enabled=yes state=started
+  when: not archive_nginx_only
 
 - name: set up nginx
   template:
@@ -46,19 +50,23 @@
 
 - name: install internet archive packages
   pacman: name=python-internetarchive state=present
+  when: not archive_nginx_only
 
 - name: create archive user
   user: name={{ archive_user_name }} shell=/bin/false home="{{ archive_user_home }}" createhome=yes
+  when: not archive_nginx_only
 
 - name: configure archive.org client
   command: ia configure --username={{ vault_archive_username }} --password={{ vault_archive_password }} creates={{ archive_user_home }}/.config/ia.ini
   become: yes
   become_user: "{{ archive_user_name }}"
+  when: not archive_nginx_only
 
 - name: clone archive uploader code
   git: repo=https://github.com/archlinux/arch-historical-archive.git dest="{{ archive_repo }}" version="{{ archive_uploader_version }}"
   become: yes
   become_user: "{{ archive_user_name }}"
+  when: not archive_nginx_only
 
 - name: install system service
   template: src={{ item }} dest=/etc/systemd/system/{{ item }} owner=root group=root mode=0644
@@ -67,6 +75,8 @@
     - archive-uploader.timer
   notify:
     - daemon reload
+  when: not archive_nginx_only
 
 - name: start uploader timer
   service: name=archive-uploader.timer state=started enabled=yes
+  when: not archive_nginx_only

roles/archive/templates/nginx.d.conf.j2

@@ -31,13 +31,15 @@ server {
     location /.all/ {
         # XXX: This regex is reused for /repos/ below! Change both if you change this!
         location ~ ^/.all/(?<filename>(?<pkgname>(?<pkgname_first_char>[^/])[^/]*)-(?<pkgver>[^-/]+)-(?<pkgrel>[0-9.]+)-(?<arch>[^-/]+)\.pkg\.tar(|\.(gz|bz2|xz|zst|lrz|lzo|Z|lz4|lz))(?<sig>\.sig)?)$ {
-            rewrite ^ /packages/$pkgname_first_char/$pkgname/$filename;
+            # https://stackoverflow.com/questions/22512112/nginx-rewrite-proxy-if-file-not-exists
+            try_files $uri @archive1;
         }
     }
 
     # XXX: This regex is the same as for .all above. Change both if you change this!
     location ~ ^/(?:repos/\d+/.*|packages/.*)/(?<filename>(?<pkgname>(?<pkgname_first_char>[^/])[^/]*)-(?<pkgver>[^-/]+)-(?<pkgrel>[0-9.]+)-(?<arch>[^-/]+)\.pkg\.tar(|\.(gz|bz2|xz|zst|lrz|lzo|Z|lz4|lz))(?<sig>\.sig)?)$ {
-        rewrite ^ /archive.org/archlinux_pkg_$pkgname/$filename last;
+        # https://stackoverflow.com/questions/22512112/nginx-rewrite-proxy-if-file-not-exists
+        try_files $uri @archive2;
     }
     # archive.org download URLs look like:
     # https://archive.org/download/archlinux_pkg_lucene__/lucene++-1.4.2-3-i686.pkg.tar.xz
@@ -54,6 +56,14 @@ server {
         rewrite ^/archive\.org/([^@\+\.]*/.*)$ https://archive.org/download/$1 redirect;
     }
 
+    location @archive1 {
+        rewrite ^ /packages/$pkgname_first_char/$pkgname/$filename;
+    }
+
+    location @archive2 {
+        rewrite ^ /archive.org/archlinux_pkg_$pkgname/$filename last;
+    }
+
     autoindex on;
     autoindex_exact_size off;
 }