feat: aurweb dev box

* playbook for aur dev machine ("review apps")
* refactor aurweb role variables
* root_ssh: install additional ssh keys

script for running the playbook:
https://archlinux.rocks/share/files/aurweb-dev-provision



Signed-off-by: moson <moson@archlinux.org>

playbooks/aur-dev.archlinux.org.yml

+- name: Setup aur development host
+  hosts: '{{ aurdev_fqdn|default("none") }}'
+  remote_user: root
+  roles:
+    - { role: common }
+    - { role: firewalld }
+    - { role: sshd, sshd_enable_includes: true }
+    - { role: root_ssh, root_additional_keys: ["moson.pub"] }
+    - { role: certbot }
+    - { role: nginx }
+    - { role: mariadb, mariadb_innodb_buffer_pool_size: '1G' }
+    - { role: sudo }
+    - { role: redis }
+    - { role: uwsgi }
+    - { role: aurweb, aurweb_domain: "{{ aurdev_fqdn }}", is_prod: false }
+    - { role: fail2ban }
+
+  post_tasks:
+    - name: Install packages for dummy data generation
+      pacman:
+        state: present
+        name:
+          - words
+          - fortune-mod
+
+    - name: Create data dir
+      file:
+        path: "{{ aurweb_dir }}/data"
+        state: directory
+        mode: "0755"
+      become: true
+      become_user: "{{ aurweb_user }}"
+
+    - name: Generate dummy data
+      command: poetry run schema/gendummydata.py data/dummy.sql
+      register: generated_data
+      args:
+        chdir: "{{ aurweb_dir }}"
+        creates: "{{ aurweb_dir }}/data/dummy.sql"
+      become: true
+      become_user: "{{ aurweb_user }}"
+
+    - name: Import dummy data
+      mysql_db:
+        name: "{{ aurweb_db }}"
+        login_host: "{{ aurweb_db_host }}"
+        login_password: "{{ vault_mariadb_users.root }}"
+        state: import
+        target: "{{ aurweb_dir }}/data/dummy.sql"
+      when: generated_data.changed

playbooks/aur.archlinux.org.yml

@@ -16,5 +16,5 @@
     - { role: borg_client, tags: ["borg"] }
     - { role: postfix_null }
     - { role: fail2ban }
-    - { role: aurweb }
+    - { role: aurweb, is_prod=true }
     - { role: wireguard }

roles/aurweb/defaults/main.yml

@@ -26,3 +26,29 @@ aurweb_window_length: '86400'
 aurweb_memcached_memory: 2048
 
 aurweb_workers: 4
+
+# aurweb timers and services are installed and restarted by default,
+# unless .install and .restart are explicitly set to "false"
+aurweb_timers:
+  - { name: "aurweb-git.timer" }
+  - { name: "aurweb-git-archive.timer" }
+  - { name: "aurweb-aurblup.timer" }
+  - { name: "aurweb-mkpkglists.timer" }
+  - { name: "aurweb-pkgmaint.timer" }
+  - { name: "aurweb-popupdate.timer" }
+  - { name: "aurweb-tuvotereminder.timer" }
+  - { name: "aurweb-usermaint.timer" }
+  - { name: "aurweb-github-mirror.timer", install: "{{ is_prod }}", restart: "{{ is_prod }}" }
+
+aurweb_services:
+  - { name: "aurweb-git.service", restart: false }
+  - { name: "aurweb-git-archive.service", restart: false }
+  - { name: "aurweb-aurblup.service", restart: false }
+  - { name: "aurweb-mkpkglists.service", restart: false }
+  - { name: "aurweb-pkgmaint.service", restart: false }
+  - { name: "aurweb-popupdate.service", restart: false }
+  - { name: "aurweb-tuvotereminder.service", restart: false }
+  - { name: "aurweb-usermaint.service", restart: false }
+  - { name: "aurweb-github-mirror.service", install: "{{ is_prod }}", restart: "{{ is_prod }}" }
+  - { name: "aurweb.service" }
+  - { name: "goaurrpc.service", install: false }

roles/aurweb/tasks/main.yml

@@ -32,22 +32,22 @@
   user: name="{{ aurweb_user }}" shell=/bin/bash createhome=yes
   register: aur_user
 
-- name: Create .ssh for the aur user
-  file: path={{ aur_user.home }}/.ssh state=directory owner={{ aur_user.name }} group={{ aur_user.name }} mode=0700
-
-- name: Install SSH key for mirroring to GitHub
-  copy: src=id_ed25519.vault dest={{ aur_user.home }}/.ssh/id_ed25519 owner={{ aur_user.name }} group={{ aur_user.name }} mode=0600
-
-- name: Fetch host keys for github.com
-  command: ssh-keyscan github.com
-  args:
-    creates: "{{ aur_user.home }}/.ssh/known_hosts"
-  register: github_host_keys
-
-- name: Write github.com host keys to the aur user's known_hosts
-  lineinfile: name={{ aur_user.home }}/.ssh/known_hosts create=yes line={{ item }} owner={{ aur_user.name }} group={{ aur_user.name }} mode=0644
-  loop: "{{ github_host_keys.stdout_lines }}"
-  when: github_host_keys.changed
+- name: Github SSH configuration tasks
+  when: is_prod
+  block:
+    - name: Install SSH key for mirroring to GitHub
+      copy: src=id_ed25519.vault dest={{ aur_user.home }}/.ssh/id_ed25519 owner={{ aur_user.name }} group={{ aur_user.name }} mode=0600
+
+    - name: Fetch host keys for github.com
+      command: ssh-keyscan github.com
+      args:
+        creates: "{{ aur_user.home }}/.ssh/known_hosts"
+      register: github_host_keys
+
+    - name: Write github.com host keys to the aur user's known_hosts
+      lineinfile: name={{ aur_user.home }}/.ssh/known_hosts create=yes line={{ item }} owner={{ aur_user.name }} group={{ aur_user.name }} mode=0644
+      loop: "{{ github_host_keys.stdout_lines }}"
+      when: github_host_keys.changed
 
 - name: Create directory
   file: path={{ aurweb_dir }} state=directory owner={{ aurweb_user }} group=http mode=0775
@@ -73,46 +73,20 @@
   check_mode: true
 
 - name: Install AUR systemd service and timers
-  template: src={{ item }}.j2 dest=/etc/systemd/system/{{ item }} owner=root group=root mode=0644
+  template: src={{ item.name }}.j2 dest=/etc/systemd/system/{{ item.name }} owner=root group=root mode=0644
   with_items:
-    - aurweb-git.service
-    - aurweb-git.timer
-    - aurweb-git-archive.service
-    - aurweb-git-archive.timer
-    - aurweb-aurblup.service
-    - aurweb-aurblup.timer
-    - aurweb-mkpkglists.service
-    - aurweb-mkpkglists.timer
-    - aurweb-pkgmaint.service
-    - aurweb-pkgmaint.timer
-    - aurweb-popupdate.service
-    - aurweb-popupdate.timer
-    - aurweb-tuvotereminder.service
-    - aurweb-tuvotereminder.timer
-    - aurweb-usermaint.service
-    - aurweb-usermaint.timer
-    - aurweb.service
-    - aurweb-github-mirror.service
-    - aurweb-github-mirror.timer
-  when: release.changed
+    - "{{ aurweb_services }}"
+    - "{{ aurweb_timers }}"
+  when: release.changed and (item.install is not defined or item.install)
 
 - name: Stop AUR systemd services and timers
-  service: name={{ item }} enabled=yes state=stopped
+  service: name={{ item.name }} enabled=yes state=stopped
   with_items:
-    - aurweb-git.timer
-    - aurweb-git-archive.timer
-    - aurweb-aurblup.timer
-    - aurweb-mkpkglists.timer
-    - aurweb-pkgmaint.timer
-    - aurweb-popupdate.timer
-    - aurweb-tuvotereminder.timer
-    - aurweb-usermaint.timer
-    - aurweb.service
-    - goaurrpc.service
-    - aurweb-github-mirror.timer
-  when: release.changed
+    - "{{ aurweb_services }}"
+    - "{{ aurweb_timers }}"
+  when: release.changed and (item.restart is not defined or item.restart)
 
-- name: Clone aurweb repo
+- name: Clone aurweb repo (prod)
   git: >
     repo={{ aurweb_repository }}
     dest="{{ aurweb_dir }}"
@@ -121,7 +95,16 @@
     gpg_whitelist='{{ aurweb_pgp_keys }}'
   become: true
   become_user: "{{ aurweb_user }}"
-  when: release.changed
+  when: release.changed and is_prod
+
+- name: Clone aurweb repo (dev)
+  git: >
+    repo={{ aurweb_repository }}
+    dest="{{ aurweb_dir }}"
+    version={{ aurweb_version }}
+  become: true
+  become_user: "{{ aurweb_user }}"
+  when: release.changed and not is_prod
 
 - name: Create necessary directories
   file: path={{ aurweb_dir }}/{{ item }} state=directory owner={{ aurweb_user }} group={{ aurweb_user }} mode=0755
@@ -314,17 +297,8 @@
     - Restart sshd
 
 - name: Start and enable AUR systemd services and timers
-  systemd: name={{ item }} enabled=yes state=started daemon_reload=yes
+  systemd: name={{ item.name }} enabled=yes state=started daemon_reload=yes
   with_items:
-    - aurweb-git.timer
-    - aurweb-git-archive.timer
-    - aurweb-aurblup.timer
-    - aurweb-mkpkglists.timer
-    - aurweb-pkgmaint.timer
-    - aurweb-popupdate.timer
-    - aurweb-tuvotereminder.timer
-    - aurweb-usermaint.timer
-    - aurweb.service
-    - goaurrpc.service
-    - aurweb-github-mirror.timer
-  when: release.changed
+    - "{{ aurweb_services }}"
+    - "{{ aurweb_timers }}"
+  when: release.changed and (item.restart is not defined or item.restart)

roles/aurweb/templates/config.j2

@@ -63,6 +63,9 @@ pkgnames-repo = {{ aurweb_dir }}/pkgnames.git
 
 [notifications]
 notify-cmd = aurweb-notify
+{% if not is_prod %}
+sendmail = {{ aurweb_dir }}/util/sendmail
+{% endif %}
 {# Gitlab project and token used for traceback reports. #}
 gitlab-instance = {{ vault_aurweb_gitlab_instance }}
 error-project = {{ vault_aurweb_error_project }}

roles/root_ssh/templates/authorized_keys.j2

@@ -9,3 +9,8 @@
 		{% endif %}
 	{% endif %}
 {% endfor %}
+{% if root_additional_keys is defined %}
+	{% for pubkey in root_additional_keys -%}
+		{{ lookup('file', role_path + '/../../pubkeys/' + pubkey ) }}
+	{% endfor %}
+{% endif %}