dbscripts: Add rsync stuff

group_vars/all/postgres.yml

+$ANSIBLE_VAULT;1.1;AES256
+36666131373264613664663533333232343233306332663433303930333063643539656232353735
+3637393832313138323238646166336531363837313731630a393035643830626464316239326131
+66356665636131346639383733616631316664313331396538376663353832323133333134623833
+3038336638346361660a626637336430303036343631666365316339643732356362643133663363
+37316531383534343361393338616332323232376332323339366338663063643238666539646330
+63343466363930326238356334656636663363363166373561393131303435353461366237623233
+356262383565623262343836623666663232

roles/dbscripts/files/gen_rsyncd.service

+[Unit]
+Description=Generate rsync config for mirrors
+
+[Service]
+Type=oneshot
+User=root
+ExecStart=/etc/rsyncd-conf-genscripts/gen_rsyncd.conf.pl

roles/dbscripts/files/gen_rsyncd.timer

+[Unit]
+Description=Generate rsync config for mirrors
+
+[Timer]
+OnBootSec=15min
+OnUnitActiveSec=1h
+
+[Install]
+WantedBy=timers.target

roles/dbscripts/files/rsyncd.conf.proto

+# DO NOT CHANGE rsync.conf, CHANGE rsync.conf.proto INSTEAD! 
+# Hosts are managed by archweb, talk to someone that has permission to
+# play with mirrors to get new IP addresses added.
+
+use chroot = no
+max connections = 12
+lock file = /var/run/rsyncd/main.lock
+syslog facility = local5
+pid file = /var/run/rsyncd.pid
+#transfer logging = yes
+transfer logging = no
+motd file = /etc/rsyncd.motd
+timeout = 600
+
+# ALLOW ONLY TIERED MIRRORS
+# This effectively disables all sections but abs, *_tier1 and *_auth
+# We keep the configuration around in case we need to revert again
+hosts allow = 127.0.0.1
+
+# DENY THE REST
+hosts deny = *
+
+# ABS
+[abs]
+	path = /srv/abs/rsync
+	comment = ABS rsync
+	list = false
+	exclude = CVS .svn .~tmp~
+	hosts allow = *
+	read only = true
+	refuse options = c compress
+	dont compress = *
+	max connections = 50
+	transfer logging = no
+	lock file = /var/run/rsyncd/abs.lock
+
+# Just the release/stable iso/packages (for most mirrors)
+[ftp]
+	path = /srv/ftp
+	comment = ftp area (most mirrors should use this)
+	exclude = /archive/ /other/ /sources/
+
+[ftp_tier1]
+	path = /srv/ftp
+	comment = ftp area (most mirrors should use this)
+	exclude = /archive/ /other/ /sources/
+	hosts allow = @@ALLOWHOSTS_TIER1@@
+	max connections = 0
+
+[ftp_auth]
+	path = /srv/ftp
+	comment = ftp area, passworded (same as 'ftp')
+	exclude = /archive/ /other/ /sources/
+	hosts allow = *
+	auth users = *
+	secrets file = /etc/rsyncd.secrets
+	max connections = 0
+
+# The whole she-bang, except /sources
+[ftpfull]
+	path = /srv/ftp
+	comment = ftp area (everything, including very old versions, except sources)
+	exclude = /sources/
+
+[ftpfull_tier1]
+	path = /srv/ftp
+	comment = ftp area (everything, including very old versions, except sources)
+	exclude = /sources/
+	hosts allow = @@ALLOWHOSTS_TIER1@@
+	max connections = 0
+
+[ftpfull_auth]
+	path = /srv/ftp
+	comment = ftp area (everything, including very old versions, except sources)
+	exclude = /sources/
+	hosts allow = *
+	auth users = *
+	secrets file = /etc/rsyncd.secrets
+	max connections = 0
+
+# The whole she-bang
+[kitchensink]
+	path = /srv/ftp
+	comment = ftp area (everything, including very old versions)
+	hosts allow = 66.211.214.132
+
+[kitchensink_tier1]
+	path = /srv/ftp
+	comment = ftp area (everything, including very old versions)
+	hosts allow = @@ALLOWHOSTS_TIER1@@
+	max connections = 0
+
+[kitchensink_auth]
+	path = /srv/ftp
+	comment = ftp area (everything, including very old versions)
+	hosts allow = *
+	auth users = *
+	secrets file = /etc/rsyncd.secrets
+	max connections = 0
+
+# Special module for ftp.archlinux.org only, allows it to always get through
+[ftp-archlinux]
+	hosts allow = 209.85.41.143 209.85.41.144 209.85.41.145
+	max connections = 4
+	lock file = /var/run/rsyncd/archftp.lock
+	path = /srv/ftp
+	list = false
+	comment = ftp (priority) for ftp.archlinux.org
+	exclude = /archive/
+
+# Individual repositories
+[core]
+	path = /srv/ftp/core
+	comment = core repository
+
+[extra]
+	path = /srv/ftp/extra
+	comment = extra repository
+
+[community]
+	path = /srv/ftp/community
+	comment = community repository
+
+[testing]
+	path = /srv/ftp/testing
+	comment = testing repository
+
+[community-testing]
+	path = /srv/ftp/community-testing
+	comment = community-testing repository
+
+[communitysvn]
+	path = /srv/repos/svn-community/svn/
+	comment = community svn rep
+	hosts allow = 66.211.214.132
+
+[packagessvn]
+	path = /srv/repos/svn-packages/svn/
+	comment = community svn rep
+	hosts allow = 66.211.214.132

roles/dbscripts/tasks/main.yml

 ---
 
-- name: install svn and git
-  pacman: name=git,subversion state=present
+- name: install svn, git, rsync and some perl stuff
+  pacman: name=git,subversion,rsync,perl-dbd-pg state=present
 
 - name: create dbscripts users
   user: name="{{ item }}" shell=/sbin/nologin
@@ -120,6 +120,21 @@
 - name: symlink /packages to /srv/repos/svn-packages/dbscripts
   file: path=/packages src=/srv/repos/svn-packages state=link
 
+- name: create rsyncd-conf-genscripts
+  file: path=/etc/rsyncd-conf-genscripts state=directory owner=root group=root mode=700
+
+- name: install rsync.conf.proto
+  copy: src=rsyncd.conf.proto dest=/etc/rsyncd-conf-genscripts/rsyncd.conf.proto owner=root group=root mode=644
+
+- name: configure gen_rsyncd.conf.pl
+  template: src=gen_rsyncd.conf.pl dest=/etc/rsyncd-conf-genscripts/gen_rsyncd.conf.pl owner=root group=root mode=700
+
+- name: generate mirror config
+  command: /etc/rsyncd-conf-genscripts/gen_rsyncd.conf.pl
+
+- name: start rsync
+  service: name=rsyncd.socket enabled=yes state=started
+
 - name: install systemd timers
   copy: src={{ item }} dest=/etc/systemd/system/{{ item }} owner=root group=root mode=644
   with_items:
@@ -131,6 +146,8 @@
     - integrity-check.service
     - lastsync.timer
     - lastsync.service
+    - gen_rsyncd.timer
+    - gen_rsyncd.service
 
 - name: activate systemd timers
   service: name={{ item }} enabled=yes state=started
@@ -139,3 +156,4 @@
     - sourceballs.timer
     - integrity-check.timer
     - lastsync.timer
+    - gen_rsyncd.timer

roles/dbscripts/templates/gen_rsyncd.conf.pl

+#!/usr/bin/env perl
+
+use strict;
+use warnings;
+use DBI;
+use Data::Dumper;
+
+# TODO put these into credentials.ini and use Config::Simple to read it
+my $user = 'archweb_rsync';
+my $pass = '{{ postgres_users.archweb_rsync }}';
+my $db = 'DBI:Pg:dbname=archweb;host=gudrun.archlinux.org;sslmode=require';
+
+my $scriptdir="/etc/rsyncd-conf-genscripts";
+my $infile="$scriptdir/rsyncd.conf.proto";
+my $outfile="/etc/rsyncd.conf";
+
+my $query = 'SELECT mrs.ip FROM mirrors_mirrorrsync mrs LEFT JOIN mirrors_mirror m ON mrs.mirror_id = m.id WHERE tier = 1 ORDER BY ip';
+
+sub burp {
+	my ($file_name, @lines) = @_;
+	open (my $fh, ">", $file_name) || die sprintf(gettext("can't create '%s': %s"), $file_name, $!);
+	print $fh @lines;
+	close $fh;
+}
+
+my $dbh = DBI->connect($db, $user, $pass);
+my $sth = $dbh->prepare($query);
+
+$sth->execute;
+
+$sth->rows > 0 or die "Failed to fetch IPs";
+
+my @whitelist_ips;
+while (my @ipaddr = $sth->fetchrow_array) {
+	push @whitelist_ips, $ipaddr[0]
+}
+
+$dbh->disconnect;
+
+open (my $fh, "<", $infile) or die "Failed to open '$infile': $!";
+my @data = <$fh>;
+close $fh;
+
+my $tier1_whitelist = join " ", @whitelist_ips;
+for (@data) {
+	s|\@\@ALLOWHOSTS_TIER1@@|$tier1_whitelist|;
+}
+
+burp($outfile, @data);