archwiki: add robots.txt for mediawiki

roles/archwiki/files/robots.txt

+User-agent: *
+Disallow: /index.php?diff=
+Disallow: /index.php?oldid=
+Disallow: /index.php?title=Help
+Disallow: /index.php?title=Image
+Disallow: /index.php?title=MediaWiki
+Disallow: /index.php?title=Special:
+Disallow: /index.php?title=Template
+Disallow: /skins/

roles/archwiki/tasks/main.yml

@@ -20,6 +20,9 @@
     - reload nginx
   tags: ['nginx']
 
+- name: configure robots.txt
+  copy: src=robots.txt dest="{{ archwiki_dir }}/robots.txt" owner=root group=root mode=0644
+
 - name: make nginx log dir
   file: path=/var/log/nginx/{{ archwiki_domain }} state=directory owner=root group=root mode=0755
 

roles/archwiki/templates/nginx.d.conf.j2

@@ -38,6 +38,10 @@ server {
         return 403;
     }
 
+    location /robots.txt {
+	alias {{ archwiki_dir }}/robots.txt;
+    }
+
     location ^~ /. {
         log_not_found off;
         deny all;