aurweb: Create robots.txt

This should i.e. forbid crawlers to index all of the git diffs which
put's unneccessary load on the server and is not really of benefit to be
indexed anyways.

Link: #610


Reviewed-by: Sven-Hendrik Haase <svenstaro@gmail.com>
Reviewed-by: Levente Polyak <anthraxx@archlinux.org>
Signed-off-by: Christian Heusel <christian@heusel.eu>

roles/aurweb/files/robots.txt

+User-agent: *
+# block the search page from indexing, as the search is done via URL parameters
+Disallow: /packages?*
+# block all interactive things from being indexed, such as posting requests
+Disallow: /pkgbase/*
+# block all account pages from being indexed, as they require login anyways
+Disallow: /account/*
+# block the cgit interface except for the useful things
+Disallow: /cgit/aur.git/*
+Allow: /cgit/aur.git/tree
+Allow: /cgit/aur.git/log
+Crawl-delay: 2

roles/aurweb/tasks/main.yml

@@ -109,6 +109,9 @@
 - name: Copy aurweb configuration file
   copy: src={{ aurweb_dir }}/conf/config.defaults dest={{ aurweb_conf_dir }}/config.defaults remote_src=yes owner=root group=root mode=0644
 
+- name: Configure robots.txt
+  copy: src=robots.txt dest="{{ aurweb_dir }}/robots.txt" owner=root group=root mode=0644
+
 - name: Install goaurrpc configuration
   template: src=goaurrpc.conf.j2 dest=/etc/goaurrpc.conf owner=root group=root mode=0644
 

roles/aurweb/templates/nginx.d.conf.j2

@@ -51,6 +51,10 @@ server {
     root    {{ aurweb_dir }}/static;
     index   index.php;
 
+    location = /robots.txt {
+        alias {{ aurweb_dir }}/robots.txt;
+    }
+
     # redirect /tu to /package-maintainer for external links
     location ~ ^/tu($|/.*) {
         return 301 https://aur.archlinux.org/package-maintainer$1;