Disallow scraping the search results on the AUR

As the queries for searching are the most 'heavy' queries we can
disallow bots to scrape through all search results. Archweb uses the
same robots.txt rules for disallowing indexing search results.

roles/aurweb/files/robots.txt

+User-agent: *
+Disallow: /packages/?
+Disallow: /packages/?*
+Crawl-delay: 2

roles/aurweb/tasks/main.yml

@@ -141,6 +141,9 @@
 - name: make nginx log dir
   file: path=/var/log/nginx/{{ aurweb_domain }} state=directory owner=root group=root mode=0755
 
+- name: configure robots.txt
+  copy: src=robots.txt dest="{{ aurweb_dir }}/robots.txt" owner=aur group=http mode=0644
+
 - name: configure php-fpm
   template:
     src=php-fpm.conf.j2 dest="/etc/php/php-fpm.d/{{ aurweb_user }}.conf"

roles/aurweb/templates/nginx.d.conf.j2

@@ -40,6 +40,10 @@ server {
     root    {{ aurweb_dir }}/web/html;
     index   index.php;
 
+    location /robots.txt {
+        alias {{ aurweb_dir }}/robots.txt
+    }
+
     location ~ ^/trusted-user/ {
         break;
     }