Added robots.txt for archmanweb

roles/archmanweb/files/robots.txt

+User-agent: *
+Disallow: /search?
+Disallow: /listing?
+Crawl-delay: 2

roles/archmanweb/tasks/main.yml

@@ -65,6 +65,9 @@
   register: config
   no_log: true
 
+- name: copy robots.txt
+  copy: src=robots.txt dest="{{ archmanweb_dir }}/repo/robots.txt" owner=root group=root mode=0644
+
 - name: create archmanweb db user
   postgresql_user: name={{ archmanweb_db_user }} password={{ vault_archmanweb_db_password }} login_host="{{ archmanweb_db_host }}" login_password="{{ vault_postgres_users.postgres }}" encrypted=yes
   no_log: true

roles/archmanweb/templates/nginx.d.conf.j2

@@ -41,6 +41,10 @@ server {
         alias {{ archmanweb_dir }}/repo/collected_static/archlinux-common/favicon.ico;
     }
 
+    location = /robots.txt {
+        alias {{ archmanweb_dir }}/repo/robots.txt;
+    }
+
     # Client-cache for Django's static assets
     location /static/ {
         expires 30d;