diff --git a/roles/archmanweb/files/robots.txt b/roles/archmanweb/files/robots.txt new file mode 100644 index 0000000000000000000000000000000000000000..bd05d078b1a5b94cbeeb9532bca9fce65c73c992 --- /dev/null +++ b/roles/archmanweb/files/robots.txt @@ -0,0 +1,4 @@ +User-agent: * +Disallow: /search? +Disallow: /listing? +Crawl-delay: 2 diff --git a/roles/archmanweb/tasks/main.yml b/roles/archmanweb/tasks/main.yml index b4897d107a6d90f96a676396c85d8f1066745725..ed839eede21f529d82711c1702c75e28bd2a6541 100644 --- a/roles/archmanweb/tasks/main.yml +++ b/roles/archmanweb/tasks/main.yml @@ -65,6 +65,9 @@ register: config no_log: true +- name: copy robots.txt + copy: src=robots.txt dest="{{ archmanweb_dir }}/repo/robots.txt" owner=root group=root mode=0644 + - name: create archmanweb db user postgresql_user: name={{ archmanweb_db_user }} password={{ vault_archmanweb_db_password }} login_host="{{ archmanweb_db_host }}" login_password="{{ vault_postgres_users.postgres }}" encrypted=yes no_log: true diff --git a/roles/archmanweb/templates/nginx.d.conf.j2 b/roles/archmanweb/templates/nginx.d.conf.j2 index a85d2f576b3cd3af84e3c05be6168fb308e25cee..35058a181cde8edefe85d7f9e45aa185f7512eb0 100644 --- a/roles/archmanweb/templates/nginx.d.conf.j2 +++ b/roles/archmanweb/templates/nginx.d.conf.j2 @@ -41,6 +41,10 @@ server { alias {{ archmanweb_dir }}/repo/collected_static/archlinux-common/favicon.ico; } + location = /robots.txt { + alias {{ archmanweb_dir }}/repo/robots.txt; + } + # Client-cache for Django's static assets location /static/ { expires 30d;