From 37f0dd08d29b3619051af17f2a58de714eaeb769 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thorben=20G=C3=BCnther?= <admin@xenrox.net>
Date: Fri, 9 Jul 2021 15:39:13 +0200
Subject: [PATCH] archweb: Add robots.txt
Closes #358
---
roles/archweb/files/robots.txt | 6 ++++++
roles/archweb/tasks/main.yml | 6 ++++++
2 files changed, 12 insertions(+)
create mode 100644 roles/archweb/files/robots.txt
diff --git a/roles/archweb/files/robots.txt b/roles/archweb/files/robots.txt
new file mode 100644
index 000000000..75f58ebd3
--- /dev/null
+++ b/roles/archweb/files/robots.txt
@@ -0,0 +1,6 @@
+User-agent: *
+Disallow: /packages/search/
+Disallow: /packages/?
+Disallow: /packages/?*
+Sitemap: https://www.archlinux.org/sitemap.xml
+Crawl-delay: 2
diff --git a/roles/archweb/tasks/main.yml b/roles/archweb/tasks/main.yml
index c90027baa..7e058e6e0 100644
--- a/roles/archweb/tasks/main.yml
+++ b/roles/archweb/tasks/main.yml
@@ -73,6 +73,12 @@
- name: fix home permissions
file: state=directory owner=archweb group=archweb mode=0755 path="{{ archweb_dir }}"
+- name: make archlinux.org dir
+ file: path="{{ archweb_dir }}/archlinux.org" state=directory owner=archweb group=archweb mode=0755
+
+- name: configure robots.txt
+ copy: src=robots.txt dest="{{ archweb_dir }}/archlinux.org/robots.txt" owner=root group=root mode=0644
+
- name: configure archweb
template: src=local_settings.py.j2 dest={{ archweb_dir }}/local_settings.py owner=archweb group=archweb mode=0660
register: config
--
GitLab