From f276e10d676df842cadd8d327f226c7498f486ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Klinkovsk=C3=BD?= Date: Sun, 24 Dec 2023 21:08:02 +0100 Subject: [PATCH] archwiki: update robots.txt - The /index.php? rule disallows all URLs with this prefix, so using longer prefixes does not make sense. - Allow indexing of the Help: namespace which contains relevant text. - Keep the intention of disallowing special namespaces, including the File: alias for Image: and using the /title/ path and colon separator for the namespace prefix. --- roles/archwiki/files/robots.txt | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/roles/archwiki/files/robots.txt b/roles/archwiki/files/robots.txt index 4cc557d4..489957f5 100644 --- a/roles/archwiki/files/robots.txt +++ b/roles/archwiki/files/robots.txt @@ -1,11 +1,8 @@ User-agent: * Disallow: /index.php? -Disallow: /index.php?diff= -Disallow: /index.php?oldid= -Disallow: /index.php?curid= -Disallow: /index.php?title=Help -Disallow: /index.php?title=Image -Disallow: /index.php?title=MediaWiki -Disallow: /index.php?title=Special: -Disallow: /index.php?title=Template Disallow: /skins/ +Disallow: /title/File: +Disallow: /title/Image: +Disallow: /title/MediaWiki: +Disallow: /title/Special: +Disallow: /title/Template: