From 45fc12c2f2483be1247808d90b628ff539d52ec7 Mon Sep 17 00:00:00 2001
From: surtur <a_mirre@utb.cz>
Date: Wed, 8 Jul 2020 15:02:41 +0200
Subject: [PATCH] feat: added robots.txt file for testing purposes

---
 static/robots.txt | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 static/robots.txt

diff --git a/static/robots.txt b/static/robots.txt
new file mode 100644
index 0000000..1979adb
--- /dev/null
+++ b/static/robots.txt
@@ -0,0 +1,10 @@
+User-agent: AdsBot*
+Disallow: /
+
+User-agent: Mediapartners*
+Disallow: /
+
+User-Agent: *
+Disallow: /404.html
+Disallow: /categories/
+Sitemap: https://dotya.ml/sitemap.xml