diff --git a/www/robots.txt b/www/robots.txt new file mode 100644 index 0000000..8e22dfa --- /dev/null +++ b/www/robots.txt @@ -0,0 +1,4 @@ +User-agent: * +Disallow: /no-robots-here/ +Crawl-delay: 6 +Request-rate: 1/1