Enable robots.txt in docs.freebsd.org

Enable again robots.txt for docs.freebsd.org

PR:             253031
Submitted by:   wosch@

documentation/themes/beastie/layouts/robots.txt

@@ -0,0 +1,19 @@
+# $FreeBSD$
+
+User-agent: *
+#Crawl-delay: 10
+Disallow: /cgi/confirm-code.cgi
+Disallow: /cgi/cvsweb.cgi
+Disallow: /cgi/dosendpr.cgi
+Disallow: /cgi/getmsg.cgi
+Disallow: /cgi/mailindex.cgi
+Disallow: /cgi/man.cgi?apropos=2&
+Disallow: /cgi/mid.cgi
+Disallow: /cgi/mirror.cgi
+Disallow: /cgi/missing_handler.cgi
+Disallow: /cgi/ports.cgi
+Disallow: /cgi/query-pr-summary.cgi
+Disallow: /cgi/query-pr.cgi
+Disallow: /cgi/search.cgi
+Disallow: /cgi/url.cgi
+Disallow: /statistic