update robots.txt

Author: Masaharu Hayashi

modules/weko-sitemap/weko_sitemap/templates/weko_sitemap/robots.txt.tmpl

@@ -1,6 +1,27 @@
 User-agent: *
 Disallow: /admin/
 Disallow: /api/
+Disallow: /oai
+Disallow: /search
+Disallow: /items/ranking
+Disallow: /items/export
+Disallow: /login/
+Disallow: /shibauthorizer
+Disallow: /Shibboleth.sso
+Disallow: /shibboleth-sp
+Disallow: /secure/
+Disallow: /weko/shib
+Disallow: /ping
+Disallow: /static
+Disallow: /data
+Disallow: /records/*/export/json
+Disallow: /records/*/export/bibtex
+
+User-agent: Bingbot
+Crawl-delay: 30
+
+User-agent: PetalBot
+Disallow: /
 
 {{ robot_txt }}