diff --git a/modules/weko-sitemap/weko_sitemap/config.py b/modules/weko-sitemap/weko_sitemap/config.py index f7a5df0cda..4233c932a4 100644 --- a/modules/weko-sitemap/weko_sitemap/config.py +++ b/modules/weko-sitemap/weko_sitemap/config.py @@ -36,6 +36,4 @@ SITEMAP_MAX_URL_COUNT = 10000 WEKO_SITEMAP__ROBOT_TXT = """ -User-agent: Bingbot -Crawl-delay: 30 """ diff --git a/modules/weko-sitemap/weko_sitemap/templates/weko_sitemap/robots.txt.tmpl b/modules/weko-sitemap/weko_sitemap/templates/weko_sitemap/robots.txt.tmpl index 015caae79d..a9b098d2c5 100644 --- a/modules/weko-sitemap/weko_sitemap/templates/weko_sitemap/robots.txt.tmpl +++ b/modules/weko-sitemap/weko_sitemap/templates/weko_sitemap/robots.txt.tmpl @@ -1,6 +1,27 @@ User-agent: * Disallow: /admin/ Disallow: /api/ +Disallow: /oai +Disallow: /search +Disallow: /items/ranking +Disallow: /items/export +Disallow: /login/ +Disallow: /shibauthorizer +Disallow: /Shibboleth.sso +Disallow: /shibboleth-sp +Disallow: /secure/ +Disallow: /weko/shib +Disallow: /ping +Disallow: /static +Disallow: /data +Disallow: /records/*/export/json +Disallow: /records/*/export/bibtex + +User-agent: Bingbot +Crawl-delay: 30 + +User-agent: PetalBot +Disallow: / {{ robot_txt }} diff --git a/scripts/instance.cfg b/scripts/instance.cfg index 90024af4a1..9e0d826e77 100644 --- a/scripts/instance.cfg +++ b/scripts/instance.cfg @@ -701,27 +701,6 @@ WEKO_AUTHORS_IMPORT_TMP_DIR='authors_import' # Robot txt WEKO_SITEMAP__ROBOT_TXT = """ -User-agent: Bingbot -Crawl-delay: 30 - -User-agent: * -Disallow: /admin/ -Disallow: /api/ -Disallow: /oai -Disallow: /search -Disallow: /items/ranking -Disallow: /items/export -Disallow: /login/ -Disallow: /shibauthorizer -Disallow: /Shibboleth.sso -Disallow: /shibboleth-sp -Disallow: /secure/ -Disallow: /weko/shib -Disallow: /ping -Disallow: /static -Disallow: /data -Disallow: /records/*/export/json -Disallow: /records/*/export/bibtex """ ENABLE_COOKIE_CONSENT = True