From a36dcd61d70b85ff194a549f8962db3dc0d72de9 Mon Sep 17 00:00:00 2001
From: Masaharu Hayashi <mhaya@nii.ac.jp>
Date: Thu, 26 Feb 2026 15:20:41 +0900
Subject: [PATCH 1/3] update robots.txt

---
 .../templates/weko_sitemap/robots.txt.tmpl    | 21 +++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/modules/weko-sitemap/weko_sitemap/templates/weko_sitemap/robots.txt.tmpl b/modules/weko-sitemap/weko_sitemap/templates/weko_sitemap/robots.txt.tmpl
index 015caae79d..a9b098d2c5 100644
--- a/modules/weko-sitemap/weko_sitemap/templates/weko_sitemap/robots.txt.tmpl
+++ b/modules/weko-sitemap/weko_sitemap/templates/weko_sitemap/robots.txt.tmpl
@@ -1,6 +1,27 @@
 User-agent: *
 Disallow: /admin/
 Disallow: /api/
+Disallow: /oai
+Disallow: /search
+Disallow: /items/ranking
+Disallow: /items/export
+Disallow: /login/
+Disallow: /shibauthorizer
+Disallow: /Shibboleth.sso
+Disallow: /shibboleth-sp
+Disallow: /secure/
+Disallow: /weko/shib
+Disallow: /ping
+Disallow: /static
+Disallow: /data
+Disallow: /records/*/export/json
+Disallow: /records/*/export/bibtex
+
+User-agent: Bingbot
+Crawl-delay: 30
+
+User-agent: PetalBot
+Disallow: /
 
 {{ robot_txt }}
 

From a73137744532f8fade2d4884fad0657abe5a6ffb Mon Sep 17 00:00:00 2001
From: Masaharu Hayashi <mhaya@nii.ac.jp>
Date: Fri, 27 Feb 2026 16:44:38 +0900
Subject: [PATCH 2/3] fix instance.cfg

---
 scripts/instance.cfg | 21 ---------------------
 1 file changed, 21 deletions(-)

diff --git a/scripts/instance.cfg b/scripts/instance.cfg
index 90024af4a1..9e0d826e77 100644
--- a/scripts/instance.cfg
+++ b/scripts/instance.cfg
@@ -701,27 +701,6 @@ WEKO_AUTHORS_IMPORT_TMP_DIR='authors_import'
 
 # Robot txt
 WEKO_SITEMAP__ROBOT_TXT = """
-User-agent: Bingbot
-Crawl-delay: 30
-
-User-agent: *
-Disallow: /admin/
-Disallow: /api/
-Disallow: /oai
-Disallow: /search
-Disallow: /items/ranking
-Disallow: /items/export
-Disallow: /login/
-Disallow: /shibauthorizer
-Disallow: /Shibboleth.sso
-Disallow: /shibboleth-sp
-Disallow: /secure/
-Disallow: /weko/shib
-Disallow: /ping
-Disallow: /static
-Disallow: /data
-Disallow: /records/*/export/json
-Disallow: /records/*/export/bibtex
 """
 
 ENABLE_COOKIE_CONSENT = True

From 462f9141d3785f0202130a78de0fa0c1081713ae Mon Sep 17 00:00:00 2001
From: Masaharu Hayashi <mhaya@nii.ac.jp>
Date: Fri, 27 Feb 2026 17:27:48 +0900
Subject: [PATCH 3/3] remote config.py

---
 modules/weko-sitemap/weko_sitemap/config.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/modules/weko-sitemap/weko_sitemap/config.py b/modules/weko-sitemap/weko_sitemap/config.py
index f7a5df0cda..4233c932a4 100644
--- a/modules/weko-sitemap/weko_sitemap/config.py
+++ b/modules/weko-sitemap/weko_sitemap/config.py
@@ -36,6 +36,4 @@
 SITEMAP_MAX_URL_COUNT = 10000
 
 WEKO_SITEMAP__ROBOT_TXT = """
-User-agent: Bingbot
-Crawl-delay: 30
 """