# robots.txt for gutenberg.co.jp
# Last updated: 2026-02-13
# Maintained by: Gutenberg Corporation

# ============================================
# Default: All crawlers allowed
# ============================================
User-agent: *
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /preview/

# Sitemap declarations
Sitemap: https://gutenberg.co.jp/sitemap.xml
Sitemap: https://gutenberg.co.jp/sitemap-static.xml

# LLM information file
# https://gutenberg.co.jp/llms.txt

# Host directive
Host: https://gutenberg.co.jp

# ============================================
# Search Engine Crawlers
# ============================================
User-agent: Googlebot
Allow: /

User-agent: Googlebot-Image
Allow: /

User-agent: bingbot
Allow: /

User-agent: Yeti
Allow: /

# ============================================
# LLM / AI Crawlers - Permitted
# ============================================
User-agent: GPTBot
Allow: /
Crawl-delay: 5

User-agent: ChatGPT-User
Allow: /

User-agent: ClaudeBot
Allow: /
Crawl-delay: 5

User-agent: PerplexityBot
Allow: /
Crawl-delay: 5

User-agent: Google-Extended
Allow: /

# ============================================
# LLM / AI Crawlers - Blocked
# (Training-only crawlers with no direct user benefit)
# ============================================
User-agent: Bytespider
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Amazonbot
Disallow: /