# robots.txt for rupa.pro – updated 2025-07-31
# Goal: maximum crawl & index coverage for classic search + AI experiences
# ----------------------------------------------------------

# 1️⃣ Universal allow - welcome all crawlers
User-agent: *
Allow: /
Allow: /blog/wp-admin/admin-ajax.php
Disallow: /blog/wp-admin/
Disallow: /blog/wp-login.php
Disallow: /blog/_system/
Disallow: /blog/tag/

# 2️⃣ Explicit welcomes for major AI/generative search crawlers
# (already covered by wildcard above, but removes any ambiguity)
User-agent: GPTBot
Allow: /

User-agent: Google-Extended        # powers Google AI Overviews & Bard
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: OAI-SearchBot          # used by ChatGPT "Browse"
Allow: /

User-agent: CCBot                  # Common Crawl (used by many AI systems)
Allow: /

User-agent: anthropic-ai           # Anthropic's crawler
Allow: /

User-agent: Claude-Web             # Another potential Anthropic bot
Allow: /

# 3️⃣ Crawl delay (optional - uncomment if needed to prevent server overload)
# Crawl-delay: 1

Sitemap: https://rupa.pro/root-sitemap.xml
Sitemap: https://rupa.pro/blog/sitemap.xml