# ═══════════════════════════════════════════════════════════════════════
# kuraib.site — robots.txt
#
# Default: open to search engines and general crawlers.
# AI answer engines (ChatGPT, Claude, Perplexity, Copilot, Gemini) may
# crawl, index and cite this site's public pages. Crawlers that harvest
# content specifically to train AI models are disallowed below.
#
# robots.txt is a voluntary signal, not an access control: well-behaved
# crawlers honour it, but it cannot force compliance from any that don't.
# ═══════════════════════════════════════════════════════════════════════

User-agent: *
Allow: /

# ── AI search & answer engines: crawl, index and cite freely ────────────
# These power citations inside chat answers and AI-generated search
# results. None of the bots below are documented as training crawlers.

# OpenAI — ChatGPT search and citations
User-agent: OAI-SearchBot
Allow: /

# OpenAI — fetches a page when a user asks ChatGPT to read it
User-agent: ChatGPT-User
Allow: /

# Anthropic — indexes pages for Claude's search feature
User-agent: Claude-SearchBot
Allow: /

# Anthropic — fetches a page when a user asks Claude to read it
User-agent: Claude-User
Allow: /

# Perplexity — search index; Perplexity states this bot is not used
# to train its underlying models
User-agent: PerplexityBot
Allow: /

# Perplexity — live fetch when answering a specific user's question
User-agent: Perplexity-User
Allow: /

# Meta AI — live fetch on a user's behalf (link previews, chat answers)
User-agent: Meta-ExternalFetcher
Allow: /

# Googlebot, Bingbot, Applebot and other standard search crawlers are
# already covered by the "*" rule above, and are what actually power
# Google AI Overviews, Bing/Copilot answers and Siri search — blocking
# them would break ordinary search visibility, not just AI features.

# ── AI training crawlers: blocked from harvesting this site for models ──
# Blocking these does not affect the search/answer bots above; each
# vendor documents the two families as controlled independently.

# Common Crawl's dataset is reused by many labs to train AI models
User-agent: CCBot
Disallow: /

# ByteDance — collects content to train AI models
User-agent: Bytespider
Disallow: /

# Other web-scale crawlers commonly used to build AI training datasets
User-agent: Omgilibot
Disallow: /

User-agent: Omgili
Disallow: /

User-agent: Timpibot
Disallow: /

User-agent: ImagesiftBot
Disallow: /

User-agent: Diffbot
Disallow: /

User-agent: YouBot
Disallow: /

# Model-training crawlers of the major AI companies. Blocking these does not
# affect Google Search, AI Overviews, Bing or the AI search bots allowed above:
# each company runs search and training under separate user agents.

# OpenAI — model training
User-agent: GPTBot
Disallow: /

# Anthropic — model training
User-agent: ClaudeBot
Disallow: /

User-agent: anthropic-ai
Disallow: /

# Google — Gemini and Vertex AI training (not Google Search or AI Overviews)
User-agent: Google-Extended
Disallow: /

# Apple — Apple Intelligence training (not Applebot search)
User-agent: Applebot-Extended
Disallow: /

# Meta — AI training (link previews use facebookexternalhit, still allowed)
User-agent: meta-externalagent
Disallow: /

User-agent: FacebookBot
Disallow: /

# Cohere, Huawei, Allen AI and others — training datasets
User-agent: cohere-ai
Disallow: /

User-agent: cohere-training-data-crawler
Disallow: /

User-agent: PanguBot
Disallow: /

User-agent: AI2Bot
Disallow: /

User-agent: Ai2Bot-Dolma
Disallow: /

Sitemap: https://kuraib.site/sitemap.xml