# CreatorDB — robots.txt
# All bots welcome, including AI/LLM crawlers used for citation in ChatGPT,
# Claude, Gemini, Perplexity, etc. Explicit Allows are documented for clarity
# even though "User-agent: *  Allow: /" already covers them.

User-agent: *
Allow: /
Disallow: /wp-admin/
Disallow: /wp-login.php

# ------------------------------------------------------------------
# Explicit allows for major AI / LLM crawlers (citation traffic)
# ------------------------------------------------------------------

# OpenAI — ChatGPT browse + training crawler
User-agent: GPTBot
Allow: /

# OpenAI — used when ChatGPT browses URLs on a user's behalf
User-agent: ChatGPT-User
Allow: /

# OpenAI — search index crawler
User-agent: OAI-SearchBot
Allow: /

# Anthropic — Claude.ai citation crawler
User-agent: ClaudeBot
Allow: /

# Anthropic — historical name
User-agent: anthropic-ai
Allow: /

# Anthropic — Claude.ai web user agent
User-agent: Claude-Web
Allow: /

# Google — Gemini / SGE generative answers
User-agent: Google-Extended
Allow: /

# Perplexity — search and answers
User-agent: PerplexityBot
Allow: /

# Perplexity — user-on-demand browser
User-agent: Perplexity-User
Allow: /

# Common Crawl — corpus used by many models
User-agent: CCBot
Allow: /

# Apple Intelligence / Apple AI
User-agent: Applebot-Extended
Allow: /

# Bytedance / Doubao
User-agent: Bytespider
Allow: /

# Meta / Facebook AI
User-agent: Meta-ExternalAgent
Allow: /
User-agent: FacebookBot
Allow: /

# Diffbot — used by many enterprise AI products
User-agent: Diffbot
Allow: /

# Cohere
User-agent: cohere-ai
Allow: /

# Mistral
User-agent: MistralAI-User
Allow: /

# DuckDuckGo Assist
User-agent: DuckAssistBot
Allow: /

# ------------------------------------------------------------------
# Sitemap + LLM discovery
# ------------------------------------------------------------------
Sitemap: https://creatordb.app/sitemap.xml
Sitemap: https://creatordb.app/sitemap-new-pages.xml