# robots.txt template: AI visibility policy
# Part of the Ditchi AI Visibility Kit.
#
# Before deploy: replace [SITE_URL], adjust the private paths (they appear
# three times on purpose: a bot uses only the rule group that names it, so a
# path listed under "User-agent: *" alone would not apply to GPTBot or
# OAI-SearchBot), and decide the training-crawler group below.
#
# robots.txt is a policy file, not a lock. Documented bots from OpenAI,
# Anthropic, Google, Perplexity, Microsoft and Apple honor it; bad actors
# ignore it. Anything private must also be enforced at the auth or WAF layer.
#
# Consecutive User-agent lines share one rule group.

# ---------- AI search indexers and user-triggered fetchers ----------
# These create AI visibility: citations and live answers. Keep allowed.
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: Claude-SearchBot
User-agent: Claude-User
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Googlebot
User-agent: Bingbot
User-agent: Applebot
# TODO: keep this list identical to the other two groups
Disallow: /app/
Disallow: /api/
Disallow: /auth/
Disallow: /account/
Disallow: /checkout/
Allow: /

# ---------- Training crawlers: a business decision ----------
# Allow when the product, brand or service is the revenue and you want models
# to know it natively. Blocking training has no effect on citations.
# Block (replace "Allow: /" with "Disallow: /" and remove the path lines)
# when the content itself is the revenue: publishers, paywalled data,
# original research. Mixed sites split by path instead.
User-agent: GPTBot
User-agent: ClaudeBot
User-agent: CCBot
User-agent: Google-Extended
User-agent: Applebot-Extended
User-agent: Meta-ExternalAgent
# TODO: keep this list identical to the other two groups
Disallow: /app/
Disallow: /api/
Disallow: /auth/
Disallow: /account/
Disallow: /checkout/
Allow: /

# ---------- Blocked: documented poor compliance ----------
User-agent: Bytespider
Disallow: /

# ---------- Everyone else ----------
User-agent: *
# TODO: keep this list identical to the other two groups
Disallow: /app/
Disallow: /api/
Disallow: /auth/
Disallow: /account/
Disallow: /checkout/
Allow: /

# TODO: the real sitemap URL
Sitemap: [SITE_URL]/sitemap.xml
