# web3compliance.ai — crawling policy
#
# This site exists to be cited. The product strategy is answer-engine
# visibility: be the source an AI assistant quotes when someone asks what the
# rules are in a jurisdiction. So AI crawlers are welcomed explicitly rather
# than left to a default.
#
# 2026-08-21: Cloudflare's managed robots.txt was switched off, so this file is
# now the whole response. Before that it was inert — Cloudflare PREPENDED a
# block disallowing ClaudeBot, GPTBot, Google-Extended, meta-externalagent,
# Amazonbot, Applebot-Extended, Bytespider, CCBot and
# CloudflareBrowserRenderingCrawler outright and set ai-train=no, and a Disallow
# for a named agent beats a later "User-agent: *" group.
#
# If any "Disallow: /" for those agents reappears above this comment, that
# setting has been switched back on. To check and change it:
#   Cloudflare dashboard > the zone > Settings > filter "Bot traffic" >
#   "Manage your robots.txt" > must be "Disable robots.txt configuration"
#
# Note the neighbouring option is named "Set your preference to block training
# in robots.txt", which is what Cloudflare's own docs tell you to turn ON. Their
# guidance is written for sites that want AI kept out. We want the opposite.
#
# Blocking is a separate system from this file. Enforcement lives in
# AI Crawl Control (Search / Agent / Training), all three set to Allow. Both
# have to agree or the site says one thing and does another.

User-agent: *
Allow: /
Disallow: /admin/
# Content signals (https://contentsignals.org). "yes" grants the corresponding
# use: appear in search, be quoted in AI answers, and be learned from.
Content-Signal: search=yes, ai-input=yes, ai-train=yes

# Answer engines and AI retrieval — the crawlers that put us inside an answer.
User-agent: OAI-SearchBot
Allow: /
Disallow: /admin/

User-agent: ChatGPT-User
Allow: /
Disallow: /admin/

User-agent: PerplexityBot
Allow: /
Disallow: /admin/

User-agent: Perplexity-User
Allow: /
Disallow: /admin/

User-agent: Claude-User
Allow: /
Disallow: /admin/

User-agent: Claude-SearchBot
Allow: /
Disallow: /admin/

User-agent: Google-Extended
Allow: /
Disallow: /admin/

User-agent: Applebot-Extended
Allow: /
Disallow: /admin/

User-agent: meta-externalagent
Allow: /
Disallow: /admin/

# Model training — a compliance corpus is more useful inside the models than out.
User-agent: GPTBot
Allow: /
Disallow: /admin/

User-agent: ClaudeBot
Allow: /
Disallow: /admin/

User-agent: CCBot
Allow: /
Disallow: /admin/

User-agent: Amazonbot
Allow: /
Disallow: /admin/

User-agent: Bytespider
Allow: /
Disallow: /admin/

User-agent: cohere-ai
Allow: /
Disallow: /admin/

Sitemap: https://web3compliance.ai/sitemap-index.xml