# galoy.io — crawler policy
# Galoy welcomes indexing by major search engines and citation by AI search providers.
# Each known AI bot is enumerated explicitly to signal intent.
# Default for unlisted user agents is allow.
#
# /data/ holds raw JSON datasets that feed site tools (e.g. the regulatory radar).
# They are excluded from search-engine crawling so they don't surface in Search
# Console as unindexable pages. AI retrieval bots remain allowed to read them.
#
# NOTE: because Googlebot, bingbot, etc. each have their own user-agent group
# below, a Disallow placed only under "User-agent: *" would NOT apply to them —
# each named group must carry its own Disallow line.
#
# Companion files at the docroot:
#   /sitemap.xml  — canonical URL list for crawlers
#   /llms.txt     — curated content map for AI retrieval (respected by Anthropic, Perplexity)

# ---- Default ----
User-agent: *
Allow: /
Disallow: /data/

# ---- Google ----
User-agent: Googlebot
Allow: /
Disallow: /data/

User-agent: Googlebot-Image
Allow: /
Disallow: /data/

User-agent: Googlebot-News
Allow: /
Disallow: /data/

# ---- Microsoft Bing / Copilot ----
User-agent: bingbot
Allow: /
Disallow: /data/

# ---- Anthropic (Claude) ----
User-agent: ClaudeBot
Allow: /
Crawl-delay: 1

User-agent: Claude-User
Allow: /

User-agent: Claude-SearchBot
Allow: /

# ---- OpenAI (ChatGPT) ----
User-agent: GPTBot
Allow: /
Crawl-delay: 1

User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

# ---- Perplexity ----
User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

# ---- Apple ----
User-agent: Applebot
Allow: /
Disallow: /data/

User-agent: Applebot-Extended
Allow: /

# ---- xAI (Grok) ----
User-agent: GrokBot
Allow: /

User-agent: xAI-Grok
Allow: /

User-agent: Grok-DeepSearch
Allow: /

# ---- Meta / DuckDuckGo / Yandex ----
User-agent: FacebookExternalHit
Allow: /

User-agent: Meta-ExternalAgent
Allow: /

User-agent: DuckDuckBot
Allow: /
Disallow: /data/

User-agent: YandexBot
Allow: /
Disallow: /data/

# ---- Sitemaps ----
Sitemap: https://galoy.io/sitemap.xml