# Fere AI — robots.txt
#
# Search engines and generative-AI crawlers share the same allow/disallow set:
#  - ALLOWED: /, /explore, /share/*, /strategies, /pricing, /compare,
#    /mcp, /llms.txt, /llms-full.txt, /index.md, /pricing.md, /compare.md,
#    /mcp.md, /vs/*, /vs/*.md, /structured-data, /.well-known/*
#  - /polymarket, /market-pulse and /discover are gated off for everyone
#    (DISABLED_PRODUCT_ROUTES) and redirect to /research — Disallow'd here.
#  - DISALLOWED: authenticated app surfaces (/app, /chat, …), /_next*, and the
#    per-user /strategies/ detail pages (bare /strategies stays crawlable)
#  - Content-Signal is omitted on purpose (not a registered robots.txt field;
#    removed in PR #1402). Crawler allow/disallow above is the access policy.
#  - Docs and blog live on subdomains and have their own robots.
#  - Sitemap index at /sitemap.xml (children under /sitemaps/*.xml).
#    Submit only /sitemap.xml in GSC.

User-agent: Googlebot
User-agent: Bingbot
User-agent: *
Disallow: /app
Disallow: /chat
Disallow: /api
Disallow: /_next
Disallow: /investement-agent
Disallow: /investment-agent
Disallow: /billing
Disallow: /wallet
Disallow: /yield
Disallow: /discover
Disallow: /polymarket
Disallow: /market-pulse
Disallow: /strategies/
Disallow: /portfolio
Disallow: /settings
Disallow: /points
Disallow: /activities
Disallow: /refer-and-earn
Disallow: /dev
Disallow: /_next
Disallow: /_next/static
Disallow: /_next/data

User-agent: GPTBot
User-agent: ChatGPT-User
User-agent: OAI-SearchBot
User-agent: PerplexityBot
User-agent: Claude-SearchBot
User-agent: ClaudeBot
User-agent: anthropic-ai
User-agent: Claude-Web
User-agent: Google-Extended
User-agent: Applebot-Extended
User-agent: CCBot
User-agent: Bytespider
User-agent: Meta-ExternalAgent
User-agent: Amazonbot
User-agent: Diffbot
User-agent: Omgilibot
User-agent: FacebookBot
User-agent: cohere-ai
Disallow: /app
Disallow: /chat
Disallow: /api
Disallow: /_next
Disallow: /investement-agent
Disallow: /investment-agent
Disallow: /billing
Disallow: /wallet
Disallow: /yield
Disallow: /discover
Disallow: /polymarket
Disallow: /market-pulse
Disallow: /strategies/
Disallow: /portfolio
Disallow: /settings
Disallow: /points
Disallow: /activities
Disallow: /refer-and-earn
Disallow: /dev
Disallow: /_next
Disallow: /_next/static
Disallow: /_next/data

# Main site — sitemap index (children: /sitemaps/core.xml, /sitemaps/shares-YYYY-MM.xml, …)
Sitemap: https://www.fereai.xyz/sitemap.xml
Sitemap: https://docs.fereai.xyz/sitemap.xml
Sitemap: https://blog.fereai.xyz/sitemap/sitemap.xml