# Search, citation, and training crawlers are welcome on marketing and help.
# Sitemap lists /, /start, /help, /demo, /privacy, /terms. /pricing is not a current page.
# AI crawler map: /llms.txt (points at /, /start.md, /help, /demo, /privacy, /terms).
# /start.md, /join/{token}, /a/{token}, /claim/{token}, and /agent may be fetched; join/agent/anchor/claim stay noindex via headers, not a robots Disallow.
# Quiet CLI binaries live at /ua/ (noindex, not in the sitemap or marketing nav).
# API paths are not for indexes.

User-agent: GPTBot
Allow: /
Disallow: /v1/
Disallow: /mcp
Disallow: /mcp/
Disallow: /api/
Disallow: /ua
Disallow: /ua/
Disallow: /*.dc.html
Disallow: /*.jsx

User-agent: ChatGPT-User
Allow: /
Disallow: /v1/
Disallow: /mcp
Disallow: /mcp/
Disallow: /api/
Disallow: /ua
Disallow: /ua/
Disallow: /*.dc.html
Disallow: /*.jsx

User-agent: OAI-SearchBot
Allow: /
Disallow: /v1/
Disallow: /mcp
Disallow: /mcp/
Disallow: /api/
Disallow: /ua
Disallow: /ua/
Disallow: /*.dc.html
Disallow: /*.jsx

User-agent: ClaudeBot
Allow: /
Disallow: /v1/
Disallow: /mcp
Disallow: /mcp/
Disallow: /api/
Disallow: /ua
Disallow: /ua/
Disallow: /*.dc.html
Disallow: /*.jsx

User-agent: Claude-User
Allow: /
Disallow: /v1/
Disallow: /mcp
Disallow: /mcp/
Disallow: /api/
Disallow: /ua
Disallow: /ua/
Disallow: /*.dc.html
Disallow: /*.jsx

User-agent: Claude-SearchBot
Allow: /
Disallow: /v1/
Disallow: /mcp
Disallow: /mcp/
Disallow: /api/
Disallow: /ua
Disallow: /ua/
Disallow: /*.dc.html
Disallow: /*.jsx

User-agent: PerplexityBot
Allow: /
Disallow: /v1/
Disallow: /mcp
Disallow: /mcp/
Disallow: /api/
Disallow: /ua
Disallow: /ua/
Disallow: /*.dc.html
Disallow: /*.jsx

User-agent: Google-Extended
Allow: /
Disallow: /v1/
Disallow: /mcp
Disallow: /mcp/
Disallow: /api/
Disallow: /ua
Disallow: /ua/
Disallow: /*.dc.html
Disallow: /*.jsx

User-agent: Applebot-Extended
Allow: /
Disallow: /v1/
Disallow: /mcp
Disallow: /mcp/
Disallow: /api/
Disallow: /ua
Disallow: /ua/
Disallow: /*.dc.html
Disallow: /*.jsx

User-agent: *
Content-Signal: search=yes,ai-input=yes,ai-train=yes,use=full
Allow: /
Allow: /help
Allow: /help/
Allow: /llms.txt
Allow: /help/llms.txt
Allow: /llms-full.txt
Allow: /help/llms-full.txt
Allow: /sitemap.xml
Disallow: /v1/
Disallow: /mcp
Disallow: /mcp/
Disallow: /api/
Disallow: /ua
Disallow: /ua/
Disallow: /*.dc.html
Disallow: /*.jsx

Sitemap: https://underarticles.com/sitemap.xml