# CostBench robots.txt
# We welcome AI assistants and search engines
# Content pages: /software/, /compare/, /best/, /stats/ — all fully crawlable
# Disallowed paths (universal):
#   /go/             — branded redirect handler, not a content page
#   /vendors/claim/  — vendor signup form, no citation value
#   /vendors/dashboard/ — vendor-only UI, no citation value
#   /vendors/pricing/ — vendor pricing surface, no citation value
#   /saved/          — user-state pages, noindex,nofollow
#   /compare/builder/ — interactive builder, noindex,nofollow
#   /compare/data/   — internal JSON data endpoints, zero citation value
#   /api/            — internal APIs

User-agent: *
Allow: /
Disallow: /go/
Disallow: /vendors/claim/
Disallow: /vendors/dashboard/
Disallow: /vendors/pricing/
Disallow: /saved/
Disallow: /compare/builder/
Disallow: /compare/data/
Disallow: /api/
Content-Signal: search=yes, ai-train=yes, ai-input=yes

# Major search engines - explicitly allowed to avoid ambiguity
User-agent: Googlebot
Allow: /
Disallow: /go/
Disallow: /vendors/claim/
Disallow: /vendors/dashboard/
Disallow: /vendors/pricing/
Disallow: /saved/
Disallow: /compare/builder/
Disallow: /compare/data/
Disallow: /api/

User-agent: Bingbot
Allow: /
Disallow: /go/
Disallow: /vendors/claim/
Disallow: /vendors/dashboard/
Disallow: /vendors/pricing/
Disallow: /saved/
Disallow: /compare/builder/
Disallow: /compare/data/
Disallow: /api/

User-agent: Slurp
Allow: /
Disallow: /go/
Disallow: /vendors/claim/
Disallow: /vendors/dashboard/
Disallow: /vendors/pricing/
Disallow: /saved/
Disallow: /compare/builder/
Disallow: /compare/data/
Disallow: /api/

User-agent: DuckDuckBot
Allow: /
Disallow: /go/
Disallow: /vendors/claim/
Disallow: /vendors/dashboard/
Disallow: /vendors/pricing/
Disallow: /saved/
Disallow: /compare/builder/
Disallow: /compare/data/
Disallow: /api/

# AI Assistants - Explicitly allowed for editorial content
# Vendor-side surfaces and internal endpoints disallowed to save crawl budget.
# Each section inherits nothing from *, so all disallows must be repeated.
User-agent: GPTBot
Allow: /
Disallow: /go/
Disallow: /vendors/claim/
Disallow: /vendors/dashboard/
Disallow: /vendors/pricing/
Disallow: /saved/
Disallow: /compare/builder/
Disallow: /compare/data/
Disallow: /api/

User-agent: ChatGPT-User
Allow: /
Disallow: /go/
Disallow: /vendors/claim/
Disallow: /vendors/dashboard/
Disallow: /vendors/pricing/
Disallow: /saved/
Disallow: /compare/builder/
Disallow: /compare/data/
Disallow: /api/

User-agent: OAI-SearchBot
Allow: /
Disallow: /go/
Disallow: /vendors/claim/
Disallow: /vendors/dashboard/
Disallow: /vendors/pricing/
Disallow: /saved/
Disallow: /compare/builder/
Disallow: /compare/data/
Disallow: /api/

User-agent: Claude-Web
Allow: /
Disallow: /go/
Disallow: /vendors/claim/
Disallow: /vendors/dashboard/
Disallow: /vendors/pricing/
Disallow: /saved/
Disallow: /compare/builder/
Disallow: /compare/data/
Disallow: /api/

User-agent: ClaudeBot
Allow: /
Disallow: /go/
Disallow: /vendors/claim/
Disallow: /vendors/dashboard/
Disallow: /vendors/pricing/
Disallow: /saved/
Disallow: /compare/builder/
Disallow: /compare/data/
Disallow: /api/

User-agent: anthropic-ai
Allow: /
Disallow: /go/
Disallow: /vendors/claim/
Disallow: /vendors/dashboard/
Disallow: /vendors/pricing/
Disallow: /saved/
Disallow: /compare/builder/
Disallow: /compare/data/
Disallow: /api/

User-agent: PerplexityBot
Allow: /
Disallow: /go/
Disallow: /vendors/claim/
Disallow: /vendors/dashboard/
Disallow: /vendors/pricing/
Disallow: /saved/
Disallow: /compare/builder/
Disallow: /compare/data/
Disallow: /api/

User-agent: Perplexity-User
Allow: /
Disallow: /go/
Disallow: /vendors/claim/
Disallow: /vendors/dashboard/
Disallow: /vendors/pricing/
Disallow: /saved/
Disallow: /compare/builder/
Disallow: /compare/data/
Disallow: /api/

User-agent: Amazonbot
Allow: /
Disallow: /go/
Disallow: /vendors/claim/
Disallow: /vendors/dashboard/
Disallow: /vendors/pricing/
Disallow: /saved/
Disallow: /compare/builder/
Disallow: /compare/data/
Disallow: /api/

User-agent: Bytespider
Allow: /
Disallow: /go/
Disallow: /vendors/claim/
Disallow: /vendors/dashboard/
Disallow: /vendors/pricing/
Disallow: /saved/
Disallow: /compare/builder/
Disallow: /compare/data/
Disallow: /api/

User-agent: Google-Extended
Allow: /
Disallow: /go/
Disallow: /vendors/claim/
Disallow: /vendors/dashboard/
Disallow: /vendors/pricing/
Disallow: /saved/
Disallow: /compare/builder/
Disallow: /compare/data/
Disallow: /api/

User-agent: Applebot-Extended
Allow: /
Disallow: /go/
Disallow: /vendors/claim/
Disallow: /vendors/dashboard/
Disallow: /vendors/pricing/
Disallow: /saved/
Disallow: /compare/builder/
Disallow: /compare/data/
Disallow: /api/

# Sitemap Index (split sitemaps for better crawling)
Sitemap: https://costbench.com/sitemap-index.xml

# LLM instructions
# See /llms.txt for AI-specific guidance on citing our data