# Compr — robots policy
#
# This is a private, authenticated SaaS. All meaningful endpoints are behind a
# session gate. We block AI TRAINING crawlers, archive crawlers and known
# scraping platforms, and we allow the AI agents that answer a user's question
# and link back.
#
# That split is deliberate (owner call 2026-08-14). This file is a request, not
# a control: a crawler chooses whether to honour it. So it cannot stop anyone
# from copying the site, because the parties worth worrying about ignore it and
# fetch the pages like any browser would. What it CAN do is turn away the
# crawlers that send customers. Blocking the search-and-cite agents was
# therefore all cost: it removed Compr from ChatGPT and Perplexity answers
# without protecting a single thing.
#
# What actually protects the product is elsewhere and unaffected by this file:
# the offer formula and its multipliers are server-side and never reach the
# browser, the client bundle is obfuscated at deploy, /app + /r/ + /api are
# session-gated, and the data (parcel rolls, comp-condition cache, per-market
# calibration) is not on any public page.
#
# To add another crawler later, append a `User-agent: <name>` line under the
# appropriate section. Getting the section right is the whole point: check
# whether the agent trains on what it fetches or answers a live question with it.

# ── Generative-AI TRAINING crawlers ─────────────────────────────────
# These ingest pages into model weights. Blocking costs no traffic.
# Google-Extended governs Gemini training only; Googlebot is allowed below, so
# ordinary Search ranking and AI Overviews are unaffected by this line.
User-agent: GPTBot
User-agent: Google-Extended
User-agent: anthropic-ai
User-agent: ClaudeBot
User-agent: Claude-Web
User-agent: CCBot
User-agent: cohere-ai
User-agent: cohere-training-data-crawler
User-agent: Applebot-Extended
User-agent: FacebookBot
User-agent: meta-externalagent
User-agent: Bytespider
User-agent: ImagesiftBot
User-agent: Diffbot
User-agent: omgilibot
User-agent: Omgili
User-agent: Webzio-Extended
User-agent: Timpibot
User-agent: Amazonbot
User-agent: PetalBot
User-agent: YouBot
User-agent: AI2Bot
User-agent: Google-CloudVertexBot
User-agent: MistralAI-User
User-agent: PanguBot
User-agent: iaskspider
User-agent: Kangaroo Bot
Disallow: /

# ── Known commercial scrapers / agent platforms ─────────────────────
User-agent: FirecrawlAgent
User-agent: firecrawl
User-agent: ScrapingBee
User-agent: ScraperAPI
User-agent: ScrapingBot
User-agent: Apify
User-agent: ApifyBot
User-agent: Bright Data
User-agent: BrightData
User-agent: SmartScraper
User-agent: Octoparse
User-agent: ParseHub
User-agent: DataForSEOBot
User-agent: SerpApi
User-agent: SemrushBot
User-agent: AhrefsBot
User-agent: MJ12bot
User-agent: dotbot
User-agent: BLEXBot
Disallow: /

# ── Archive crawlers (we don't want indefinite snapshots) ───────────
User-agent: ia_archiver
User-agent: archive.org_bot
Disallow: /

# ── AI SEARCH + CITATION agents: allowed on purpose ─────────────────
# These fetch a page to answer somebody's question now and link back. They are
# a referral channel, not a training pipeline, which is why they sit here and
# not in the block above. Same surface as any other search engine: the public
# marketing pages, never the app.
#
# OAI-SearchBot   ChatGPT's search index
# ChatGPT-User    a user asked ChatGPT to look at a page
# PerplexityBot   Perplexity's index
# Perplexity-User a user asked Perplexity to look at a page
# DuckAssistBot   DuckDuckGo's assistant answers
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: DuckAssistBot
Disallow: /api/
Disallow: /app
Disallow: /app.html
Disallow: /r/
Disallow: /v/
Disallow: /demo
Disallow: /demo.html
Disallow: /admin
Disallow: /login
Disallow: /login.html
Allow: /

# ── Default policy for everything else ──────────────────────────────
# Allow legit search-engine crawlers but disallow the actual app /
# API surface. Everything behind /app, /api, /r is auth-gated anyway.
User-agent: *
Disallow: /api/
Disallow: /app
Disallow: /app.html
Disallow: /r/
Disallow: /demo
Disallow: /demo.html
Disallow: /admin
Disallow: /login
Disallow: /login.html

# The "no public content to index" note that used to sit here is out of date:
# the homepage runs the free-comp funnel, and features / pricing / how-to /
# support all render in full signed out. Those are the pages in the sitemap.
Sitemap: https://thecompr.ai/sitemap.xml

# A plain-text summary for LLMs and agents (what Compr does, plans, how to
# call it) lives at https://thecompr.ai/llms.txt