# www.getmiravi.com
#
# Policy, and the reasoning, are published here:
#   https://www.getmiravi.com/blog/robots-txt-for-ai-engines
# If you change this file, change that post too.

# ---------------------------------------------------------------
# Retrieval crawlers: build the index AI search answers from.
# Always allowed. Blocking one of these removes us from that
# engine's answers, which is the opposite of the point.
# ---------------------------------------------------------------
User-agent: OAI-SearchBot
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: DuckAssistBot
Allow: /

# ---------------------------------------------------------------
# User-triggered fetches: a person asked a question and the
# assistant came to read the page to answer them. Never blocked.
# ---------------------------------------------------------------
User-agent: ChatGPT-User
Allow: /

User-agent: Perplexity-User
Allow: /

User-agent: Claude-User
Allow: /

# ---------------------------------------------------------------
# Training crawlers: allowed deliberately. They send no visitors
# and produce no citations, but being in the weights means a model
# can name us without fetching anything. For a company nobody has
# heard of yet, that trade is worth making. A publisher whose
# archive is the product should choose differently.
# ---------------------------------------------------------------
User-agent: GPTBot
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: CCBot
Allow: /

# ---------------------------------------------------------------
# ByteDance. Crawls aggressively, has a long record of ignoring
# this file, and there is no surface where our buyers ask it
# anything. A cost decision, not a rights position. Enforcing it
# would need an edge rule; this line is only a request.
# ---------------------------------------------------------------
User-agent: Bytespider
Disallow: /

User-agent: *
Allow: /

Sitemap: https://www.getmiravi.com/sitemap.xml