# SOURCE OF TRUTH: this static file is authoritative. Cloudflare's "Manage your
# robots.txt" feature (Content Signals Policy / Instruct AI bots) is intentionally
# set to "Disable robots.txt configuration" so it does not override or duplicate
# what is defined here. Do not re-enable it; edit this file instead.
#
# Content Signals declare how this site's content may be used, separate from
# crawl access (Allow/Disallow). We permit AI search and answer-engine citation
# (search, ai-input) but decline AI training (ai-train). See contentsignals.org.
# The directive is repeated per bot group because a crawler obeys only its most
# specific matching group, so a signal under "*" alone would not reach the
# named AI crawlers below.

User-agent: *
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /
# Public, read-only data feeds stay crawlable; the private analytics functions
# (/api/event, /api/stats) remain blocked by the /api/ disallow below. Longest-match
# Allow wins for the two explicit paths.
Allow: /api/destinations.json
Allow: /api/comparisons.json
Disallow: /search/
Disallow: /404/
Disallow: /admin/
Disallow: /api/
Disallow: /cdn-cgi/

# AI answer-engine and search crawlers are explicitly allowed to fetch the site.
# This is a verified reference source for airline baggage, cruise lines,
# airports, and packing lists. Citations back to travelvient.com are welcome.
# Per the Content-Signal above, training use is declined while citation is not.
# Each group restates the "*" path rules because a crawler that matches a named
# group ignores "*" entirely (RFC 9309). Keep these rule sets identical to "*".
User-agent: GPTBot
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /
Allow: /api/destinations.json
Allow: /api/comparisons.json
Disallow: /search/
Disallow: /404/
Disallow: /admin/
Disallow: /api/
Disallow: /cdn-cgi/

User-agent: ChatGPT-User
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /
Allow: /api/destinations.json
Allow: /api/comparisons.json
Disallow: /search/
Disallow: /404/
Disallow: /admin/
Disallow: /api/
Disallow: /cdn-cgi/

User-agent: OAI-SearchBot
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /
Allow: /api/destinations.json
Allow: /api/comparisons.json
Disallow: /search/
Disallow: /404/
Disallow: /admin/
Disallow: /api/
Disallow: /cdn-cgi/

User-agent: ClaudeBot
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /
Allow: /api/destinations.json
Allow: /api/comparisons.json
Disallow: /search/
Disallow: /404/
Disallow: /admin/
Disallow: /api/
Disallow: /cdn-cgi/

User-agent: Claude-User
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /
Allow: /api/destinations.json
Allow: /api/comparisons.json
Disallow: /search/
Disallow: /404/
Disallow: /admin/
Disallow: /api/
Disallow: /cdn-cgi/

User-agent: Claude-SearchBot
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /
Allow: /api/destinations.json
Allow: /api/comparisons.json
Disallow: /search/
Disallow: /404/
Disallow: /admin/
Disallow: /api/
Disallow: /cdn-cgi/

User-agent: Claude-Web
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /
Allow: /api/destinations.json
Allow: /api/comparisons.json
Disallow: /search/
Disallow: /404/
Disallow: /admin/
Disallow: /api/
Disallow: /cdn-cgi/

User-agent: anthropic-ai
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /
Allow: /api/destinations.json
Allow: /api/comparisons.json
Disallow: /search/
Disallow: /404/
Disallow: /admin/
Disallow: /api/
Disallow: /cdn-cgi/

User-agent: PerplexityBot
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /
Allow: /api/destinations.json
Allow: /api/comparisons.json
Disallow: /search/
Disallow: /404/
Disallow: /admin/
Disallow: /api/
Disallow: /cdn-cgi/

User-agent: Perplexity-User
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /
Allow: /api/destinations.json
Allow: /api/comparisons.json
Disallow: /search/
Disallow: /404/
Disallow: /admin/
Disallow: /api/
Disallow: /cdn-cgi/

User-agent: Google-Extended
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /
Allow: /api/destinations.json
Allow: /api/comparisons.json
Disallow: /search/
Disallow: /404/
Disallow: /admin/
Disallow: /api/
Disallow: /cdn-cgi/

User-agent: Meta-ExternalAgent
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /
Allow: /api/destinations.json
Allow: /api/comparisons.json
Disallow: /search/
Disallow: /404/
Disallow: /admin/
Disallow: /api/
Disallow: /cdn-cgi/

User-agent: DuckAssistBot
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /
Allow: /api/destinations.json
Allow: /api/comparisons.json
Disallow: /search/
Disallow: /404/
Disallow: /admin/
Disallow: /api/
Disallow: /cdn-cgi/

# Training-only crawlers are blocked outright. CCBot (Common Crawl), Bytespider
# (ByteDance), Applebot-Extended (Apple's training opt-out token, it never crawls)
# and cohere-ai feed model-training corpora only and run no answer engine that
# cites back to the site, so blocking them costs no citations and makes crawl
# access match ai-train=no. Apple search (Applebot) still follows "*".
User-agent: CCBot
Content-Signal: search=yes, ai-input=yes, ai-train=no
Disallow: /

User-agent: Applebot-Extended
Content-Signal: search=yes, ai-input=yes, ai-train=no
Disallow: /

User-agent: Bytespider
Content-Signal: search=yes, ai-input=yes, ai-train=no
Disallow: /

User-agent: cohere-ai
Content-Signal: search=yes, ai-input=yes, ai-train=no
Disallow: /

Sitemap: https://travelvient.com/sitemap-index.xml
Sitemap: https://travelvient.com/sitemap-images.xml