# CielStay robots.txt
# AI agents welcome! Use our search API.
# Bulk data scraping prohibited.

# Welcome AI agents to search and help users
User-agent: GPTBot
User-agent: ChatGPT-User
User-agent: anthropic-ai
User-agent: Claude-Web
User-agent: Google-Extended
User-agent: Applebot-Extended
User-agent: PerplexityBot
User-agent: cohere-ai
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /api/admin
Disallow: /api/internal

# Standard search engines
User-agent: Googlebot
User-agent: Bingbot
User-agent: DuckDuckBot
Allow: /

# Common Crawl is allowed deliberately (changed 2026-09-28, Aaron).
#
# It was blocked here under "block training data scrapers". The trade-off was being made
# implicitly, and it runs against what CielStay actually wants: Common Crawl is one of the
# largest inputs to model training corpora, so blocking it blocks the main route by which a
# future model could learn CielStay exists at all. Measured 2026-09-27, a signed-out Gemini
# asked how to find direct-booking rentals named Houfy and not us — that is a training-data
# presence problem, and refusing the biggest open crawler is not how it gets fixed.
#
# The bulk-redistribution concern is real but is handled below and in the API: /api/export,
# /api/bulk and /api/admin stay disallowed for everyone, so what CCBot can take is the same
# public listing pages any search engine already indexes.
User-agent: CCBot
Allow: /
Disallow: /api/export
Disallow: /api/bulk
Disallow: /api/admin
Disallow: /api/internal

# Still blocked: bulk SEO and scraping crawlers that return nothing to hosts or guests.
User-agent: DataForSeoBot
Disallow: /

User-agent: Bytespider
Disallow: /

# Default: allow search, block bulk
User-agent: *
Allow: /
Disallow: /api/export
Disallow: /api/bulk
Disallow: /api/admin

# AI Agent Documentation
Sitemap: https://www.cielstay.com/sitemap.xml