# robots.txt for SpaceOdysseyHub
# Strategy: invite AI search/answer crawlers (they cite + link back);
# block pure-training harvesters and SEO-only scrapers (zero referral).
#
# Content-Signal directives (contentsignals.org / EU Article 4):
#   search=yes    — search engines may index
#   ai-train=no   — AI providers may NOT train models on this content
#   ai-input=yes  — AI providers MAY use this content as RAG/inference input
#                   (this is what powers cited answers — we want it)

# Search & answer engines — invited (cite us, send referral traffic)
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Claude-User
User-agent: Claude-SearchBot
User-agent: ClaudeBot
User-agent: Googlebot
User-agent: Googlebot-News
User-agent: Bingbot
User-agent: Applebot
User-agent: DuckDuckBot
User-agent: YandexBot
Content-Signal: search=yes, ai-train=no, ai-input=yes
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /api/articles
Allow: /api/related-articles
Allow: /api/space-data
Allow: /api/space-news
Allow: /api/space-weather
Allow: /api/iss-crew
Allow: /api/launches
Allow: /api/sky-tonight
Allow: /api/search-index
Allow: /api/markdown
Allow: /.well-known/api-catalog
Allow: /rss.xml
Allow: /sitemap.xml
Allow: /news-sitemap.xml
Disallow: /admin/
Disallow: /api/admin/
Disallow: /api/auth/
Disallow: /search
Disallow: /api/badges
Disallow: /api/reactions
Disallow: /compare
Disallow: /compare/*
Disallow: /api/compare/*
Disallow: /moon/compare?*
Disallow: /companies/compare?*
Disallow: /launches/compare?*
Crawl-delay: 1

# Pure-training / bulk-harvest crawlers — fully disallowed (take, never cite)
User-agent: GPTBot
User-agent: Google-Extended
User-agent: Applebot-Extended
User-agent: anthropic-ai
User-agent: Claude-Web
User-agent: CCBot
User-agent: Bytespider
User-agent: Amazonbot
User-agent: meta-externalagent
User-agent: FacebookBot
Disallow: /

# SEO/data scrapers — fully disallowed (no user value, only competitive harm)
User-agent: AhrefsBot
User-agent: SemrushBot
User-agent: MJ12bot
User-agent: DotBot
User-agent: BLEXBot
User-agent: PetalBot
User-agent: DataForSeoBot
User-agent: MegaIndex
Disallow: /

# Catch-all — allow with positive Content-Signal and slower crawl rate
User-agent: *
Content-Signal: search=yes, ai-train=no, ai-input=yes
Allow: /
Disallow: /admin/
Disallow: /api/admin/
Disallow: /api/auth/
Disallow: /search
Disallow: /api/badges
Disallow: /api/reactions
Disallow: /compare
Disallow: /compare/*
Disallow: /api/compare/*
Disallow: /moon/compare?*
Disallow: /companies/compare?*
Disallow: /launches/compare?*
Crawl-delay: 5

Sitemap: https://spaceodysseyhub.com/sitemap.xml
Sitemap: https://spaceodysseyhub.com/news-sitemap.xml