# https://www.spekboom.org robots.txt

# ===========================================
# Default rules for all crawlers
# ===========================================
User-agent: *
# Disallow admin, API, and internal routes
Disallow: /api/
Disallow: /admin/
Disallow: /manage/
Disallow: /claim/
Disallow: /register/
Disallow: /_next/
Disallow: /static/
Disallow: /directory
# Allow important public pages
Allow: /about
# MCP connector docs — keep discoverable so the Spekboom AI connector
# (Claude, ChatGPT, Perplexity) surfaces in search and AI answers.
Allow: /mcp
# Developer & agent resource hub (MCP server, OAuth scopes, llms.txt index)
Allow: /developers
Allow: /our-story
Allow: /contact
Allow: /places/
Allow: /search
Allow: /trust
Allow: /causes/
Allow: /explore/
Allow: /guides/
Allow: /stay/
Allow: /ask-dassie
Allow: /plan-a-trip
Allow: /routes/
Allow: /journeys
Allow: /regenerative-travel
Allow: /gobirding
Allow: /faq
Allow: /help

# ===========================================
# Sitemaps
# ===========================================
Sitemap: https://www.spekboom.org/sitemap.xml
Sitemap: https://www.spekboom.org/image-sitemap.xml

# ===========================================
# LLM Context Files
# ===========================================
# llms.txt: https://www.spekboom.org/llms.txt
# llms-full.txt: https://www.spekboom.org/llms-full.txt

# ===========================================
# Search Engine Crawlers
# ===========================================

# Google
# NOTE: a specific user-agent group REPLACES the * group for that bot, so the
# private routes must be repeated here; until 2026-11-04 these groups only
# blocked /directory and /api/go/, which left /api/, /admin/, /manage/ and
# /claim/ open to the main search engines. /_next/ stays crawlable for them:
# Google needs the scripts and styles to render the page.
User-agent: Googlebot
Allow: /
Disallow: /directory
Disallow: /api/
Disallow: /admin/
Disallow: /manage/
Disallow: /claim/
Disallow: /register/

# Bing / Microsoft Copilot
User-agent: Bingbot
Allow: /
Disallow: /directory
Disallow: /api/
Disallow: /admin/
Disallow: /manage/
Disallow: /claim/
Disallow: /register/

# Apple (Siri, Spotlight)
User-agent: Applebot
Allow: /
Disallow: /directory
Disallow: /api/
Disallow: /admin/
Disallow: /manage/
Disallow: /claim/
Disallow: /register/

# ===========================================
# AI/LLM Crawlers — Allowed
# We welcome AI crawlers to help users discover Spekboom
# ===========================================
# A named group REPLACES the * group for that bot, so each one repeats the
# private-route disallows; `Allow: /` alone opened /api/ and /manage/ to them.

# OpenAI GPTBot (training + search)
User-agent: GPTBot
Allow: /
Disallow: /directory
Disallow: /api/
Disallow: /admin/
Disallow: /manage/
Disallow: /claim/

# OpenAI Search (ChatGPT search results)
User-agent: OAI-SearchBot
Allow: /
Disallow: /directory
Disallow: /api/
Disallow: /admin/
Disallow: /manage/
Disallow: /claim/

# ChatGPT live browsing
User-agent: ChatGPT-User
Allow: /
Disallow: /directory
Disallow: /api/
Disallow: /admin/
Disallow: /manage/
Disallow: /claim/

# Anthropic ClaudeBot (training)
User-agent: ClaudeBot
Allow: /
Disallow: /directory
Disallow: /api/
Disallow: /admin/
Disallow: /manage/
Disallow: /claim/
User-agent: anthropic-ai
Allow: /
Disallow: /directory
Disallow: /api/
Disallow: /admin/
Disallow: /manage/
Disallow: /claim/

# Anthropic Claude search
User-agent: Claude-SearchBot
Allow: /
Disallow: /directory
Disallow: /api/
Disallow: /admin/
Disallow: /manage/
Disallow: /claim/

# Anthropic Claude live browsing
User-agent: Claude-User
Allow: /
Disallow: /directory
Disallow: /api/
Disallow: /admin/
Disallow: /manage/
Disallow: /claim/

# Google AI (Gemini training)
User-agent: Google-Extended
Allow: /
Disallow: /directory
Disallow: /api/
Disallow: /admin/
Disallow: /manage/
Disallow: /claim/

# Perplexity AI
User-agent: PerplexityBot
Allow: /
Disallow: /directory
Disallow: /api/
Disallow: /admin/
Disallow: /manage/
Disallow: /claim/

# Perplexity live browsing / citation fetcher
User-agent: Perplexity-User
Allow: /
Disallow: /directory
Disallow: /api/
Disallow: /admin/
Disallow: /manage/
Disallow: /claim/

# Common Crawl (used by many AI models)
User-agent: CCBot
Allow: /
Disallow: /directory
Disallow: /api/
Disallow: /admin/
Disallow: /manage/
Disallow: /claim/

# Meta FacebookBot (link previews)
User-agent: FacebookBot
Allow: /
Disallow: /directory
Disallow: /api/
Disallow: /admin/
Disallow: /manage/
Disallow: /claim/

# Cohere AI
User-agent: cohere-ai
Allow: /
Disallow: /directory
Disallow: /api/
Disallow: /admin/
Disallow: /manage/
Disallow: /claim/

# Amazon / Alexa
User-agent: Amazonbot
Allow: /
Disallow: /directory
Disallow: /api/
Disallow: /admin/
Disallow: /manage/
Disallow: /claim/

# Apple AI training (Siri, Apple Intelligence)
User-agent: Applebot-Extended
Allow: /
Disallow: /directory
Disallow: /api/
Disallow: /admin/
Disallow: /manage/
Disallow: /claim/

# ===========================================
# AI/LLM Crawlers — Blocked (aggressive training-only)
# ===========================================

# ByteDance / TikTok (aggressive training crawler)
User-agent: Bytespider
Disallow: /

# Meta AI training (separate from FacebookBot link previews)
User-agent: Meta-ExternalAgent
Disallow: /