# robots.txt for https://cernitur.com (Cernitur Technologies LLC)

# -----------------------------------------------------------------------------
# Frontier AI Answer Engines & Search Crawlers (Explicitly Permitted)
# -----------------------------------------------------------------------------

# Google (Search & AI Overviews)
User-agent: Googlebot
Allow: /

User-agent: Google-Extended
Allow: /

# Microsoft Bing & Copilot
User-agent: bingbot
Allow: /

# OpenAI (Search, Live Browse & Training)
User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: GPTBot
Allow: /

# Anthropic (Claude Search, Live Retrieval & Training)
User-agent: Claude-SearchBot
Allow: /

User-agent: Claude-User
Allow: /

User-agent: ClaudeBot
Allow: /

# Perplexity AI (Agent API & Search Engine)
User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

# Apple (Siri, Spotlight & Apple Intelligence)
User-agent: Applebot
Allow: /

User-agent: Applebot-Extended
Allow: /

# Meta AI (Llama Web Search & Ingestion)
User-agent: Meta-ExternalAgent
Allow: /

User-agent: Meta-ExternalFetcher
Allow: /

# Amazon (Amazon Q & Alexa Search)
User-agent: Amazonbot
Allow: /

# -----------------------------------------------------------------------------
# Default Rules & Disallow Boundaries
# -----------------------------------------------------------------------------
User-agent: *
Allow: /
Disallow: /api/
Disallow: /dashboard/

# -----------------------------------------------------------------------------
# Machine-Readable Endpoints
# AI agents can ingest the canonical Markdown summary at /llms.txt
# -----------------------------------------------------------------------------
Sitemap: https://cernitur.com/sitemap.xml