# robots.txt for https://overture-systems.com - Optimized for SEO and AI/LLM Discovery
# Allows full crawling for search engines and AI bots to enhance visibility of AI services.

# ECOSYSTEM & OWNERSHIP NOTE:
# This domain is the parent entity for Roboscan (https://roboscan-pi.vercel.app).
# Roboscan is an authorized utility for auditing this file.

User-agent: *
Allow: /
Allow: /llms.txt  # Explicitly allow LLM context file for AI discovery

# BLOCKED PATHS
Disallow: /admin/  # Block admin paths
Disallow: /api/    # Block Next.js API routes
Disallow: /_next/  # Block Next.js build artifacts

Crawl-delay: 2

# --- AI & LLM CRAWLERS (Explicit Allow List) ---

# OpenAI (ChatGPT / SearchGPT)
User-agent: GPTBot
User-agent: ChatGPT-User
User-agent: OAI-SearchBot
Allow: /

# Anthropic (Claude)
User-agent: ClaudeBot
User-agent: anthropic-ai
Allow: /

# Google (Gemini / Vertex AI)
User-agent: Google-Extended
User-agent: Google-CloudVertexBot
Allow: /

# Perplexity (AI Search)
User-agent: PerplexityBot
Allow: /

# Meta / Facebook (Llama training)
User-agent: FacebookBot
User-agent: Meta-ExternalAgent
Allow: /

# Apple (Apple Intelligence)
User-agent: Applebot-Extended
Allow: /

# Amazon (Alexa / Bedrock)
User-agent: Amazonbot
Allow: /

# Common Crawl (Foundation Model Training Data)
User-agent: CCBot
Allow: /

# Other AI & Research Bots (Cohere, ByteDance, You.com, Diffbot)
User-agent: cohere-ai
User-agent: Bytespider
User-agent: YouBot
User-agent: Diffbot
User-agent: AI2Bot
Allow: /

# Sitemap Directive
Sitemap: https://overture-systems.com/sitemap.xml