# synterminal.com — robots.txt
# Record. Measure. Verify.

# ── DEFAULT ──────────────────────────────────────────────────
User-agent: *
Allow: /
Disallow: /node_modules/
Disallow: /.git/
Crawl-delay: 1

# ── SITEMAPS ─────────────────────────────────────────────────
Sitemap: https://synterminal.com/sitemap.xml
Sitemap: https://synterminal.com/sitemap-research.xml
Sitemap: https://synterminal.com/llms.txt

# ── AI CRAWLERS — EXPLICITLY WELCOME ─────────────────────────

# OpenAI — training
User-agent: GPTBot
Allow: /
Crawl-delay: 0

# OpenAI — ChatGPT search
User-agent: OAI-SearchBot
Allow: /
Crawl-delay: 0

# OpenAI — live URL fetch
User-agent: ChatGPT-User
Allow: /
Crawl-delay: 0

# Anthropic — training
User-agent: ClaudeBot
Allow: /
Crawl-delay: 0

# Anthropic — live fetch
User-agent: Claude-User
Allow: /
Crawl-delay: 0

# Google — AI Overviews / Gemini grounding
User-agent: Google-Extended
Allow: /
Crawl-delay: 0

# Perplexity
User-agent: PerplexityBot
Allow: /
Crawl-delay: 0

# Apple
User-agent: Applebot-Extended
Allow: /
Crawl-delay: 0

# Amazon
User-agent: Amazonbot
Allow: /
Crawl-delay: 0

# You.com
User-agent: YouBot
Allow: /
Crawl-delay: 0

# Cohere
User-agent: cohere-ai
Allow: /
Crawl-delay: 0

# Meta
User-agent: meta-externalagent
Allow: /
Crawl-delay: 0

# Diffbot
User-agent: Diffbot
Allow: /
Crawl-delay: 0