# ================================
# Robots.txt for https://www.convertcake.com
# Full access for search engines & AI crawlers
# ================================

# --- Google ---
User-agent: Googlebot
Allow: /

User-agent: Googlebot-Image
Allow: /

User-agent: Googlebot-News
Allow: /

User-agent: Googlebot-Video
Allow: /

# --- Bing ---
User-agent: Bingbot
Allow: /

# --- AI / LLM Crawlers ---
User-agent: GPTBot          # OpenAI / ChatGPT
Allow: /

User-agent: ChatGPT-User    # ChatGPT browsing mode
Allow: /

User-agent: PerplexityBot   # Perplexity AI
Allow: /

User-agent: ClaudeBot       # Anthropic Claude
Allow: /

User-agent: anthropic-ai    # Anthropic crawler variant
Allow: /

User-agent: CCBot           # Common Crawl (used by many AI datasets)
Allow: /

# --- Other major engines ---
User-agent: DuckDuckBot
Allow: /

User-agent: Baiduspider
Allow: /

User-agent: Yandex
Allow: /

User-agent: Applebot
Allow: /

User-agent: ia_archiver     # Archive.org
Allow: /

# --- SEO tools (allowed) ---
User-agent: AhrefsBot
Allow: /

# --- Special files ---
Allow: /llms.txt

# ================================
# Block known spammy scrapers
# ================================
User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: Scrapy
Disallow: /

User-agent: curl
Disallow: /

User-agent: wget
Disallow: /

# ================================
# Default rule for everything else
# ================================
User-agent: *
Disallow: /wp-admin/          # keep admin private
Allow: /wp-admin/admin-ajax.php
Allow: /

# ================================
# Sitemap
# ================================
Sitemap: https://www.convertcake.com/sitemap.xml