# robots.txt for xuquan-ai.com
# Generated by OpenCMO Sitemap & Robot Ops
# Last updated: 2026-06-14

# =====================
# 通用搜索引擎（全部允许）
# =====================
User-agent: *
Allow: /

# =====================
# AI Agent 专用爬虫
# =====================

# OpenAI GPT 训练/索引
User-agent: GPTBot
Allow: /

# Anthropic Claude 索引
User-agent: ClaudeBot
Allow: /

# Google AI 训练（Google-Extended 覆盖 Gemini/Bard 训练源）
User-agent: Google-Extended
Allow: /

# Common Crawl（LLM 常用训练源）
User-agent: CCBot
Allow: /

# Perplexity 搜索索引
User-agent: PerplexityBot
Allow: /

# Amazon AI
User-agent: Amazonbot
Allow: /

# =====================
# AI 搜索/Agent 索引
# =====================

# You.com AI 搜索
User-agent: YouBot
Allow: /

# Facebook/Meta AI
User-agent: Meta-ExternalAgent
Allow: /

# ByteDance 爬虫
User-agent: Bytespider
Allow: /

# Apple AI
User-agent: Applebot-Extended
Allow: /

# =====================
# 非必要抓取（限制）
# =====================

User-agent: SemrushBot
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

# =====================
# Sitemap 指向
# =====================

Sitemap: https://xuquan-ai.com/sitemap.xml