User-Agent: Googlebot
Allow: /

User-Agent: Bingbot
Allow: /

User-Agent: DuckDuckBot
Allow: /

User-Agent: Applebot
Allow: /

User-Agent: Baiduspider
Allow: /

User-Agent: YandexBot
Allow: /

# Block AI model training bots
User-Agent: GPTBot
Disallow: /

User-Agent: ChatGPT-User
Disallow: /

User-Agent: ClaudeBot
Disallow: /

User-Agent: Claude-Web
Disallow: /

User-Agent: Meta-ExternalAgent
Disallow: /

User-Agent: PerplexityBot
Disallow: /

User-Agent: CCBot
Disallow: /

# Block generic crawlers that may be used for datasets
User-Agent: ia_archiver
Disallow: /

# Default rule for unknown AI crawlers
User-Agent: *
Disallow: /