User-agent: *
Crawl-delay: 10
Disallow: /heavy-page/


User-agent: ClaudeBot
Disallow: /

User-agent: Claude-User
Disallow: /

User-agent: Claude-SearchBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

# Allow major search engines
User-agent: Googlebot
Disallow:
User-agent: Bingbot
Disallow:
User-agent: DuckDuckBot
Disallow:

# Block AI and SEO scrapers
User-agent: ClaudeBot
Disallow: /
User-agent: Claude-User
Disallow: /
User-agent: Claude-SearchBot
Disallow: /
User-agent: SemrushBot
Disallow: /
User-agent: DataForSeoBot
Disallow: /
User-agent: AhrefsBot
Disallow: /
User-agent: MJ12bot
Disallow: /
User-agent: DotBot
Disallow: /

# Optional: generic AI opt-outs
User-agent: GPTBot
Disallow: /
User-agent: ChatGPT-User
Disallow: /
User-agent: OmgiliBot
Disallow: /
User-agent: FacebookBot
Disallow: /
User-agent: Amazonbot
Disallow: /
User-agent: SemrushBot
Disallow: /
User-agent: PerplexityBot
Disallow: /