# =============================================
# robots.txt — crazytime.es
# Priority: Google > standard search engines > others
# =============================================

# ============================================
# GOOGLE — приоритет (no crawl-delay, full access)
# ============================================

User-agent: Googlebot
Allow: /
Allow: /assets/
Disallow: /404
Disallow: /404.html
Disallow: /_headers
Disallow: /_redirects
Disallow: /*?*

User-agent: Googlebot-Image
Allow: /
Allow: /assets/

User-agent: Googlebot-News
Allow: /

User-agent: Googlebot-Video
Allow: /

User-agent: Mediapartners-Google
Allow: /

User-agent: AdsBot-Google
Allow: /

User-agent: Google-InspectionTool
Allow: /

# ============================================
# BING — стандартные правила + crawl-delay
# ============================================

User-agent: Bingbot
Allow: /
Disallow: /404
Disallow: /*?*
Crawl-delay: 5

User-agent: msnbot
Allow: /
Crawl-delay: 5

User-agent: BingPreview
Allow: /

# ============================================
# ДРУГИЕ ПОИСКОВЫЕ БОТЫ — стандартные правила + crawl-delay
# ============================================

User-agent: Slurp
Allow: /
Disallow: /*?*
Crawl-delay: 5

User-agent: DuckDuckBot
Allow: /
Crawl-delay: 3

User-agent: YandexBot
Allow: /
Disallow: /*?*
Crawl-delay: 5

User-agent: Baiduspider
Allow: /
Crawl-delay: 10

# ============================================
# AI БОТЫ (для AI Overviews / ChatGPT / Claude / etc)
# Разрешаем индексировать контент, но не редиректы
# ============================================

User-agent: Google-Extended
Allow: /

User-agent: GPTBot
Allow: /
Crawl-delay: 2

User-agent: ChatGPT-User
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Applebot
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: Amazonbot
Allow: /
Crawl-delay: 2

User-agent: Bytespider
Allow: /
Crawl-delay: 5

# ============================================
# БЛОКИРУЕМ — агрессивные SEO-краулеры
# (жрут трафик, продают данные конкурентам)
# ============================================

User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: SemrushBot-SA
Disallow: /

User-agent: SemrushBot-BA
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: rogerbot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: SeekportBot
Disallow: /

User-agent: serpstatbot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

# ============================================
# БЛОКИРУЕМ — спам/email скраперы
# ============================================

User-agent: SiteAuditBot
Disallow: /

User-agent: ZoominfoBot
Disallow: /

User-agent: linkfluence_bot
Disallow: /

User-agent: ImagesiftBot
Disallow: /

User-agent: ChatGPT
Disallow: /

User-agent: CCBot
Disallow: /

# ============================================
# DEFAULT — все остальные боты (fallback)
# ============================================

User-agent: *
Allow: /
Disallow: /404
Disallow: /404.html
Disallow: /_headers
Disallow: /_redirects
Disallow: /*?*
Disallow: /*&*
Crawl-delay: 10

# ============================================
# Sitemap
# ============================================

Sitemap: https://www.crazytime.es/sitemap.xml