# =============================================================
# Search engines - full access
# =============================================================
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: DuckDuckBot
Allow: /

# =============================================================
# AI answer engines - ALLOWED on purpose.
# These cite and link back, so they send traffic.
# =============================================================
User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: Claude-User
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: YouBot
Allow: /

# =============================================================
# AI TRAINING crawlers - blocked.
# These consume content without citation or referral traffic.
# Blocking these has no effect on search ranking:
#   Google-Extended    -> training only, separate from Googlebot
#   Applebot-Extended  -> training only, separate from Applebot
# =============================================================
User-agent: GPTBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: meta-externalagent
Disallow: /

User-agent: Meta-ExternalAgent
Disallow: /

User-agent: Amazonbot
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: AI2Bot
Disallow: /

User-agent: Diffbot
Disallow: /

User-agent: Omgilibot
Disallow: /

User-agent: omgili
Disallow: /

User-agent: Timpibot
Disallow: /

User-agent: ImagesiftBot
Disallow: /

User-agent: Scrapy
Disallow: /

# =============================================================
# Everything else
# =============================================================
User-agent: *
Allow: /

Sitemap: https://andreyka26.com/sitemap.xml