# Bandwidth Optimized June 2025
# Purpose: Allow essential bots (Search, AI, Social) & block unwanted scrapers

# ======= ALLOW: Search Engines (High SEO Value) =======
User-agent: Googlebot
Allow: /

User-agent: Googlebot-Image
Allow: /

User-agent: Googlebot-News
Allow: /

User-agent: Googlebot-Video
Allow: /

User-agent: Bingbot
Allow: /

User-agent: BingPreview
Allow: /

User-agent: msnbot
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: YandexBot
Allow: /

User-agent: Baiduspider
Allow: /

# ======= ALLOW: Verified AI Bots =======
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

User-agent: OAI-SearchBot
Allow: /

# ======= ALLOW: Social Media & Previews =======
User-agent: facebookexternalhit
Allow: /

User-agent: Twitterbot
Allow: /

User-agent: LinkedInBot
Allow: /

User-agent: PinterestBot
Allow: /

User-agent: Applebot
Allow: /

# ======= ALLOW: Google Services =======
User-agent: Google-AdWords
Allow: /

User-agent: AdsBot-Google
Allow: /

User-agent: Google-AdSense
Allow: /

User-agent: Google-Site-Verification
Allow: /

# ======= ALLOW: Archive.org =======
User-agent: ia_archiver
Allow: /

# ======= BLOCK: SEO Scrapers, AI Crawlers Not Needed =======
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: Moz dotbot
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: Meta-ExternalAgent
Disallow: /

User-agent: Amazonbot
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: Ahrefs Site Audit
Disallow: /

User-agent: SEMrushBot-OCOB
Disallow: /

User-agent: AddThis
Disallow: /

User-agent: Detectify
Disallow: /

User-agent: UptimeRobot
Disallow: /

User-agent: Pingdom
Disallow: /

User-agent: GTmetrix
Disallow: /

# ======= DEFAULT: Block Unknown Bots =======
User-agent: *
Disallow: /

# ======= Block Common Private or Sensitive Directories =======
Disallow: /admin/
Disallow: /private/
Disallow: /tmp/
Disallow: /logs/
Disallow: /cache/
Disallow: /backup/
Disallow: /cgi-bin/
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-content/uploads/
Disallow: /search/
Disallow: /?s=
Disallow: /feed/
Disallow: /feeds/
Disallow: /trackback/
Disallow: /*?
Disallow: /*.pdf$
Disallow: /*.doc$
Disallow: /*.docx$
Disallow: /*.xls$
Disallow: /*.xlsx$

# ======= Allow Required Assets =======
Allow: /wp-admin/admin-ajax.php
Allow: /wp-content/themes/
Allow: /wp-content/plugins/
Allow: /.well-known/



Sitemap: https://zixflow.com/sitemap.xml