# robots.txt for estherderby.com
 # Goal: Reduce server load from bot traffic on
 resource-constrained VPS

 # Google - allow indexing with crawl delay
 User-agent: Googlebot
 Crawl-delay: 10
 Disallow: /wp-admin/
 Disallow: /wp-includes/
 Disallow: /wp-content/plugins/
 Disallow: /wp-content/cache/
 Disallow: /wp-json/
 Disallow: /feed/
 Disallow: /comments/feed/
 Disallow: /trackback/
 Disallow: /?s=
 Disallow: /search/
 Disallow: /refer/
 Disallow: /readme.html
 Allow: /wp-content/uploads/
 Allow: /wp-admin/admin-ajax.php

 # Bing
 User-agent: Bingbot
 Crawl-delay: 15
 Disallow: /wp-admin/
 Disallow: /wp-includes/
 Disallow: /wp-content/plugins/
 Disallow: /wp-content/cache/
 Disallow: /wp-json/
 Disallow: /feed/
 Disallow: /comments/feed/
 Disallow: /trackback/
 Disallow: /?s=
 Disallow: /search/
 Disallow: /refer/
 Disallow: /readme.html
 Allow: /wp-content/uploads/
 Allow: /wp-admin/admin-ajax.php

 # Known resource-heavy bots - block entirely
 User-agent: AhrefsBot
 Disallow: /

 User-agent: SemrushBot
 Disallow: /

 User-agent: DotBot
 Disallow: /

 User-agent: MJ12bot
 Disallow: /

 User-agent: BLEXBot
 Disallow: /

 User-agent: PetalBot
 Disallow: /

 User-agent: YandexBot
 Disallow: /

 User-agent: Bytespider
 Disallow: /

 User-agent: GPTBot
 Disallow: /

 User-agent: CCBot
 Disallow: /

 User-agent: ClaudeBot
 Disallow: /

 User-agent: Google-Extended
 Disallow: /

 User-agent: FacebookExternalHit
 Crawl-delay: 30
 Disallow: /wp-admin/

 # All other bots - aggressive throttling
 User-agent: *
 Crawl-delay: 30
 Disallow: /wp-admin/
 Disallow: /wp-includes/
 Disallow: /wp-content/plugins/
 Disallow: /wp-content/cache/
 Disallow: /wp-json/
 Disallow: /feed/
 Disallow: /comments/feed/
 Disallow: /trackback/
 Disallow: /?s=
 Disallow: /search/
 Disallow: /refer/
 Disallow: /readme.html
 Allow: /wp-content/uploads/
 Allow: /wp-admin/admin-ajax.php

 Sitemap: https://estherderby.com/sitemap_index.xml